@caelo-cms/provisioning 0.10.53 → 0.10.55

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@caelo-cms/provisioning",
-  "version": "0.10.53",
+  "version": "0.10.55",
   "private": false,
   "license": "MPL-2.0",
   "description": "Pulumi-driven provisioning for Caelo CMS — `bunx @caelo-cms/provisioning --provider <self-hosted|gcp|aws|azure>` brings up a TLS-served install with Postgres + storage + secrets manager + the admin + the API gateway.",
@@ -40,16 +40,16 @@
     "prepublishOnly": "tsc -b --force && bun build ../platform-updater/src/agent-db.ts --target=bun --outfile=dist/agent-db.js && tsc --project stacks/gcp/tsconfig.json && tsc --project stacks/gcp-firebase/tsconfig.json"
   },
   "dependencies": {
-    "@caelo-cms/shared": "^0.10.53",
+    "@caelo-cms/shared": "^0.10.55",
     "@clack/prompts": "1.8.1",
     "@pulumi/command": "^1",
     "@pulumi/gcp": "^10",
     "@pulumi/pulumi": "^3",
-    "@pulumi/random": "^4.21.2",
+    "@pulumi/random": "^4.21.4",
     "kleur": "4.1.5",
     "ts-node": "^10",
     "typescript": "^5",
-    "zod": "4.6.5"
+    "zod": "4.6.7"
   },
   "devDependencies": {
     "@pulumi/aws": "^7",

package/src/agent/run.test.ts

@@ -35,6 +35,7 @@ const POLICY: StoredUpdatePolicy = {
   autoSecurity: false,
   autoPatch: false,
   autoMinor: false,
+  autoConverge: false,
   channel: "stable",
   windowStart: null,
   windowEnd: null,

package/src/audit-job-setup.test.ts

@@ -0,0 +1,234 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * The quality-audit job's setup (audit-job-setup.ts) against a fake gcloud.
+ * The load-bearing assertions are the trust boundary (#875): the job's
+ * service account may create results, read the staged build (gcp only) and
+ * write logs — no secret, database, IAM or run rights, no VPC — and the
+ * admin may run the job with overrides, read its executions and read the
+ * results, nothing else.
+ */
+
+import { describe, expect, it } from "bun:test";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import {
+  AUDIT_FORBIDDEN_ROLES,
+  AUDIT_PARALLELISM_DEFAULT,
+  AUDIT_TASK_SCRIPT,
+  type AuditJobTarget,
+  ensureAuditJob,
+} from "./audit-job-setup.js";
+import type { GcloudResult } from "./gcloud.js";
+
+const ok = (stdout = ""): GcloudResult => ({ ok: true, stdout, stderr: "", exitCode: 0 });
+const fail = (stderr: string): GcloudResult => ({ ok: false, stdout: "", stderr, exitCode: 1 });
+
+const AUDIT = "serviceAccount:caelo-production-audit@acme.iam.gserviceaccount.com";
+const ADMIN_SA = "serviceAccount:caelo-production-run-sa@acme.iam.gserviceaccount.com";
+const ADMIN_IMAGE = `europe-west1-docker.pkg.dev/caelo-website/caelo-cms-images/admin@sha256:${"a".repeat(64)}`;
+const RESULTS = "acme-caelo-production-audit-results";
+
+const TARGET: AuditJobTarget = {
+  provider: "gcp",
+  projectId: "acme",
+  region: "europe-west1",
+  env: "production",
+  adminImageRef: ADMIN_IMAGE,
+  parallelism: 15,
+};
+
+function fakeGcloud(overrides: [RegExp, GcloudResult][] = []) {
+  const calls: string[][] = [];
+  const rules: [RegExp, GcloudResult][] = [
+    ...overrides,
+    [/^iam service-accounts describe/, fail("NOT_FOUND")],
+    [/^storage buckets list/, ok("gs://acme-caelo-production-static/\n")],
+  ];
+  const run = async (args: string[]) => {
+    calls.push(args);
+    const line = args.join(" ");
+    return rules.find(([re]) => re.test(line))?.[1] ?? ok("");
+  };
+  return { run, calls };
+}
+
+const flag = (args: string[], name: string) =>
+  args.find((a) => a.startsWith(`--${name}=`))?.slice(name.length + 3);
+
+/** Every IAM grant the setup made, as {where, member, role}. */
+function grants(calls: string[][]) {
+  return calls
+    .filter((c) => c.includes("add-iam-policy-binding"))
+    .map((c) => ({
+      where: c.slice(0, c.indexOf("add-iam-policy-binding") + 2).join(" "),
+      member: flag(c, "member"),
+      role: flag(c, "role") as string,
+    }));
+}
+
+const deployOf = (calls: string[][]) =>
+  calls.find((c) => c[0] === "run" && c[1] === "jobs" && c[2] === "deploy") as string[];
+
+describe("ensureAuditJob", () => {
+  it("grants the job only results-create, build-read (gcp) and logs", async () => {
+    const { run, calls } = fakeGcloud();
+    const r = await ensureAuditJob(TARGET, { run, sleep: async () => {} });
+    expect(r.ok).toBe(true);
+    const job = grants(calls).filter((g) => g.member === AUDIT);
+    expect(job.map((g) => [g.where, g.role])).toEqual([
+      [`storage buckets add-iam-policy-binding gs://${RESULTS}`, "roles/storage.objectCreator"],
+      [
+        "storage buckets add-iam-policy-binding gs://acme-caelo-production-staging",
+        "roles/storage.objectViewer",
+      ],
+      [
+        "storage buckets add-iam-policy-binding gs://acme-caelo-production-static",
+        "roles/storage.objectViewer",
+      ],
+      ["projects add-iam-policy-binding acme", "roles/logging.logWriter"],
+    ]);
+    for (const g of job) {
+      for (const forbidden of AUDIT_FORBIDDEN_ROLES)
+        expect(g.role.startsWith(forbidden)).toBe(false);
+    }
+  });
+
+  it("gives the admin run-with-overrides and view on the job and read on the results, nothing else", async () => {
+    const { run, calls } = fakeGcloud();
+    await ensureAuditJob(TARGET, { run, sleep: async () => {} });
+    expect(
+      grants(calls)
+        .filter((g) => g.member === ADMIN_SA)
+        .map((g) => [g.where, g.role]),
+    ).toEqual([
+      [`storage buckets add-iam-policy-binding gs://${RESULTS}`, "roles/storage.objectViewer"],
+      [
+        "run jobs add-iam-policy-binding caelo-production-audit",
+        "roles/run.jobsExecutorWithOverrides",
+      ],
+      ["run jobs add-iam-policy-binding caelo-production-audit", "roles/run.viewer"],
+    ]);
+  });
+
+  it("points the job at a task script the admin image ships", () => {
+    // The image copies the repo's packages/ to /app/packages (apps/admin/Dockerfile);
+    // a moved script would only fail when the first audit starts.
+    const repoRoot = join(import.meta.dir, "..", "..", "..");
+    expect(AUDIT_TASK_SCRIPT.startsWith("/app/packages/")).toBe(true);
+    expect(existsSync(join(repoRoot, AUDIT_TASK_SCRIPT.slice("/app/".length)))).toBe(true);
+  });
+
+  it("deploys the admin image as its own SA with no secrets and no VPC", async () => {
+    const { run, calls } = fakeGcloud();
+    await ensureAuditJob(TARGET, { run, sleep: async () => {} });
+    const deploy = deployOf(calls);
+    expect(flag(deploy, "image")).toBe(ADMIN_IMAGE);
+    expect(flag(deploy, "service-account")).toBe(
+      "caelo-production-audit@acme.iam.gserviceaccount.com",
+    );
+    expect(flag(deploy, "args")).toBe(`--bun,${AUDIT_TASK_SCRIPT}`);
+    expect(flag(deploy, "max-retries")).toBe("0");
+    expect(flag(deploy, "parallelism")).toBe("15");
+    expect(flag(deploy, "set-env-vars")).toBe(
+      `CAELO_PROVIDER=gcp,CAELO_AUDIT_RESULTS_BUCKET=${RESULTS},CAELO_STAGING_BUCKET=acme-caelo-production-staging,CAELO_STATIC_BUCKET=acme-caelo-production-static`,
+    );
+    const line = deploy.join(" ");
+    for (const banned of ["--network", "--subnet", "--vpc", "--set-secrets", "--update-secrets"]) {
+      expect(line).not.toContain(banned);
+    }
+  });
+
+  it("creates the private results bucket with a one-day lifecycle when it is missing", async () => {
+    const { run, calls } = fakeGcloud();
+    await ensureAuditJob(TARGET, { run, sleep: async () => {} });
+    const create = calls.find(
+      (c) => c.slice(0, 3).join(" ") === "storage buckets create",
+    ) as string[];
+    expect(create).toContain(`gs://${RESULTS}`);
+    expect(create).toContain("--public-access-prevention");
+    expect(create).toContain("--uniform-bucket-level-access");
+    expect(calls.some((c) => c.join(" ").includes("--lifecycle-file="))).toBe(true);
+  });
+
+  it("keeps an existing bucket and an operator's parallelism on converge", async () => {
+    const { run, calls } = fakeGcloud([
+      [/^storage buckets list/, ok(`gs://${RESULTS}/\n`)],
+      [/^run jobs describe caelo-production-audit /, ok(`${ADMIN_IMAGE}\n`)],
+    ]);
+    const { parallelism: _, ...converge } = TARGET;
+    expect((await ensureAuditJob(converge, { run, sleep: async () => {} })).ok).toBe(true);
+    expect(calls.some((c) => c.slice(0, 3).join(" ") === "storage buckets create")).toBe(false);
+    expect(flag(deployOf(calls), "parallelism")).toBeUndefined();
+  });
+
+  it("gives a job the converge creates the default parallelism, as a fresh install's", async () => {
+    const { run, calls } = fakeGcloud([
+      [/^run jobs describe/, fail("ERROR: Cannot find job [caelo-production-audit]. NOT_FOUND")],
+    ]);
+    const { parallelism: _, ...converge } = TARGET;
+    const r = await ensureAuditJob(converge, { run, sleep: async () => {} });
+    expect(r.ok).toBe(true);
+    expect(flag(deployOf(calls), "parallelism")).toBe(String(AUDIT_PARALLELISM_DEFAULT));
+  });
+
+  it("refuses to redeploy a job it cannot read on converge", async () => {
+    const { run, calls } = fakeGcloud([[/^run jobs describe/, fail("PERMISSION_DENIED")]]);
+    const { parallelism: _, ...converge } = TARGET;
+    const r = await ensureAuditJob(converge, { run, sleep: async () => {} });
+    expect(r).toMatchObject({ ok: false, error: expect.stringContaining("PERMISSION_DENIED") });
+    expect(deployOf(calls)).toBeUndefined();
+  });
+
+  it("on gcp-firebase reads no bucket: the task audits the public preview channel", async () => {
+    const { run, calls } = fakeGcloud();
+    await ensureAuditJob({ ...TARGET, provider: "gcp-firebase" }, { run, sleep: async () => {} });
+    const job = grants(calls).filter((g) => g.member === AUDIT);
+    expect(job.map((g) => g.role)).toEqual([
+      "roles/storage.objectCreator",
+      "roles/logging.logWriter",
+    ]);
+    expect(flag(deployOf(calls), "set-env-vars")).toBe(
+      `CAELO_PROVIDER=gcp-firebase,CAELO_AUDIT_RESULTS_BUCKET=${RESULTS}`,
+    );
+  });
+
+  it("refuses an image that is not pinned by digest", async () => {
+    const { run, calls } = fakeGcloud();
+    const r = await ensureAuditJob({ ...TARGET, adminImageRef: "x/admin:latest" }, { run });
+    expect(r.ok).toBe(false);
+    expect(calls).toEqual([]);
+  });
+
+  it("refuses a parallelism outside 1-15", async () => {
+    const { run } = fakeGcloud();
+    const r = await ensureAuditJob({ ...TARGET, parallelism: 16 }, { run });
+    expect(r).toMatchObject({ ok: false, error: expect.stringContaining("1 to 15") });
+  });
+
+  it("stops at the first failed grant and names it", async () => {
+    const { run, calls } = fakeGcloud([
+      [
+        /^storage buckets add-iam-policy-binding gs:\/\/acme-caelo-production-audit-results --member=serviceAccount:caelo-production-audit/,
+        fail("PERMISSION_DENIED"),
+      ],
+    ]);
+    const r = await ensureAuditJob(TARGET, { run, sleep: async () => {} });
+    expect(r).toMatchObject({
+      ok: false,
+      error: expect.stringContaining("roles/storage.objectCreator"),
+    });
+    expect(calls.some((c) => c[2] === "deploy")).toBe(false);
+  });
+
+  it("says when the bucket name is taken by another project", async () => {
+    const { run } = fakeGcloud([
+      [/^storage buckets create/, fail("HTTPError 409: already exists")],
+    ]);
+    const r = await ensureAuditJob(TARGET, { run, sleep: async () => {} });
+    expect(r).toMatchObject({
+      ok: false,
+      error: expect.stringContaining("does not belong to project acme"),
+    });
+  });
+});

package/src/audit-job-setup.ts

@@ -0,0 +1,360 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * #875 — set up (and converge) the parallel quality-audit job on a Google
+ * Cloud install (`gcp`, `gcp-firebase`). Run by the wizard after `pulumi up`
+ * and by `cms-provision upgrade`'s converge, both as the operator's own
+ * gcloud identity. App-roll moves the job's image with the admin's
+ * (upgrade/app-roll.ts), because the job runs the admin image.
+ *
+ * Why a separate job at all (CLAUDE.md §11.B Tier 5 exception): Lighthouse
+ * is CPU-bound and simulates the CPU slowdown from observed task durations,
+ * so a run that shares a CPU with another run or with the editor's chat
+ * measures that interference too. One task per run measures the page alone,
+ * and all runs finish in about the time of one.
+ *
+ * Why the CLI owns it and not the stacks: `upgrade` never runs Pulumi, so
+ * existing installs get the job only from here, and a bucket or job both
+ * Pulumi and `upgrade` create would collide on the next wizard re-run (the
+ * same reasoning as operator-access.ts and upgrader-setup.ts).
+ *
+ * Trust boundary. The job's service account, `caelo-<env>-audit`, holds:
+ *
+ *   - `roles/storage.objectCreator` on the audit results bucket — create
+ *     only: it can neither read results back nor overwrite one;
+ *   - on `gcp` only, `roles/storage.objectViewer` on the staging and static
+ *     buckets: it serves the staged build to its own browser, as the admin's
+ *     audit does (admin-core quality/staged-origin.ts). gcp-firebase audits
+ *     the public preview channel and needs no bucket;
+ *   - `roles/logging.logWriter`.
+ *
+ * Nothing else: no Secret Manager, no Cloud SQL, and no VPC egress, so the
+ * database's private IP is not even routable from a task. Plugin data on a
+ * page comes the way any visitor gets it: the task's browser calls the
+ * site's public `/api/*` (the gateway, behind Cloud Armor) over the
+ * internet. The admin's run
+ * SA may run the job WITH overrides (`roles/run.jobsExecutorWithOverrides`:
+ * it passes each audit's batch of orders), read its executions
+ * (`roles/run.viewer`) and read the results bucket. Unlike the
+ * operator-access and upgrader jobs, overrides are acceptable here: a
+ * compromised admin can make the job load other URLs, but the job holds
+ * nothing worth taking. {@link AUDIT_FORBIDDEN_ROLES} names that line; the
+ * tests assert no grant crosses it.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { gcloud as defaultGcloud, type GcloudResult } from "./gcloud.js";
+import { type GcloudRunner, realSleep, runWithRetry, type Sleep } from "./gcloud-retry.js";
+import {
+  auditJobName,
+  auditServiceAccountEmail,
+  auditServiceAccountId,
+  gcpBucketName,
+  runServiceAccountEmail,
+} from "./gcp-names.js";
+import { readAdminConfig } from "./migration-runner.js";
+import type { GcpProvider } from "./stack-contract.js";
+import { describeAuditJobImage } from "./upgrade/audit-job-roll.js";
+
+/** Where the admin image ships the task's entry point. */
+export const AUDIT_TASK_SCRIPT = "/app/packages/admin-core/src/quality/audit-task.ts";
+
+/** How many audit tasks run at once by default: 5 pages x 3 runs. */
+export const AUDIT_PARALLELISM_DEFAULT = 15;
+
+/**
+ * Per task: container start, browser start, CPU benchmark and one
+ * Lighthouse run. Matches admin-core `parallelAuditTimeoutMs`, which the
+ * admin also passes per execution.
+ */
+const TASK_TIMEOUT = "5m";
+
+/**
+ * Roles the audit job's identity must never hold anywhere. Prefix matches
+ * (`roles/secretmanager.` = every Secret Manager role).
+ */
+export const AUDIT_FORBIDDEN_ROLES = [
+  "roles/owner",
+  "roles/editor",
+  "roles/run.",
+  "roles/iam.",
+  "roles/secretmanager.",
+  "roles/cloudsql.",
+  "roles/compute.",
+  "roles/resourcemanager.",
+  "roles/storage.admin",
+  "roles/storage.objectAdmin",
+  "roles/storage.objectUser",
+] as const;
+
+/** Delete results after a day: the admin reads them within minutes. */
+export const AUDIT_RESULTS_LIFECYCLE = {
+  rule: [{ action: { type: "Delete" }, condition: { age: 1 } }],
+};
+
+export interface AuditJobTarget {
+  readonly provider: GcpProvider;
+  readonly projectId: string;
+  readonly region: string;
+  /** Pulumi stack name (`production`). */
+  readonly env: string;
+  /** The admin image the job runs, digest-pinned (`…/admin@sha256:…`). */
+  readonly adminImageRef: string;
+  /**
+   * Tasks at once (`auditParallelism`, 1–15). Absent on converge: an
+   * existing job keeps the value it has, so an operator's choice survives
+   * `upgrade`; a job the converge creates gets
+   * {@link AUDIT_PARALLELISM_DEFAULT}, as a fresh install's does.
+   */
+  readonly parallelism?: number;
+}
+
+export type AuditJobResult =
+  | { readonly ok: true; readonly done: string[] }
+  | { readonly ok: false; readonly done: string[]; readonly error: string };
+
+/** Ensure the audit job and everything it needs. Never throws for a gcloud failure. */
+export async function ensureAuditJob(
+  target: AuditJobTarget,
+  deps: { run?: GcloudRunner; sleep?: Sleep } = {},
+): Promise<AuditJobResult> {
+  const run = deps.run ?? defaultGcloud;
+  const sleep = deps.sleep ?? realSleep;
+  const retry = (args: string[]) => runWithRetry(run, sleep, args);
+  const { projectId, region, env, provider } = target;
+  const project = `--project=${projectId}`;
+  const sa = auditServiceAccountEmail(projectId, env);
+  const member = `serviceAccount:${sa}`;
+  const runSaMember = `serviceAccount:${runServiceAccountEmail(projectId, env)}`;
+  const job = auditJobName(env);
+  const results = gcpBucketName(projectId, env, "audit-results");
+  const resultsUrl = `gs://${results}`;
+  const done: string[] = [];
+  const fail = (step: string, r: GcloudResult): AuditJobResult => ({
+    ok: false,
+    done,
+    error: `${step}: ${r.stderr.trim()}`,
+  });
+
+  if (!/@sha256:[0-9a-f]{64}$/.test(target.adminImageRef)) {
+    return {
+      ok: false,
+      done,
+      error: `the audit job's image must be pinned by digest, got ${target.adminImageRef}`,
+    };
+  }
+  if (
+    target.parallelism !== undefined &&
+    (!Number.isInteger(target.parallelism) || target.parallelism < 1 || target.parallelism > 15)
+  ) {
+    return {
+      ok: false,
+      done,
+      error: `auditParallelism must be a whole number from 1 to 15, got ${target.parallelism}`,
+    };
+  }
+
+  // 1. The job's own service account.
+  const describe = await run([
+    "iam",
+    "service-accounts",
+    "describe",
+    sa,
+    project,
+    "--format=value(email)",
+  ]);
+  if (!describe.ok) {
+    const create = await run([
+      "iam",
+      "service-accounts",
+      "create",
+      auditServiceAccountId(env),
+      "--display-name=Caelo quality audit",
+      "--description=Runs the quality audit's Lighthouse runs in parallel; reads the staged build, writes audit results, nothing else.",
+      project,
+    ]);
+    if (!create.ok && !/already exists/i.test(create.stderr)) return fail(`create ${sa}`, create);
+  }
+  done.push(`service account ${sa}`);
+
+  // 2. The results bucket: private, results deleted after a day.
+  const owned = await run(["storage", "buckets", "list", project, "--format=value(name)"]);
+  if (!owned.ok) return fail(`list the buckets of project ${projectId}`, owned);
+  const names = owned.stdout.split("\n").map((l) =>
+    l
+      .trim()
+      .replace(/^gs:\/\//, "")
+      .replace(/\/$/, ""),
+  );
+  if (!names.includes(results)) {
+    const created = await run([
+      "storage",
+      "buckets",
+      "create",
+      resultsUrl,
+      project,
+      `--location=${region}`,
+      "--default-storage-class=STANDARD",
+      "--uniform-bucket-level-access",
+      "--public-access-prevention",
+      "--soft-delete-duration=0",
+    ]);
+    if (!created.ok) {
+      return {
+        ok: false,
+        done,
+        error: /already exists|409/i.test(created.stderr)
+          ? `${resultsUrl} exists but does not belong to project ${projectId}; delete or rename that bucket, this install needs the name`
+          : `create ${resultsUrl}: ${created.stderr.trim()}`,
+      };
+    }
+  }
+  // A private directory (mkdtemp, mode 700): no other local user can swap
+  // the file between write and read.
+  const dir = mkdtempSync(join(tmpdir(), "caelo-audit-"));
+  try {
+    const file = join(dir, "lifecycle.json");
+    writeFileSync(file, JSON.stringify(AUDIT_RESULTS_LIFECYCLE), { mode: 0o600 });
+    const lifecycle = await run([
+      "storage",
+      "buckets",
+      "update",
+      resultsUrl,
+      project,
+      `--lifecycle-file=${file}`,
+    ]);
+    if (!lifecycle.ok) return fail(`set the lifecycle of ${resultsUrl}`, lifecycle);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+  done.push(`bucket ${resultsUrl} (results deleted after 1 day)`);
+
+  // 3. Bucket grants: the job creates results and (gcp) reads the build;
+  //    the admin reads results.
+  const bucketGrants: { bucket: string; member: string; role: string; why: string }[] = [
+    { bucket: results, member, role: "roles/storage.objectCreator", why: "tasks write results" },
+    ...(provider === "gcp"
+      ? (["staging", "static"] as const).map((b) => ({
+          bucket: gcpBucketName(projectId, env, b),
+          member,
+          role: "roles/storage.objectViewer",
+          why: "tasks serve the staged build to their browser",
+        }))
+      : []),
+    {
+      bucket: results,
+      member: runSaMember,
+      role: "roles/storage.objectViewer",
+      why: "the admin reads the results",
+    },
+  ];
+  for (const g of bucketGrants) {
+    const r = await retry([
+      "storage",
+      "buckets",
+      "add-iam-policy-binding",
+      `gs://${g.bucket}`,
+      `--member=${g.member}`,
+      `--role=${g.role}`,
+      project,
+      "--quiet",
+      "--format=none",
+    ]);
+    if (!r.ok) return fail(`grant ${g.role} on gs://${g.bucket} (${g.why})`, r);
+    done.push(`${g.member === member ? "audit job" : "admin"}: ${g.role} on gs://${g.bucket}`);
+  }
+
+  // 4. Logs.
+  const logs = await retry([
+    "projects",
+    "add-iam-policy-binding",
+    projectId,
+    `--member=${member}`,
+    "--role=roles/logging.logWriter",
+    "--condition=None",
+    "--quiet",
+    "--format=none",
+  ]);
+  if (!logs.ok) return fail("grant the audit job roles/logging.logWriter", logs);
+  done.push("audit job: roles/logging.logWriter on the project");
+
+  // 5. The job: the admin image, its own SA, no secrets, no VPC. A job
+  //    created without a parallelism would run every task at once with no
+  //    cap; an existing job keeps its own.
+  let parallelism = target.parallelism;
+  if (parallelism === undefined) {
+    const existing = await describeAuditJobImage(run, { projectId, region, env });
+    if (!existing.ok) return { ok: false, done, error: `read job ${job}: ${existing.error}` };
+    if (existing.image === null) parallelism = AUDIT_PARALLELISM_DEFAULT;
+  }
+  const envVars = [
+    `CAELO_PROVIDER=${provider}`,
+    `CAELO_AUDIT_RESULTS_BUCKET=${results}`,
+    ...(provider === "gcp"
+      ? [
+          `CAELO_STAGING_BUCKET=${gcpBucketName(projectId, env, "staging")}`,
+          `CAELO_STATIC_BUCKET=${gcpBucketName(projectId, env, "static")}`,
+        ]
+      : []),
+  ];
+  const deploy = await run([
+    "run",
+    "jobs",
+    "deploy",
+    job,
+    `--image=${target.adminImageRef}`,
+    `--region=${region}`,
+    project,
+    `--service-account=${sa}`,
+    "--command=bun",
+    `--args=--bun,${AUDIT_TASK_SCRIPT}`,
+    `--set-env-vars=${envVars.join(",")}`,
+    "--max-retries=0",
+    `--task-timeout=${TASK_TIMEOUT}`,
+    // Lighthouse wants two cores: the browser and the trace processing.
+    "--cpu=2",
+    "--memory=2Gi",
+    ...(parallelism !== undefined ? [`--parallelism=${parallelism}`] : []),
+    "--quiet",
+  ]);
+  if (!deploy.ok) return fail(`deploy job ${job}`, deploy);
+  done.push(`job ${job} (${target.adminImageRef.split("@")[1]?.slice(0, 19)}...)`);
+
+  // 6. The admin may run it with its batch and read its executions.
+  for (const role of ["roles/run.jobsExecutorWithOverrides", "roles/run.viewer"]) {
+    const r = await retry([
+      "run",
+      "jobs",
+      "add-iam-policy-binding",
+      job,
+      `--region=${region}`,
+      project,
+      `--member=${runSaMember}`,
+      `--role=${role}`,
+      "--quiet",
+      "--format=none",
+    ]);
+    if (!r.ok) return fail(`grant the admin ${role} on ${job}`, r);
+    done.push(`admin: ${role} on ${job}`);
+  }
+  return { ok: true, done };
+}
+
+/**
+ * {@link ensureAuditJob}'s image: the deployed admin's, so the job runs the
+ * code the admin runs.
+ */
+export async function resolveAdminImage(
+  opts: { readonly projectId: string; readonly region: string; readonly env: string },
+  deps: { run?: GcloudRunner } = {},
+): Promise<{ ok: true; imageRef: string } | { ok: false; error: string }> {
+  const cfg = await readAdminConfig(opts, deps.run ?? defaultGcloud);
+  if (!cfg)
+    return {
+      ok: false,
+      error: "could not read the admin service's image (is the admin deployed?)",
+    };
+  return { ok: true, imageRef: cfg.imageRef };
+}

package/src/caddy-redirects.docker.test.ts

@@ -47,6 +47,8 @@ const LIVE = "example.test";
 const STAGING = "staging.example.test";
 const CADDY_DIR = resolve(import.meta.dir, "../caddy");
 const SERVED = ["index.html", "about/index.html", "blog/index.html"];
+/** Consecutive matching answers that count as "the reload settled". */
+const SETTLED_PROBES = 5;
 
 function docker(args: string[]): { ok: boolean; out: string } {
   const r = Bun.spawnSync(["docker", ...args], { stdout: "pipe", stderr: "pipe" });
@@ -100,11 +102,25 @@ describe.skipIf(!dockerUp)(`self-hosted redirects through ${CADDY_IMAGE}`, () =>
     const location = head.match(/^location: ([^\r\n]*)/im)?.[1]?.trim() ?? null;
     return { status, location };
   };
+  /**
+   * Waits until `path` answers with `location` on {@link SETTLED_PROBES}
+   * consecutive connections. One matching answer is not enough: Caddy's
+   * graceful reload keeps the old config's listener open next to the new
+   * one (SO_REUSEPORT) while it drains, so the connection after the first
+   * new answer can still reach the old config (CI runs 38867358899 and
+   * 39242522450). Only a streak of new answers shows the reload settled.
+   */
   const eventually = async (host: string, path: string, location: string | null) => {
     const deadline = Date.now() + 30_000;
+    let streak = 0;
     for (;;) {
       const got = probe(host, path).location;
-      if (got === location) return;
+      streak = got === location ? streak + 1 : 0;
+      if (streak >= SETTLED_PROBES) return;
+      if (got === location) {
+        await Bun.sleep(200);
+        continue;
+      }
       if (Date.now() > deadline) {
         throw new Error(
           `${host}${path} → ${got}, expected ${location}: ${docker(["logs", name]).out}`,
@@ -241,11 +257,8 @@ describe.skipIf(!dockerUp)(`self-hosted redirects through ${CADDY_IMAGE}`, () =>
   it("the reload loop picks up a deploy's new snippet", async () => {
     write("production", snippet([["/brand-new", "/blog", 301]]));
     await eventually(LIVE, "/brand-new", "/blog/");
-    // Caddy's graceful reload keeps the old config's listener open for a
-    // moment next to the new one (SO_REUSEPORT), so a connection right
-    // after the first new answer can still reach the old config: the old
-    // redirect may answer once more (CI run 38867358899). What must hold is
-    // that the new snippet REPLACES the old one once the reload settled.
+    // What must hold is that the new snippet REPLACES the old one once the
+    // reload settled (see `eventually` for why one answer is not enough).
     await eventually(LIVE, "/old", null);
     expect(probe(LIVE, "/old").status).toBe(404);
     expect(probe(LIVE, "/brand-new").location).toBe("/blog/");