@caelo-cms/provisioning 0.10.53 → 0.10.55

package/src/tar-entries.test.ts

@@ -35,18 +35,160 @@ describe("listTarEntries", () => {
   });
 
   it("refuses a truncated archive instead of skipping members", () => {
-    const header = new Uint8Array(512);
-    header.set(new TextEncoder().encode("a.txt"), 0);
-    header.set(new TextEncoder().encode("00000001000\0"), 124); // 512 bytes of data, none present
-    header[156] = "0".charCodeAt(0);
-    expect(() => listTarEntries(header)).toThrow(/truncated/);
+    expect(() => listTarEntries(header("a.txt", "0", { sizeField: 512 }))).toThrow(/truncated/);
+  });
+
+  it("rejects PAX global extended headers ('g')", () => {
+    const g = member("PaxHeader", "g", paxLine("path=/etc/passwd"));
+    expect(() => listTarEntries(concat(g, member("safe.txt", "0"), END))).toThrow(
+      /pax global extended header/,
+    );
+  });
+
+  it("refuses a member named by two pax headers — GNU tar keeps the last, bsdtar drops the second", () => {
+    // Both tars extract `../../evil` here (the second header carries no
+    // path); reading the first header's path would have checked `safe.txt`.
+    const archive = concat(
+      member("PaxHeader", "x", paxLine("path=srv/caelo/safe.txt")),
+      member("PaxHeader", "x", paxLine("mtime=1")),
+      member("../../evil", "0"),
+      END,
+    );
+    expect(() => listTarEntries(archive)).toThrow(/several extension headers \(xx\)/);
+  });
+
+  it("refuses a GNU long name next to a pax path, and two long names", () => {
+    // bsdtar extracts the `L` name (`../../evil`), GNU tar the pax path.
+    const mixed = concat(
+      member("././@LongLink", "L", "../../evil\0"),
+      member("PaxHeader", "x", paxLine("path=srv/caelo/safe.txt")),
+      member("x", "0"),
+      END,
+    );
+    expect(() => listTarEntries(mixed)).toThrow(/several extension headers \(Lx\)/);
+    const twice = concat(
+      member("././@LongLink", "L", "../../evil\0"),
+      member("././@LongLink", "L", "safe.txt\0"),
+      member("x", "0"),
+      END,
+    );
+    expect(() => listTarEntries(twice)).toThrow(/several extension headers \(LL\)/);
+  });
+
+  it("still reads a GNU long name together with a long link", () => {
+    const archive = concat(
+      member("././@LongLink", "K", "target\0"),
+      member("././@LongLink", "L", `${"a".repeat(120)}\0`),
+      member("short", "2", "", { link: "ignored" }),
+      END,
+    );
+    expect(listTarEntries(archive)).toEqual([
+      { name: "a".repeat(120), type: "symlink", linkTarget: "target" },
+    ]);
+  });
+
+  it("refuses a header whose checksum does not match — GNU tar would skip it and read its body as headers", () => {
+    const archive = concat(
+      header("srv/caelo/safe.txt", "0", { sizeField: 512, badChecksum: true }),
+      header("../../evil", "0"),
+      END,
+    );
+    expect(() => listTarEntries(archive)).toThrow(/checksum mismatch/);
+  });
+
+  it("refuses a directory or symlink with a body — bsdtar reads the next block as a header", () => {
+    for (const [name, type] of [
+      ["srv/caelo/link", "2"],
+      ["srv/caelo/dir/", "5"],
+    ] as const) {
+      const archive = concat(
+        header(name, type, { sizeField: 512, link: "a" }),
+        header("../../evil", "0"),
+        END,
+      );
+      expect(() => listTarEntries(archive)).toThrow(/with a body/);
+    }
+  });
+
+  it("walks a member's body by its pax size, so a member hidden behind the header size is seen", () => {
+    // tar -x reads `safe.txt` as empty (pax size=0) and then extracts the
+    // `../../evil` header that the ustar size field claims is file data.
+    const archive = concat(
+      member("PaxHeader", "x", paxLine("size=0")),
+      header("safe.txt", "0", { sizeField: 512 }),
+      member("../../evil", "0"),
+      END,
+    );
+    expect(listTarEntries(archive).map((e) => e.name)).toEqual(["safe.txt", "../../evil"]);
+  });
+
+  it("rejects GNU sparse pax records and an unreadable pax size", () => {
+    const sparse = member("PaxHeader", "x", paxLine("GNU.sparse.name=../../evil"));
+    expect(() => listTarEntries(concat(sparse, member("safe.txt", "0")))).toThrow(/GNU sparse/);
+    const badSize = member("PaxHeader", "x", paxLine("size=-1"));
+    expect(() => listTarEntries(concat(badSize, member("safe.txt", "0")))).toThrow(
+      /unreadable pax size/,
+    );
   });
 });
 
-describe("escapesRoot", () => {
-  it("flags absolute and parent-relative member names", () => {
+const END = new Uint8Array(1024);
+
+/**
+ * One ustar header block with a valid checksum (unless `badChecksum`).
+ * `sizeField` is what the header claims, independent of any body after it.
+ */
+function header(
+  name: string,
+  typeflag: string,
+  opts: { sizeField?: number; link?: string; badChecksum?: boolean } = {},
+): Uint8Array {
+  const enc = new TextEncoder();
+  const h = new Uint8Array(512);
+  h.set(enc.encode(name), 0);
+  h.set(enc.encode(`${(opts.sizeField ?? 0).toString(8).padStart(11, "0")}\0`), 124);
+  h[156] = typeflag.charCodeAt(0);
+  if (opts.link) h.set(enc.encode(opts.link), 157);
+  h.set(enc.encode("ustar\0"), 257);
+  h.fill(0x20, 148, 156);
+  const sum = h.reduce((n, b) => n + b, 0) + (opts.badChecksum ? 1 : 0);
+  h.set(enc.encode(`${sum.toString(8).padStart(6, "0")}\0 `), 148);
+  return h;
+}
+
+/** A header plus its padded body. */
+function member(name: string, typeflag: string, bodyText = "", opts: { link?: string } = {}) {
+  const body = new TextEncoder().encode(bodyText);
+  const out = new Uint8Array(512 + Math.ceil(body.length / 512) * 512);
+  out.set(header(name, typeflag, { sizeField: body.length, link: opts.link }), 0);
+  out.set(body, 512);
+  return out;
+}
+
+/** One pax record; its leading length counts its own digits. */
+function paxLine(kv: string): string {
+  const base = ` ${kv}\n`;
+  const len = String(base.length + 2).length + base.length;
+  return `${len}${base}`;
+}
+
+function concat(...parts: Uint8Array[]): Uint8Array {
+  const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
+  let at = 0;
+  for (const p of parts) {
+    out.set(p, at);
+    at += p.length;
+  }
+  return out;
+}
+
+describe("escapesRoot & normalizeMemberName", () => {
+  it("flags absolute, parent-relative, and dot-segment member names", () => {
     expect(escapesRoot("/etc/passwd")).toBe(true);
     expect(escapesRoot("srv/caelo/../../app")).toBe(true);
+    expect(escapesRoot(".//../etc/passwd")).toBe(true);
+    expect(escapesRoot("a/./b")).toBe(true);
     expect(escapesRoot("./srv/caelo/a..b")).toBe(false);
+    expect(normalizeMemberName(".//srv//caelo/")).toBe("srv/caelo");
   });
 });

package/src/tar-entries.ts

@@ -7,12 +7,34 @@
  * otherwise land outside the target directory).
  *
  * Parses ustar headers plus the two extensions GNU tar and bsdtar emit for
- * long names: GNU `L`/`K` records and pax `x`/`g` records (`path=`,
- * `linkpath=`). Anything it cannot read is an error, never a skipped member.
+ * long names: GNU `L`/`K` records and pax `x` records (`path=`, `linkpath=`,
+ * `size=`). Anything it cannot read is an error, never a skipped member.
+ *
+ * The parser must walk the archive exactly as `tar -x` will: a pax `size=`
+ * overrides the header's size field for the next member, so ignoring it
+ * would let an archive hide a member from this check inside what the parser
+ * takes for a body. Records that rename a member in ways it does not model
+ * (pax global `g` headers, GNU sparse `GNU.sparse.*` keys) are refused.
+ *
+ * Wherever GNU tar and bsdtar read the same bytes differently, the archive is
+ * refused rather than read one way: no Caelo backup is ever written like that,
+ * and whichever reading this module picked, the other tar would extract
+ * something it never checked.
+ * - A header whose checksum does not match: GNU tar skips it and reads the
+ *   next block as a header, so a member hidden in the "body" gets extracted.
+ * - A directory or symlink with a non-zero size: bsdtar reads no body for
+ *   them, so the block after the header is the next member's header.
+ * - Two name records for one member (two pax `x` headers, two `L`/`K`, or
+ *   `L`/`K` next to `x`): GNU tar keeps only the last `x` header, bsdtar
+ *   flags the second as malformed and drops it, and the two tars disagree
+ *   on whether a GNU long name or a pax path wins.
  */
 
 const BLOCK = 512;
 
+/** The extension headers one member may carry: one pax record, or a GNU long name and/or long link. */
+const SINGLE_NAMING = new Set(["x", "L", "K", "LK", "KL"]);
+
 export type TarEntryType = "file" | "directory" | "symlink" | "other";
 
 export interface TarEntry {
@@ -30,15 +52,34 @@ export function listTarEntries(tar: Uint8Array): TarEntry[] {
   let longLink: string | null = null;
   let paxPath: string | null = null;
   let paxLink: string | null = null;
+  let paxSize: number | null = null;
+  // The extension headers already seen for the member being read.
+  let extensions = "";
   while (offset + BLOCK <= tar.length) {
     const header = tar.subarray(offset, offset + BLOCK);
     if (header.every((b) => b === 0)) break; // end-of-archive marker
-    const size = octal(header.subarray(124, 136));
+    verifyChecksum(header);
     const typeflag = String.fromCharCode(header[156] ?? 0);
+    // A pax `size=` describes the member that follows the pax header, never
+    // another extension header (GNU tar applies it the same way).
+    const isExtension =
+      typeflag === "x" || typeflag === "g" || typeflag === "L" || typeflag === "K";
+    const size = !isExtension && paxSize !== null ? paxSize : octal(header.subarray(124, 136));
     const body = tar.subarray(offset + BLOCK, offset + BLOCK + size);
     if (body.length < size) throw new Error("truncated tar archive");
     offset += BLOCK + Math.ceil(size / BLOCK) * BLOCK;
 
+    if (typeflag === "g") {
+      throw new Error("tar archive contains unsupported pax global extended header");
+    }
+    if (isExtension) {
+      extensions += typeflag;
+      if (!SINGLE_NAMING.has(extensions)) {
+        throw new Error(
+          `tar archive names one member through several extension headers (${extensions}) — tar implementations disagree on which one wins`,
+        );
+      }
+    }
     if (typeflag === "L") {
       longName = cString(body);
       continue;
@@ -47,11 +88,18 @@ export function listTarEntries(tar: Uint8Array): TarEntry[] {
       longLink = cString(body);
       continue;
     }
-    if (typeflag === "x" || typeflag === "g") {
+    if (typeflag === "x") {
       const pax = paxRecords(body);
-      if (typeflag === "x") {
-        paxPath = pax.path ?? null;
-        paxLink = pax.linkpath ?? null;
+      if (Object.keys(pax).some((key) => key.startsWith("GNU.sparse."))) {
+        throw new Error("tar archive contains an unsupported GNU sparse member");
+      }
+      paxPath = pax.path ?? null;
+      paxLink = pax.linkpath ?? null;
+      if (pax.size !== undefined) {
+        if (!/^[0-9]+$/.test(pax.size)) {
+          throw new Error(`unreadable pax size ${JSON.stringify(pax.size)}`);
+        }
+        paxSize = Number.parseInt(pax.size, 10);
       }
       continue;
     }
@@ -60,6 +108,8 @@ export function listTarEntries(tar: Uint8Array): TarEntry[] {
     const name = paxPath ?? longName ?? (prefix ? `${prefix}/${shortName}` : shortName);
     const link = paxLink ?? longLink ?? cString(header.subarray(157, 257));
     longName = longLink = paxPath = paxLink = null;
+    paxSize = null;
+    extensions = "";
     const type: TarEntryType =
       typeflag === "0" || typeflag === "\0" || typeflag === "7"
         ? "file"
@@ -68,6 +118,9 @@ export function listTarEntries(tar: Uint8Array): TarEntry[] {
           : typeflag === "2"
             ? "symlink"
             : "other";
+    if (type !== "file" && size !== 0) {
+      throw new Error(`tar member ${JSON.stringify(name)} is a ${type} with a body`);
+    }
     entries.push(type === "symlink" ? { name, type, linkTarget: link } : { name, type });
   }
   return entries;
@@ -75,12 +128,40 @@ export function listTarEntries(tar: Uint8Array): TarEntry[] {
 
 /** `./a/b/` → `a/b`; the archive root (`./`) → "". */
 export function normalizeMemberName(name: string): string {
-  return name.replace(/^(\.\/)+/, "").replace(/\/+$/, "");
+  return name
+    .replace(/\/+/g, "/")
+    .replace(/^(\.\/)+/, "")
+    .replace(/\/+$/, "");
 }
 
 /** Whether a member name could write outside the extraction directory. */
 export function escapesRoot(name: string): boolean {
-  return name.startsWith("/") || name.split("/").some((part) => part === "..");
+  const normalized = name.replace(/\/+/g, "/").replace(/^(\.\/)+/, "");
+  return (
+    name.startsWith("/") || normalized.split("/").some((part) => part === ".." || part === ".")
+  );
+}
+
+/**
+ * Refuse a header whose checksum does not match, as `tar -x` would — except
+ * GNU tar then goes on reading at the next block, which this check must not.
+ * The sum counts the checksum field as spaces; old tars summed signed bytes,
+ * which GNU tar and bsdtar both still accept.
+ */
+function verifyChecksum(header: Uint8Array): void {
+  const recorded = cString(header.subarray(148, 156)).trim();
+  if (!/^[0-7]+$/.test(recorded)) throw new Error("unreadable tar header checksum");
+  let unsigned = 0;
+  let signed = 0;
+  for (let i = 0; i < BLOCK; i++) {
+    const b = i >= 148 && i < 156 ? 0x20 : (header[i] ?? 0);
+    unsigned += b;
+    signed += b > 127 ? b - 256 : b;
+  }
+  const expected = Number.parseInt(recorded, 8);
+  if (expected !== unsigned && expected !== signed) {
+    throw new Error("tar header checksum mismatch — the archive is damaged or was altered");
+  }
 }
 
 function octal(field: Uint8Array): number {

package/src/upgrade/app-roll.ts

@@ -42,6 +42,7 @@ import {
   rollService,
   serviceRollArgs,
 } from "../stack-converge.js";
+import { describeAuditJobImage, setAuditJobImage } from "./audit-job-roll.js";
 import { restoreCommand, type SqlBackup } from "./backup.js";
 import {
   describeServiceJson,
@@ -366,7 +367,17 @@ export async function executeAppRoll(
   }
 
   const rolled: RollPlan[] = [];
+  /** The audit job's image before this roll moved it (#875); null = not moved. */
+  let auditJobPriorImage: string | null = null;
   const rollBack = async (alsoCurrent?: RollPlan): Promise<void> => {
+    if (auditJobPriorImage) {
+      const back = await setAuditJobImage(run, install, auditJobPriorImage);
+      if (back.ok) r.info(`  audit job image restored to ${auditJobPriorImage.slice(-19)}`);
+      else
+        r.error(
+          `  audit job rollback FAILED (${back.error}) — set its image to ${auditJobPriorImage} by hand`,
+        );
+    }
     const services = alsoCurrent ? [...rolled, alsoCurrent] : rolled;
     if (services.length === 0) return;
     r.warn(`Rolling back ${services.length} service(s) to the revisions that served before...`);
@@ -447,6 +458,34 @@ export async function executeAppRoll(
     rolled.push(p);
   }
 
+  // #875 — the audit job runs the admin's image; it follows the admin. A
+  // failure here does not undo the release: audits then fail loudly with
+  // the version mismatch and the fix, until the next upgrade moves it.
+  const sAudit = r.step("Moving the quality-audit job onto the admin's image...");
+  const described = await describeAuditJobImage(run, install);
+  const auditPrior = described.ok ? described.image : null;
+  if (!described.ok) {
+    sAudit.warned(
+      `Could not read the quality-audit job (${described.error}); Stage audits fail until the next upgrade moves it.`,
+    );
+  } else if (!auditPrior) {
+    sAudit.warned(
+      "The install has no quality-audit job yet; Stage audits fail until `cms-provision upgrade` sets it up.",
+    );
+  } else if (auditPrior === admin.imageRef) {
+    sAudit.done("Quality-audit job already on the admin's image");
+  } else {
+    const moved = await setAuditJobImage(run, install, admin.imageRef);
+    if (moved.ok) {
+      auditJobPriorImage = auditPrior;
+      sAudit.done(`Quality-audit job on ${admin.digest.slice(0, 19)}...`);
+    } else {
+      sAudit.warned(
+        `Quality-audit job not moved (${moved.error}); Stage audits fail until the next upgrade moves it.`,
+      );
+    }
+  }
+
   if (deps.health) {
     const s = r.step("Deep health check (database, migrations, version)...");
     // A probe that throws is a failed check: roll back, never leave the

package/src/upgrade/audit-job-roll.ts

@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * #875 — the quality-audit job runs the admin image (admin-core
+ * quality/audit-task.ts) and refuses a batch from an admin on another
+ * version. So every app-roll moves the job onto the admin's new image right
+ * after the services rolled, and a rollback moves it back.
+ *
+ * `run jobs update --image` changes nothing else of the job (identity, env,
+ * parallelism), so this stays inside the app-roll's rights: the upgrader's
+ * `roles/run.developer` plus actAs on the job's service account
+ * (upgrader-setup.ts).
+ */
+
+import type { GcloudRunner } from "../gcloud-retry.js";
+import { auditJobName } from "../gcp-names.js";
+
+/**
+ * The job's image now; `image: null` only when the install has no audit job
+ * yet. Any other gcloud failure (permissions, API, network) is an error, not
+ * "no job", so the app-roll reports it instead of a misleading hint.
+ */
+export async function describeAuditJobImage(
+  run: GcloudRunner,
+  install: { readonly projectId: string; readonly region: string; readonly env: string },
+): Promise<
+  | { readonly ok: true; readonly image: string | null }
+  | { readonly ok: false; readonly error: string }
+> {
+  const r = await run([
+    "run",
+    "jobs",
+    "describe",
+    auditJobName(install.env),
+    `--region=${install.region}`,
+    `--project=${install.projectId}`,
+    "--format=value(spec.template.spec.template.spec.containers[0].image)",
+  ]);
+  if (!r.ok) {
+    return /NOT_FOUND|not found|could not be found/i.test(r.stderr)
+      ? { ok: true, image: null }
+      : { ok: false, error: r.stderr.trim() };
+  }
+  const image = r.stdout.trim();
+  if (image.length === 0) {
+    return { ok: false, error: "the job's description names no container image" };
+  }
+  return { ok: true, image };
+}
+
+/** Point the audit job at `imageRef`. */
+export async function setAuditJobImage(
+  run: GcloudRunner,
+  install: { readonly projectId: string; readonly region: string; readonly env: string },
+  imageRef: string,
+): Promise<{ readonly ok: boolean; readonly error?: string }> {
+  const r = await run([
+    "run",
+    "jobs",
+    "update",
+    auditJobName(install.env),
+    `--image=${imageRef}`,
+    `--region=${install.region}`,
+    `--project=${install.projectId}`,
+    "--quiet",
+  ]);
+  return r.ok ? { ok: true } : { ok: false, error: r.stderr.trim() };
+}

package/src/upgrade/converge.ts

@@ -7,7 +7,8 @@
  * CLI-generated runtime secrets, the stack's IAM/CDN/firewall invariants,
  * MCP access through IAP, public_role's Cloud SQL role memberships (#633),
  * the gateway's and public_role's database logins, the operator-access sync
- * job, and dropping bindings nothing reads any more.
+ * job, the quality-audit job (#875), and dropping bindings nothing reads any
+ * more.
  *
  * It needs the rights of whoever provisioned the install, so only
  * `cms-provision upgrade` (the operator's own identity) runs it — never the
@@ -24,6 +25,7 @@
  * {@link AppRollHooks} the app-roll calls in between, and a part after it.
  */
 
+import { ensureAuditJob, resolveAdminImage } from "../audit-job-setup.js";
 import { findSqlInstance, planPublicRoleSwitch } from "../gateway-credentials.js";
 import type { GcloudRunner, Sleep } from "../gcloud-retry.js";
 import { upgraderServiceAccountEmail } from "../gcp-names.js";
@@ -40,6 +42,8 @@ import {
 } from "../runtime-secrets.js";
 import { selfCommand } from "../self-command.js";
 import {
+  AUDIT_JOB_ENV_VAR,
+  AUDIT_RESULTS_BUCKET_ENV_VAR,
   MCP_ENV_VAR,
   OPERATOR_ACCESS_JOB_ENV_VAR,
   UPGRADER_JOB_ENV_VAR,
@@ -268,6 +272,42 @@ export function startConverge(
     incomplete = true;
   }
 
+  async function auditJob(): Promise<void> {
+    // #875 — the job the quality audit fans its Lighthouse runs out to.
+    // Deployed with the admin's CURRENT image (the admin still runs it until
+    // the app-roll moves both together, and the job refuses a batch from
+    // another version). A failure only costs Stage audits, so it warns and
+    // the admin keeps its current env.
+    const s = r.step("Ensuring the quality-audit job (parallel Lighthouse runs)...");
+    const image = await resolveAdminImage(
+      { projectId: install.projectId, region: install.region, env: install.env },
+      { run },
+    );
+    const aj = image.ok
+      ? await ensureAuditJob(
+          {
+            provider: install.provider,
+            projectId: install.projectId,
+            region: install.region,
+            env: install.env,
+            adminImageRef: image.imageRef,
+          },
+          runDeps,
+        )
+      : { ok: false as const, done: [], error: image.error };
+    for (const step of aj.done) r.info(`  ${step}`);
+    if (aj.ok) {
+      s.done("Quality-audit job ready (Stage audits run in parallel)");
+      return;
+    }
+    s.warned(`Quality-audit job not set up: ${aj.error}`);
+    r.warn(
+      "The upgrade continues; Stage audits fail with the missing job until this succeeds. Re-run upgrade once the cause is fixed.",
+    );
+    incomplete = true;
+    leaveUntouched.push(AUDIT_JOB_ENV_VAR, AUDIT_RESULTS_BUCKET_ENV_VAR);
+  }
+
   /** The admin's IAP resource: the service on gcp-firebase, the LB backend on gcp. */
   async function adminIapResource(): Promise<IapResource | null> {
     if (install.provider === "gcp-firebase") {
@@ -406,6 +446,7 @@ export function startConverge(
       if (!(await publicRoleIsolation())) return false;
       await mcpAccess();
       await urlMapRedirectsAccess();
+      await auditJob();
       return true;
     },
 

package/src/upgrade/upgrade.test.ts

@@ -60,7 +60,9 @@ const serviceJson = (image: string, env: unknown[]) =>
     },
   });
 
-const ADMIN_JSON = serviceJson("registry/admin@sha256:old", [
+/** The admin image the install runs before the upgrade (digest-pinned, as deployed). */
+const OLD_ADMIN_IMAGE = `registry/admin@sha256:${"0".repeat(64)}`;
+const ADMIN_JSON = serviceJson(OLD_ADMIN_IMAGE, [
   { name: "CAELO_FIREBASE_SITE", value: "caelo-production-site-abc123" },
   {
     name: "ADMIN_DATABASE_URL",
@@ -98,6 +100,10 @@ function fakeInstall(events: string[], overrides: [RegExp, Answer][] = []) {
     [new RegExp(`describe ${ADMIN} .*--format=json`), ok(ADMIN_JSON)],
     [new RegExp(`describe ${GATEWAY} .*--format=json`), ok(GATEWAY_JSON)],
     [/describe (\S+) .*status\.url/, (l) => ok(`https://${l.split(" ")[3]}-xyz.a.run.app\n`)],
+    [
+      /^run jobs describe caelo-production-audit .*containers\[0\]\.image/,
+      ok(`${OLD_ADMIN_IMAGE}\n`),
+    ],
     [/^sql instances list/, ok(`${SQL}\n`)],
     [/^sql backups list/, ok(`${BACKUP_ID}\n`)],
     [
@@ -223,7 +229,7 @@ const CONVERGE_WRITE =
   /add-iam-policy-binding|remove-iam-policy-binding|set-iam-policy|service-accounts create|secrets create|secrets versions add|services enable|roles create|roles update|roles undelete|firewall-rules delete|users assign-roles/;
 /** Calls that move a service or run migrations — app-roll-only. */
 const APP_ROLL_WRITE =
-  /^run services update |^run services update-traffic|jobs execute caelo-migrate/;
+  /^run services update |^run services update-traffic|jobs execute caelo-migrate|^run jobs update caelo-production-audit /;
 /** Any gcloud call that changes something. */
 const ANY_WRITE =
   /\b(create|update|update-traffic|delete|add-iam-policy-binding|remove-iam-policy-binding|set-iam-policy|execute|enable|add|restore)\b/;
@@ -285,6 +291,69 @@ describe("upgradeGcpInstall — converge, then app-roll", () => {
     );
   });
 
+  it("sets up the quality-audit job on the CURRENT admin image before the roll, then moves it with the admin (#875)", async () => {
+    const events: string[] = [];
+    const { deps, lines } = upgradeDeps(events);
+    expect(await upgradeGcpInstall(TARGET, deps)).toBe(true);
+    const deploy = indexOf(events, /^gcloud run jobs deploy caelo-production-audit /);
+    const adminRoll = indexOf(events, new RegExp(`^gcloud run services update ${ADMIN} `));
+    const move = indexOf(events, /^gcloud run jobs update caelo-production-audit /);
+    const deep = indexOf(events, /^health deep/);
+    expect(events[deploy]).toContain(`--image=${OLD_ADMIN_IMAGE}`);
+    expect(deploy < adminRoll && adminRoll < move && move < deep).toBe(true);
+    expect(events[move]).toContain(`--image=${IMAGES.admin.imageRef}`);
+    expect(lines).toContain("done: Quality-audit job ready (Stage audits run in parallel)");
+  });
+
+  it("a failed audit-job setup leaves the converge incomplete and the admin's audit env untouched", async () => {
+    const events: string[] = [];
+    const { deps, recorded, lines } = upgradeDeps(events, {
+      overrides: [[/^run jobs deploy caelo-production-audit/, fail("PERMISSION_DENIED")]],
+    });
+    expect(await upgradeGcpInstall(TARGET, deps)).toBe(true);
+    expect(recorded.contract).toEqual([]);
+    expect(lines.some((l) => l.includes("Quality-audit job not set up"))).toBe(true);
+    expect(
+      events.some((e) => /services update caelo-production-admin.*CAELO_AUDIT_JOB=/.test(e)),
+    ).toBe(false);
+  });
+
+  it("an unreadable audit job is reported as an error, not as a missing job, and is not moved", async () => {
+    const events: string[] = [];
+    const { deps, lines } = upgradeDeps(events, {
+      overrides: [
+        [
+          /^run jobs describe caelo-production-audit .*containers\[0\]\.image/,
+          fail("PERMISSION_DENIED"),
+        ],
+      ],
+    });
+    expect(await upgradeGcpInstall(TARGET, deps)).toBe(true);
+    expect(
+      lines.some((l) => l.includes("Could not read the quality-audit job (PERMISSION_DENIED)")),
+    ).toBe(true);
+    expect(lines.some((l) => l.includes("has no quality-audit job yet"))).toBe(false);
+    expect(events.some((e) => /^gcloud run jobs update caelo-production-audit /.test(e))).toBe(
+      false,
+    );
+  });
+
+  it("a missing audit job is reported as missing", async () => {
+    const events: string[] = [];
+    const { deps, lines } = upgradeDeps(events, {
+      overrides: [
+        [
+          /^run jobs describe caelo-production-audit .*containers\[0\]\.image/,
+          fail(
+            "ERROR: (gcloud.run.jobs.describe) Cannot find job [caelo-production-audit]. NOT_FOUND",
+          ),
+        ],
+      ],
+    });
+    expect(await upgradeGcpInstall(TARGET, deps)).toBe(true);
+    expect(lines.some((l) => l.includes("has no quality-audit job yet"))).toBe(true);
+  });
+
   it("without an ops image the upgrader step fails, the roll goes on, the converge is incomplete", async () => {
     const events: string[] = [];
     const { deps, recorded, lines } = upgradeDeps(events);
@@ -378,6 +447,11 @@ describe("upgradeGcpInstall — converge, then app-roll", () => {
         `${ADMIN} --region europe-west1 --project acme --to-revisions ${ADMIN}-00001-old=100`,
       ),
     ]);
+    // The audit job follows the admin back to its prior image (#875).
+    const jobBack = events
+      .slice(deep)
+      .filter((e) => e.startsWith("gcloud run jobs update caelo-production-audit "));
+    expect(jobBack).toEqual([expect.stringContaining(`--image=${OLD_ADMIN_IMAGE}`)]);
     expect(recorded.digests).toEqual([]);
     expect(recorded.contract).toEqual([]);
     const restore = `gcloud sql backups restore ${BACKUP_ID} --restore-instance=${SQL} --project=acme`;

package/src/upgrader-assess.test.ts

@@ -30,6 +30,7 @@ const POLICY: StoredUpdatePolicy = {
   autoSecurity: true,
   autoPatch: true,
   autoMinor: true,
+  autoConverge: false,
   channel: "stable",
   windowStart: null,
   windowEnd: null,

package/src/upgrader-job.test.ts

@@ -49,6 +49,7 @@ const POLICY: StoredUpdatePolicy = {
   autoSecurity: false,
   autoPatch: false,
   autoMinor: false,
+  autoConverge: false,
   channel: "stable",
   windowStart: null,
   windowEnd: null,

package/src/upgrader-setup.test.ts

@@ -83,6 +83,7 @@ describe("ensureUpgraderJob", () => {
         "iam service-accounts add-iam-policy-binding caelo-production-gateway-sa@acme.iam.gserviceaccount.com → roles/iam.serviceAccountUser",
         "iam service-accounts add-iam-policy-binding caelo-production-upgrader@acme.iam.gserviceaccount.com → roles/iam.serviceAccountUser",
         "iam service-accounts add-iam-policy-binding caelo-mcp@acme.iam.gserviceaccount.com → roles/iam.serviceAccountTokenCreator",
+        "iam service-accounts add-iam-policy-binding caelo-production-audit@acme.iam.gserviceaccount.com → roles/iam.serviceAccountUser",
         "storage buckets add-iam-policy-binding gs://acme-caelo-ops → roles/storage.objectUser",
         "storage buckets add-iam-policy-binding gs://acme-caelo-ops → projects/acme/roles/caeloUpgraderOpsList",
       ].sort(),