@caelo-cms/provisioning 0.10.53 → 0.10.57

package/src/upgrader-job.ts

@@ -12,18 +12,34 @@
  * but cannot change IAM, service accounts or secrets. One run:
  *
  *   1. claim the oldest pending request (ending a run that died mid-way);
- *   2. read what the install runs from install.json in its ops bucket;
+ *   2. read what the install runs from install.json in its ops bucket; when
+ *      it names other images than the ones serving, judge against the
+ *      serving release if a signed release manifest names both its images,
+ *      else stop (upgrader-records.ts, #860);
  *   3. fetch the release manifest from the FIXED registry and verify its
  *      signature (release-manifest.ts) — no registry, identity or digest
  *      ever comes from the request, the env or argv;
  *   4. decide (upgrader-assess.ts): refuse a bad signature, a downgrade, a
  *      release below its minFromVersion, an automatic run the policy does
- *      not allow; hand an infrastructure change to Cloud Shell;
+ *      not allow; hand an infrastructure change back (the admin routes
+ *      it to the converge job, #819; Cloud Shell on installs without one);
+ *   -- ask IAM whether the job's account holds every permission the rest
+ *      of the run needs (upgrader-iam-preflight.ts, #818); any missing one
+ *      ends the run as needs-Cloud-Shell, all of them named, nothing changed;
+ *   -- only now take the install's state lock (#815), confirm install.json
+ *      is unchanged since step 2, and write step 2's correction. Everything
+ *      above only reads, so no lock problem can hide the verdict; a stop
+ *      verdict still corrects the records when the lock is free. Only a
+ *      verdict judged on drift asks for the lock even when it stops: the
+ *      serving images were read without it, and a run that holds it may be
+ *      half-way through its own roll — then the answer is "another install
+ *      operation is running", never a drift or Cloud Shell verdict;
  *   5. take a Cloud SQL backup — no backup, no change;
  *   6. app-roll (upgrade/app-roll.ts) with no converge hooks: migrations
  *      with the new admin image, admin then gateway, deep health check,
  *      traffic back onto the previous revisions on any failure;
- *   7. record the new digests + version in install.json;
+ *   7. record the new digests + version in install.json (a failed write
+ *      ends the run as failed, never as succeeded);
  *   8. move this job onto the release's own ops image, last.
  *
  * Every step is narrated into `platform_update_events` (upgrader-reporter.ts).
@@ -52,10 +68,26 @@ import {
   type RollSlug,
 } from "./upgrade/app-roll.js";
 import { restoreCommand, takeSqlBackup } from "./upgrade/backup.js";
-import { findServingImageDigest, resolveGcpResourceName } from "./upgrade/gcp-lookup.js";
+import {
+  findServingImage,
+  resolveGcpResourceName,
+  servingImageDigests,
+} from "./upgrade/gcp-lookup.js";
 import type { HealthProbe } from "./upgrade/health.js";
 import { releaseImageRef } from "./upgrade/release-images.js";
 import { assessUpgradeRequest, type RefusalCode } from "./upgrader-assess.js";
+import {
+  type IamTester,
+  preflightErrorReason,
+  preflightMissingReason,
+  runIamPreflight,
+  upgraderAccessChecks,
+} from "./upgrader-iam-preflight.js";
+import {
+  findServingRelease,
+  recordMatchesServing,
+  type ServingDigests,
+} from "./upgrader-records.js";
 import { eventReporter, type JobLog } from "./upgrader-reporter.js";
 
 /** A request the job claimed, with the policy as it stood at claim time. */
@@ -125,15 +157,22 @@ export interface UpgraderJobDeps {
   readonly env: string;
   /** This Cloud Run execution, for the log link; null outside Cloud Run. */
   readonly execution: string | null;
+  /**
+   * Asks IAM which permissions the job's own account holds (#818), for the
+   * self-check right after the decision. Never the action itself.
+   */
+  readonly iam: IamTester;
   /** fetchVerifiedReleaseManifest against the fixed registry. */
   readonly fetchManifest: (version: string) => Promise<ReleaseManifestResult>;
   /**
-   * Take the install's state lock (the one `pulumi up` and `state migrate`
-   * hold) for the whole run; returns its release. Throws when another run
-   * holds it.
+   * Take the install's state lock (the one `pulumi up`, `state migrate` and
+   * the CLI `upgrade` hold) right before the run's first write; returns its
+   * release. Breaks a lock whose holder has ended and reports that through
+   * `onBreak` (state-store.ts). Throws StateLockHeldError when a live run
+   * holds it, any other error when it cannot be taken or judged.
    */
-  readonly lockState: () => () => void;
-  /** install.json from the install's ops bucket (null when absent). */
+  readonly lockState: (onBreak: (message: string) => void) => () => void;
+  /** install.json as the ops bucket holds it NOW (null when absent) — never cached. */
   readonly readInstall: () => InstallMetadata | null;
   readonly writeInstall: (meta: InstallMetadata) => void;
   /** Runs both migration targets with the NEW admin image. */
@@ -157,8 +196,24 @@ export interface UpgraderRunResult {
   readonly status: UpdateEndStatus | null;
 }
 
+/** Why the run cannot go on: how its request ends. */
+interface Blocked {
+  readonly status: UpdateEndStatus;
+  readonly codes: readonly string[];
+  readonly reason: string;
+}
+
+type LockOutcome = { readonly ok: true } | ({ readonly ok: false } & Blocked);
+
 const ROLLED_PHASES = new Set(["roll", "health"]);
 
+/**
+ * Lock outcomes that mean another run is changing the install right now
+ * (#815): a verdict judged from serving images read without the lock may
+ * describe that run's half-finished roll, so it is not reported.
+ */
+const STALE_VERDICT: ReadonlySet<string> = new Set(["state-locked", "install-state-changed"]);
+
 /** Attempts at recording a request's end state before the run fails. */
 const FINISH_ATTEMPTS = 3;
 
@@ -221,43 +276,104 @@ export async function runUpgraderJob(deps: UpgraderJobDeps): Promise<UpgraderRun
       `Upgrader started${deps.execution ? ` (run ${deps.execution})` : ""}: installing Caelo ${request.targetVersion}.`,
     );
 
-    // Exclusive against the installer's own state writers for the whole run.
-    try {
-      releaseLock = deps.lockState();
-    } catch (e) {
-      if (e instanceof StateLockHeldError) {
-        return stopWith(
-          "failed",
-          ["state-locked"],
-          `Nothing was changed: another install operation is running on this site (${errorText(e)}). Try again when it has finished.`,
-        );
+    // #815 — the state lock is taken only right before the first write
+    // (lockBeforeWrite below), never before the decision: any lock problem
+    // must not hide the verdict (needs Cloud Shell, refused), and a run
+    // that changes nothing has no business blocking another.
+    const lockBeforeWrite = (): LockOutcome => {
+      try {
+        releaseLock = deps.lockState((message) => r.info(message));
+        return { ok: true };
+      } catch (e) {
+        if (e instanceof StateLockHeldError) {
+          return {
+            ok: false,
+            status: "failed",
+            codes: ["state-locked"],
+            reason: `Nothing was changed: another install operation is running on this site (${errorText(e)}). Try again when it has finished.`,
+          };
+        }
+        // Not contention: the job cannot reach its own ops bucket or ask
+        // Cloud Run about the holder. Retrying will not help; a Cloud Shell
+        // upgrade re-converges the job's grants.
+        return {
+          ok: false,
+          status: "needs_cloud_shell",
+          codes: ["state-access"],
+          reason: `Nothing was changed: the upgrader could not take the install's state lock (${errorText(e)}). Run the upgrade once with Cloud Shell; it repairs the upgrader's access.`,
+        };
       }
-      // Not contention: the job cannot reach its own ops bucket. Retrying
-      // will not help; a Cloud Shell upgrade re-converges the job's grants.
-      return stopWith(
-        "needs_cloud_shell",
-        ["state-access"],
-        `Nothing was changed: the upgrader could not take the install's state lock (${errorText(e)}). Run the upgrade once with Cloud Shell; it repairs the upgrader's access.`,
-      );
-    }
+    };
 
     // 2. What the install runs — from its own state, never from the admin.
     r.phase("verify");
-    const meta = deps.readInstall();
-    if (!meta || meta.projectId !== deps.projectId) {
+    const read = deps.readInstall();
+    if (!read || read.projectId !== deps.projectId) {
       return stopWith(
         "needs_cloud_shell",
         ["install-state"],
         "The install's records (install.json) are not in its own cloud project yet, so the upgrader cannot tell what it runs. Run the upgrade once with Cloud Shell; it moves them there.",
       );
     }
-    if (meta.provider !== "gcp" && meta.provider !== "gcp-firebase") {
+    if (read.provider !== "gcp" && read.provider !== "gcp-firebase") {
       return stopWith(
         "refused",
         ["provider"],
-        `The upgrader job only upgrades Google Cloud installs; this one is ${meta.provider}.`,
+        `The upgrader job only upgrades Google Cloud installs; this one is ${read.provider}.`,
       );
     }
+    // Rebound when the records are refreshed below; `provider` is narrowed.
+    let meta: InstallMetadata & { provider: "gcp" | "gcp-firebase" } = {
+      ...read,
+      provider: read.provider,
+    };
+    /** The corrected records (#860), when install.json is stale; written under the lock. */
+    let heal: InstallMetadata | null = null;
+
+    /**
+     * Take the lock, confirm install.json is still what was judged, and
+     * write the pending correction. Returns null when all of that held.
+     */
+    const lockAndHeal = (): Blocked | null => {
+      const locked = lockBeforeWrite();
+      if (!locked.ok) return locked;
+      // Another writer may have changed the records between our read and
+      // the lock; everything above was judged against the old ones.
+      // A failed re-read is a Blocked like any other, never a throw: on the
+      // stop-verdict path a throw here would replace the verdict with
+      // failed/internal.
+      let now: InstallMetadata | null;
+      try {
+        now = deps.readInstall();
+      } catch (e) {
+        return {
+          status: "failed",
+          codes: ["install-state-read"],
+          reason: `Nothing was changed: the install's records (install.json) could not be read again once the lock was held (${errorText(e)}). Try again.`,
+        };
+      }
+      if (JSON.stringify(now) !== JSON.stringify(read)) {
+        return {
+          status: "failed",
+          codes: ["install-state-changed"],
+          reason:
+            "Nothing was changed: another install operation changed the install's records (install.json) while this update was being checked. Try again.",
+        };
+      }
+      if (!heal) return null;
+      const v = heal.releaseVersion;
+      try {
+        deps.writeInstall(heal);
+      } catch (e) {
+        return {
+          status: "failed",
+          codes: ["install-state-write"],
+          reason: `Nothing was changed: the install's records (install.json) name another release than the one that serves (v${v}), and they could not be corrected (${errorText(e)}).`,
+        };
+      }
+      r.success(`Caelo refreshed its install records: v${v} serves.`);
+      return null;
+    };
 
     // install.json must describe what actually serves: if a previous run
     // rolled but could not record it, the recorded version is stale and
@@ -266,17 +382,34 @@ export async function runUpgraderJob(deps: UpgraderJobDeps): Promise<UpgraderRun
     if (!serving.ok) {
       return stopWith("failed", ["lookup"], `Nothing was changed: ${serving.error}.`);
     }
-    const recorded = meta.imageDigests;
-    if (
-      !recorded ||
-      recorded.admin !== serving.digests.admin ||
-      recorded.gateway !== serving.digests.gateway
-    ) {
-      return stopWith(
-        "needs_cloud_shell",
-        ["install-state-drift"],
-        "The release that serves now is not the one the install's records (install.json) name, so the upgrader cannot tell safely what it runs. Run the upgrade once with Cloud Shell; it records the serving release again.",
-      );
+    if (!recordMatchesServing(meta.imageDigests, serving.digests)) {
+      // #860 — re-record the serving release ourselves, but only one whose
+      // admin AND gateway images a signed release manifest names.
+      const found = await findServingRelease(meta, serving.digests, deps);
+      if (!found.ok) {
+        if (found.retryable) return stopWith("failed", ["registry"], found.reason);
+        // #815 — the serving images were read without the lock: a run that
+        // holds it may be half-way through its roll, and that is not drift.
+        // Only once this run holds the lock (or cannot take it at all) is
+        // the mismatch the install's real state.
+        const blocked = lockAndHeal();
+        if (blocked && STALE_VERDICT.has(blocked.codes[0] ?? "")) {
+          return stopWith(blocked.status, blocked.codes, blocked.reason);
+        }
+        return stopWith("needs_cloud_shell", ["install-state-drift"], found.reason);
+      }
+      // The manifest's digests, not the serving ones: the signed form is the
+      // one every later comparison and deploy uses (gcp-lookup.ts ServingImage).
+      meta = {
+        ...meta,
+        imageDigests: {
+          admin: found.manifest.digests.admin,
+          gateway: found.manifest.digests.gateway,
+        },
+        releaseVersion: found.manifest.version,
+      };
+      // Judged against now, written under the lock once decided (#815).
+      heal = meta;
     }
 
     // 3 + 4. The signed manifest decides.
@@ -300,6 +433,21 @@ export async function runUpgraderJob(deps: UpgraderJobDeps): Promise<UpgraderRun
     }
     if (decision.kind === "stop") {
       r.phase("assess");
+      if (heal) {
+        // The records still get corrected, but the verdict is the answer:
+        // a lock or write problem here must not replace it (#815).
+        // Except when another run holds the lock or changed the records:
+        // then the drift this verdict was judged on may be that run's
+        // half-finished work, and "try again once it has finished" is the
+        // true answer.
+        const blocked = lockAndHeal();
+        if (blocked && STALE_VERDICT.has(blocked.codes[0] ?? "")) {
+          return stopWith(blocked.status, blocked.codes, blocked.reason);
+        }
+        if (blocked) {
+          r.warn(`The install's records could not be corrected this time: ${blocked.reason}`);
+        }
+      }
       return stopWith(decision.status, decision.codes, decision.reason);
     }
     r.phase("assess");
@@ -308,6 +456,25 @@ export async function runUpgraderJob(deps: UpgraderJobDeps): Promise<UpgraderRun
       `${meta.releaseVersion} → ${decision.manifest.version}: no cloud-setup change needed; installing.`,
     );
 
+    // #818 — decided to run: ask IAM for every permission the rest needs
+    // before the lock, the backup or the roll. A grant that looks right but
+    // cannot match (#799) ends the run here, all of them named, not halfway.
+    const access = await runIamPreflight(
+      upgraderAccessChecks({ projectId: deps.projectId, region: deps.region, env: deps.env }),
+      deps.iam,
+    );
+    if (access.kind === "missing") {
+      return stopWith(
+        "needs_cloud_shell",
+        ["iam-preflight"],
+        preflightMissingReason(access.missing),
+      );
+    }
+    if (access.kind === "error") {
+      return stopWith("failed", ["iam-preflight"], preflightErrorReason(access.errors));
+    }
+    r.info("The upgrader has every permission this update needs.");
+
     const m = decision.manifest;
     const images: Record<RollSlug, ReleaseImage> = {
       admin: { digest: m.digests.admin, imageRef: releaseImageRef("admin", m.digests.admin) },
@@ -333,6 +500,11 @@ export async function runUpgraderJob(deps: UpgraderJobDeps): Promise<UpgraderRun
     }
     for (const w of planned.plan.warnings) r.warn(w);
 
+    // Decided, planned: the backup (or the records correction) is the first
+    // write. Held through the roll, install.json and the self-update.
+    const blocked = lockAndHeal();
+    if (blocked) return stopWith(blocked.status, blocked.codes, blocked.reason);
+
     // 5. No backup, no change.
     r.phase("backup");
     const sBak = r.step("Backing up the database before anything changes...");
@@ -363,6 +535,7 @@ export async function runUpgraderJob(deps: UpgraderJobDeps): Promise<UpgraderRun
     }
 
     // 6 + 7. The roll; it rolls back by itself on failure.
+    let recordError: string | null = null;
     const rolled = await executeAppRoll(
       {
         plan: planned.plan,
@@ -377,6 +550,9 @@ export async function runUpgraderJob(deps: UpgraderJobDeps): Promise<UpgraderRun
         reporter: r,
         migrate: deps.migrate,
         health: probe.probe,
+        // The roll already succeeded, so a failed write must not undo it;
+        // it is held here and ends the run as failed below (no fallbacks:
+        // never "succeeded" while the records name the previous release).
         recordDigests: async (digests) => {
           r.phase("record");
           try {
@@ -384,8 +560,9 @@ export async function runUpgraderJob(deps: UpgraderJobDeps): Promise<UpgraderRun
             deps.writeInstall({ ...fresh, imageDigests: digests, releaseVersion: m.version });
             r.info(`Recorded ${m.version} as this install's release.`);
           } catch (e) {
-            r.warn(
-              `The new release serves, but install.json could not be updated (${errorText(e)}). The next update will stop and ask for one upgrade with Cloud Shell, which records it.`,
+            recordError = errorText(e);
+            r.error(
+              `The new release serves, but install.json could not be updated (${recordError}).`,
             );
           }
         },
@@ -408,6 +585,13 @@ export async function runUpgraderJob(deps: UpgraderJobDeps): Promise<UpgraderRun
         `Stopped during ${rolled.phase === "migrations" ? "the database migrations" : "the pre-check"}; the previous release keeps serving.${restore}`,
       );
     }
+    if (recordError !== null) {
+      return stopWith(
+        "failed",
+        ["record"],
+        `Caelo ${m.version} serves and passed its checks, but the install's records (install.json) could not be updated (${recordError}), so they still name the previous release. The next update corrects them by itself once the upgrader can write its ops bucket again; check the upgrader's access to it.`,
+      );
+    }
 
     // 8. Self-update, last: the next run uses this release's upgrader.
     r.phase("self-update");
@@ -446,7 +630,7 @@ export async function runUpgraderJob(deps: UpgraderJobDeps): Promise<UpgraderRun
  * @returns whether the end was written (false after the last attempt failed)
  */
 export async function recordUpdateEnd(
-  queue: UpdateQueue,
+  queue: Pick<UpdateQueue, "finish">,
   requestId: string,
   end: UpdateEnd,
   log: JobLog,
@@ -467,11 +651,11 @@ export async function recordUpdateEnd(
   return false;
 }
 
-/** The digests the revisions serving traffic run, per service. */
+/** Every digest the revisions serving traffic are known by, per service. */
 async function servingDigests(
   deps: UpgraderJobDeps,
-): Promise<{ ok: true; digests: Record<RollSlug, string> } | { ok: false; error: string }> {
-  const digests: Partial<Record<RollSlug, string>> = {};
+): Promise<{ ok: true; digests: ServingDigests } | { ok: false; error: string }> {
+  const digests: Partial<Record<RollSlug, readonly string[]>> = {};
   for (const slug of ["admin", "gateway"] as const) {
     const prefix = `${gcpNamePrefix(deps.env)}-${slug}`;
     const service = await resolveGcpResourceName(
@@ -481,13 +665,13 @@ async function servingDigests(
       deps.projectId,
       deps.region,
     );
-    const digest = service
-      ? await findServingImageDigest(deps.run, deps.projectId, deps.region, service)
+    const image = service
+      ? await findServingImage(deps.run, deps.projectId, deps.region, service)
       : null;
-    if (!digest) return { ok: false, error: `could not read which image the ${slug} serves` };
-    digests[slug] = digest;
+    if (!image) return { ok: false, error: `could not read which image the ${slug} serves` };
+    digests[slug] = servingImageDigests(image);
   }
-  return { ok: true, digests: digests as Record<RollSlug, string> };
+  return { ok: true, digests: digests as ServingDigests };
 }
 
 async function recordFacts(

package/src/upgrader-records.ts

@@ -0,0 +1,151 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * Does install.json describe what serves, and if not, which signed release
+ * does (#860)?
+ *
+ * The upgrader judges a request (downgrade, minFromVersion, infra contract)
+ * against the version install.json records, so it must first know the
+ * record is true. When the recorded digests differ from the serving ones,
+ * the only repair a human could make is "record the release that serves",
+ * and the job can do that itself without widening its trust: it records a
+ * release only when BOTH serving images (admin + gateway) are the images of
+ * one release manifest that verified against Caelo's release workflow
+ * (fetchVerifiedReleaseManifest, the same cosign check every install takes).
+ * Images that belong to no signed release are a deployment nobody can
+ * vouch for, so the run stops and says so.
+ *
+ * Which releases to try comes from untrusted hints — the version
+ * install.json names, and the release tags pointing at the serving digests
+ * in the fixed mirror (releaseVersionForDigest). A wrong hint costs one
+ * manifest fetch; only the signed manifest decides.
+ */
+
+import type { GcloudRunner } from "./gcloud-retry.js";
+import type { ImageDigests, InstallMetadata } from "./install-state.js";
+import type { ReleaseManifest, ReleaseManifestResult } from "./release-manifest.js";
+import type { RollSlug } from "./upgrade/app-roll.js";
+import { releaseVersionForDigest } from "./upgrade/release-images.js";
+
+/** Every digest each serving image is known by (gcp-lookup.ts servingImageDigests). */
+export type ServingDigests = Readonly<Record<RollSlug, readonly string[]>>;
+
+const SLUGS: readonly RollSlug[] = ["admin", "gateway"];
+
+/** Whether `recorded` names the images that serve, in any form they are known by. */
+export function recordMatchesServing(
+  recorded: ImageDigests | undefined,
+  serving: ServingDigests,
+): boolean {
+  if (!recorded) return false;
+  return SLUGS.every((slug) => serving[slug].includes(recorded[slug]));
+}
+
+/** What {@link findServingRelease} learned. */
+export type ServingRelease =
+  | { readonly ok: true; readonly manifest: ReleaseManifest }
+  | {
+      readonly ok: false;
+      /**
+       * True when no release matched but a candidate's manifest could not be
+       * fetched because the registry was unreachable or failing (5xx, 429,
+       * network): an outage, not a verdict, so the run must end `failed` and
+       * be retried, not send the operator to Cloud Shell. A manifest the
+       * registry says does not exist (404) never appears on retry, so it
+       * counts as rejected like a bad signature.
+       */
+      readonly retryable: boolean;
+      readonly reason: string;
+    };
+
+export interface ServingReleaseDeps {
+  readonly run: GcloudRunner;
+  /** fetchVerifiedReleaseManifest against the fixed registry. */
+  readonly fetchManifest: (version: string) => Promise<ReleaseManifestResult>;
+}
+
+/**
+ * The signed release whose admin AND gateway images both serve, or why there
+ * is none (an operator-facing sentence naming the serving digests).
+ *
+ * @param meta - install.json as read; its `releaseVersion` is the first hint.
+ */
+export async function findServingRelease(
+  meta: InstallMetadata,
+  serving: ServingDigests,
+  deps: ServingReleaseDeps,
+): Promise<ServingRelease> {
+  const partial: string[] = [];
+  const unverified: string[] = [];
+  const unreachable: string[] = [];
+  const tried: string[] = [];
+  for await (const version of candidateVersions(meta, serving, deps.run)) {
+    if (tried.includes(version)) continue;
+    tried.push(version);
+    const fetched = await deps.fetchManifest(version);
+    if (!fetched.ok) {
+      if (fetched.error.code === "registry" && fetched.error.transient) {
+        unreachable.push(`v${version} (${fetched.error.message.slice(0, 200)})`);
+        continue;
+      }
+      unverified.push(`v${version} (${fetched.error.message.slice(0, 200)})`);
+      continue;
+    }
+    const m = fetched.manifest;
+    const matching = SLUGS.filter((slug) => serving[slug].includes(m.digests[slug]));
+    if (matching.length === SLUGS.length) return { ok: true, manifest: m };
+    if (matching.length > 0) {
+      const other = SLUGS.filter((s) => !matching.includes(s));
+      partial.push(
+        `the ${matching.join(", ")} serves v${m.version}'s image, the ${other.join(", ")} does not`,
+      );
+    }
+  }
+  const servingText = SLUGS.map((s) => `${s} ${serving[s][0] ?? "(unknown)"}`).join(", ");
+  if (unreachable.length > 0) {
+    // A release we could not read may be the serving one, so this is no
+    // verdict on the images: retry later rather than escalate.
+    return {
+      ok: false,
+      retryable: true,
+      reason: `Nothing was changed: the images that serve now (${servingText}) are not the ones install.json names, and the Caelo release registry could not be reached to check which release they belong to (${unreachable.join("; ")}). Try again later.`,
+    };
+  }
+  return {
+    ok: false,
+    retryable: false,
+    reason: [
+      `The images that serve now (${servingText}) are not the ones install.json names, and they do not belong to one signed Caelo release${tried.length === 0 ? " (no release tag could be found for them)" : ""}, so the upgrader cannot tell what this install runs.`,
+      ...(partial.length > 0 ? [`${capitalize(partial.join("; "))}.`] : []),
+      ...(unverified.length > 0
+        ? [`These release records could not be verified: ${unverified.join("; ")}.`]
+        : []),
+      "Nothing was changed. Check who deployed these images (the Cloud Run revision history of both services); an upgrade with Cloud Shell puts a signed release back.",
+    ].join(" "),
+  };
+}
+
+/**
+ * Release versions worth checking, most likely first. Lazy: the version
+ * install.json names usually is the serving one, and then no tag listing
+ * runs at all.
+ */
+async function* candidateVersions(
+  meta: InstallMetadata,
+  serving: ServingDigests,
+  run: GcloudRunner,
+): AsyncGenerator<string> {
+  if (meta.releaseVersion) yield meta.releaseVersion;
+  // Release tags point at the index digest a deploy pins (the deployed form,
+  // listed first); the platform digest Cloud Run resolved has no tag.
+  for (const slug of SLUGS) {
+    for (const digest of serving[slug]) {
+      const version = await releaseVersionForDigest(run, slug, digest);
+      if (version) yield version;
+    }
+  }
+}
+
+function capitalize(s: string): string {
+  return s.charAt(0).toUpperCase() + s.slice(1);
+}

package/src/upgrader-reporter.test.ts

@@ -9,6 +9,12 @@ describe("phaseOfStep", () => {
     ["Applying DB migrations (idempotent)...", "migrations"],
     ["Rolling admin → sha256:aaaa...", "roll"],
     ["Deep health check (database, migrations, version)...", "health"],
+    // #819 — the full upgrade the converge job runs.
+    ["Pre-flight: resolving services + image digests + current revisions...", "verify"],
+    ["Taking a pre-upgrade Cloud SQL backup...", "backup"],
+    ["Ensuring the gateway service account + generated runtime secrets...", "converge"],
+    ["Removing public_role from the Cloud SQL superuser group...", "converge"],
+    ["Preparing the gateway's database login (public_role)...", "converge"],
     ["Something else", null],
   ])("%s → %s", (label, phase) => {
     expect(phaseOfStep(label)).toBe(phase as never);

package/src/upgrader-reporter.ts

@@ -41,10 +41,15 @@ export interface EventReporter extends UpgradeReporter {
 }
 
 /**
- * The phase an app-roll step belongs to, from its label (app-roll.ts names
- * its steps). Unknown labels keep the current phase.
+ * The phase an upgrade step belongs to, from its label (app-roll.ts,
+ * converge.ts and upgrade.ts name their steps). Unknown labels keep the
+ * current phase.
  */
 export function phaseOfStep(label: string): PlatformUpdatePhase | null {
+  // The full upgrade (upgrade/upgrade.ts) the converge job runs (#819).
+  if (label.startsWith("Pre-flight")) return "verify";
+  if (label.startsWith("Taking a pre-upgrade")) return "backup";
+  if (/^(Ensuring|Preparing|Removing) /.test(label)) return "converge";
   if (label.startsWith("Checking the health check")) return "health-baseline";
   if (label.startsWith("Applying DB migrations")) return "migrations";
   if (label.startsWith("Rolling ")) return "roll";

package/src/upgrader-setup.test.ts

@@ -75,7 +75,7 @@ describe("ensureUpgraderJob", () => {
     const mine = grants(calls).filter((g) => g.member === UPGRADER);
     expect(mine.map((g) => `${g.where} → ${g.role}`).sort()).toEqual(
       [
-        "projects add-iam-policy-binding acme → roles/run.developer",
+        "projects add-iam-policy-binding acme → projects/acme/roles/caeloUpgraderRun",
         "projects add-iam-policy-binding acme → roles/cloudsql.instanceUser",
         "projects add-iam-policy-binding acme → roles/logging.logWriter",
         "projects add-iam-policy-binding acme → projects/acme/roles/caeloUpgraderSqlBackup",
@@ -83,6 +83,7 @@ describe("ensureUpgraderJob", () => {
         "iam service-accounts add-iam-policy-binding caelo-production-gateway-sa@acme.iam.gserviceaccount.com → roles/iam.serviceAccountUser",
         "iam service-accounts add-iam-policy-binding caelo-production-upgrader@acme.iam.gserviceaccount.com → roles/iam.serviceAccountUser",
         "iam service-accounts add-iam-policy-binding caelo-mcp@acme.iam.gserviceaccount.com → roles/iam.serviceAccountTokenCreator",
+        "iam service-accounts add-iam-policy-binding caelo-production-audit@acme.iam.gserviceaccount.com → roles/iam.serviceAccountUser",
         "storage buckets add-iam-policy-binding gs://acme-caelo-ops → roles/storage.objectUser",
         "storage buckets add-iam-policy-binding gs://acme-caelo-ops → projects/acme/roles/caeloUpgraderOpsList",
       ].sort(),
@@ -169,11 +170,11 @@ describe("ensureUpgraderJob", () => {
 
   it("stops at the first failed grant and says which", async () => {
     const { run, calls } = fakeGcloud([
-      [/add-iam-policy-binding acme .*run\.developer/, fail("PERMISSION_DENIED")],
+      [/add-iam-policy-binding acme .*caeloUpgraderRun/, fail("PERMISSION_DENIED")],
     ]);
     const r = await ensureUpgraderJob(TARGET, { run, sleep: async () => {} });
     expect(r.ok ? "" : r.error).toContain(
-      "grant the upgrader roles/run.developer: PERMISSION_DENIED",
+      "grant the upgrader projects/acme/roles/caeloUpgraderRun: PERMISSION_DENIED",
     );
     expect(calls.some((c) => c.slice(0, 3).join(" ") === "run jobs deploy")).toBe(false);
   });