@caelo-cms/provisioning 0.10.61 → 0.10.62

package/src/upgrade/health.ts

@@ -24,9 +24,18 @@
  * that account). The gateway is public by design and gets no credential.
  *
  * Before anything changes, {@link HealthProbe.baseline} proves the probe can
- * reach both services at all. Otherwise a DNS, certificate or IAP problem
- * would only show after migrations, as a "failed" health check that rolls a
- * healthy release back.
+ * reach both services at all — each answering with its own JSON, so a
+ * Google front-end 404 for a gateway whose ingress refuses the request
+ * counts as unreachable (#733). Otherwise a DNS, certificate, ingress or IAP
+ * problem would only show after migrations, as a "failed" health check that
+ * rolls a healthy release back.
+ *
+ * IAM is eventually consistent (Google: typically under 2 minutes, up to
+ * 7). When the converge step that ran just before changed the access the
+ * probe relies on — the operator's right to sign as `caelo-mcp@`, that
+ * account's IAP access, the gateway's ingress — the baseline keeps retrying
+ * a refusal for {@link IAM_PROPAGATION_WINDOW_MS} instead of failing the
+ * upgrade on a grant that has not taken effect yet (#734).
  */
 
 import type { GcloudRunner, Sleep } from "../gcloud-retry.js";
@@ -38,8 +47,13 @@ export type HealthOutcome =
   | { readonly ok: false; readonly error: string };
 
 export interface HealthProbe {
-  /** Reach both services as they serve now (before any change). */
-  baseline(): Promise<HealthOutcome>;
+  /**
+   * Reach both services as they serve now (before any change).
+   *
+   * @param onWait told why the baseline waits, each time it waits for
+   *   access the converge step just granted to take effect
+   */
+  baseline(onWait?: (reason: string) => void): Promise<HealthOutcome>;
   /**
    * Deep-check both services after the roll, retrying while the new
    * revisions warm up. `version` is the release the roll targets (null when
@@ -169,10 +183,23 @@ export function healthOrigins(target: HealthTarget): { admin: string; gateway: s
       };
 }
 
+/**
+ * How long the baseline treats a refusal as access still propagating: IAM's
+ * documented worst case for a new binding to take effect.
+ */
+export const IAM_PROPAGATION_WINDOW_MS = 7 * 60_000;
+
 export interface HealthProbeDeps {
   readonly run: GcloudRunner;
   readonly http: HttpFetch;
   readonly sleep: Sleep;
+  /**
+   * Whether the converge step that ran just before changed the access the
+   * probe relies on (IAM grants, the gateway's ingress). Only then does the
+   * baseline wait for a refusal to clear: a grant that existed before this
+   * run and still refuses is missing, and waiting would only delay saying so.
+   */
+  readonly accessJustChanged: boolean;
   /** Deep-check attempts while the new revisions warm up (default 12). */
   readonly attempts?: number;
   /** Pause between attempts in ms (default 5000). */
@@ -181,7 +208,21 @@ export interface HealthProbeDeps {
   readonly now?: () => number;
 }
 
+/**
+ * Why the baseline could not reach the install. The refusals other than
+ * `other` are what fresh access looks like before it applies: the baseline
+ * waits them out after a converge that just changed access.
+ */
+type BaselineRefusal = "sign-denied" | "iap-refused" | "gateway-not-caelo" | "other";
+
+type BaselineOutcome =
+  | { readonly ok: true; readonly detail: string }
+  | { readonly ok: false; readonly error: string; readonly refusal: BaselineRefusal };
+
 const JWT_LIFETIME_S = 600;
+/** First pause between baseline attempts while access propagates; doubles up to the cap. */
+const SETTLE_FIRST_DELAY_MS = 5_000;
+const SETTLE_MAX_DELAY_MS = 30_000;
 const IAM_CREDENTIALS = "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts";
 
 /** The IAP-backed probe of a gcp / gcp-firebase install. */
@@ -197,24 +238,35 @@ export function createIapHealthProbe(target: HealthTarget, deps: HealthProbeDeps
    * account. Never throws: a transport failure is a failed outcome, so the
    * retries and the app-roll's rollback still run.
    */
-  async function iapToken(): Promise<{ ok: true; jwt: string } | { ok: false; error: string }> {
+  async function iapToken(): Promise<SignOutcome> {
     try {
       return await signIapToken();
     } catch (e) {
       return {
         ok: false,
         error: `could not sign an IAP credential as ${signer}: ${e instanceof Error ? e.message : String(e)}`,
+        denied: false,
       };
     }
   }
 
-  async function signIapToken(): Promise<{ ok: true; jwt: string } | { ok: false; error: string }> {
+  type SignOutcome =
+    | { ok: true; jwt: string }
+    | {
+        ok: false;
+        error: string;
+        /** IAM refused the signing (HTTP 403): the token-creator grant is missing or not in effect yet. */
+        denied: boolean;
+      };
+
+  async function signIapToken(): Promise<SignOutcome> {
     const token = await deps.run(["auth", "print-access-token"]);
     const bearer = token.ok ? token.stdout.trim() : "";
     if (!bearer) {
       return {
         ok: false,
         error: `gcloud auth print-access-token: ${token.stderr.trim() || "empty"}`,
+        denied: false,
       };
     }
     const iat = now();
@@ -232,14 +284,43 @@ export function createIapHealthProbe(target: HealthTarget, deps: HealthProbeDeps
       }),
     });
     const text = await res.text();
+    if (res.status === 403) {
+      return {
+        ok: false,
+        error: `could not sign an IAP credential as ${signer} (HTTP 403): ${text.slice(0, 300)}`,
+        denied: true,
+      };
+    }
     if (!res.ok) {
       return {
         ok: false,
-        error: `could not sign an IAP credential as ${signer} (HTTP ${res.status}). Your account needs roles/iam.serviceAccountTokenCreator on it — upgrade's converge step grants it to every IAP-allowlisted user. ${text.slice(0, 300)}`,
+        error: `could not sign an IAP credential as ${signer} (HTTP ${res.status}): ${text.slice(0, 300)}`,
+        denied: false,
       };
     }
     const jwt = parseJson<{ signedJwt?: string }>(text)?.signedJwt;
-    return jwt ? { ok: true, jwt } : { ok: false, error: "signJwt returned no signedJwt" };
+    return jwt
+      ? { ok: true, jwt }
+      : { ok: false, error: "signJwt returned no signedJwt", denied: false };
+  }
+
+  /**
+   * What to do about a refused signJwt, depending on whether this run's
+   * converge just granted the access or it was there before.
+   */
+  function signDeniedHelp(waitedMs: number): string {
+    const role = "roles/iam.serviceAccountTokenCreator";
+    if (deps.accessJustChanged) {
+      return `The upgrade's converge step changed who may sign as ${signer} just now (it grants ${role} to every user on the admin's IAP allowlist), and Google IAM had not applied it after ${Math.round(waitedMs / 60_000)} minutes of waiting. The grant exists; nothing was changed: re-run upgrade in a few minutes.`;
+    }
+    return `Your account may not sign as ${signer}: it lacks ${role} on it. Upgrade's converge step grants that role to every user on the admin's IAP allowlist, so run upgrade as an allowlisted account (gcloud auth login), or add yours to the allowlist. If an upgrade run in the last few minutes granted it, IAM may still be applying it: re-run upgrade in a few minutes.`;
+  }
+
+  /** Why the gateway's answer is not the gateway's own, and what fixes it. */
+  function gatewayRoutingHelp(): string {
+    return target.provider === "gcp-firebase"
+      ? `Requests to its run.app URL do not reach the Caelo gateway: its Cloud Run ingress likely refuses them. The gcp-firebase stack declares ingress "all"; upgrade's converge step sets it. Check it with: gcloud run services describe <gateway service> --region=<region> --project=${target.projectId} --format='value(metadata.annotations."run.googleapis.com/ingress")'`
+      : `Requests to ${origins.gateway}/api/ do not reach the Caelo gateway: the load balancer's URL map must route /api/* to the gateway backend. Check the install's load balancer, then re-run upgrade.`;
   }
 
   async function get(url: string, headers: Record<string, string>) {
@@ -251,30 +332,73 @@ export function createIapHealthProbe(target: HealthTarget, deps: HealthProbeDeps
     }
   }
 
+  async function baselineOnce(): Promise<BaselineOutcome> {
+    const token = await iapToken();
+    if (!token.ok) {
+      return { ok: false, error: token.error, refusal: token.denied ? "sign-denied" : "other" };
+    }
+    const auth = { Authorization: `Bearer ${token.jwt}` };
+    const admin = await get(`${origins.admin}/_caelo/health`, auth);
+    if (!admin.reached) {
+      return {
+        ok: false,
+        error: `admin unreachable at ${origins.admin}: ${admin.error}`,
+        refusal: "other",
+      };
+    }
+    // The serving (old) admin answers its shallow health JSON. Anything
+    // else — IAP's login redirect, a 401/403, an HTML error page — means
+    // the probe would not see the new revision either.
+    const body = parseJson<{ version?: string }>(admin.text);
+    if (!body?.version) {
+      return {
+        ok: false,
+        error: `admin at ${origins.admin} did not answer its health check (HTTP ${admin.status}); IAP may not admit ${signer}`,
+        // IAP refuses with 401/403 until a fresh grant to caelo-mcp applies.
+        refusal: admin.status === 401 || admin.status === 403 ? "iap-refused" : "other",
+      };
+    }
+    const gateway = await get(`${origins.gateway}/api/_caelo/health`, {});
+    if (!gateway.reached) {
+      return {
+        ok: false,
+        error: `gateway unreachable at ${origins.gateway}: ${gateway.error}`,
+        refusal: "other",
+      };
+    }
+    // Any JSON counts, whatever the status: a gateway older than its health
+    // route answers its own JSON 404. An HTML or empty answer comes from
+    // Google's front end, not from Caelo (#733).
+    if (parseJson<unknown>(gateway.text) === null) {
+      return {
+        ok: false,
+        error: `gateway at ${origins.gateway} answered HTTP ${gateway.status} without JSON. ${gatewayRoutingHelp()}`,
+        // A just-opened ingress takes a moment, like an IAM grant.
+        refusal: "gateway-not-caelo",
+      };
+    }
+    return { ok: true, detail: `admin ${body.version} and gateway reachable` };
+  }
+
   return {
-    async baseline() {
-      const token = await iapToken();
-      if (!token.ok) return token;
-      const auth = { Authorization: `Bearer ${token.jwt}` };
-      const admin = await get(`${origins.admin}/_caelo/health`, auth);
-      if (!admin.reached) {
-        return { ok: false, error: `admin unreachable at ${origins.admin}: ${admin.error}` };
+    async baseline(onWait) {
+      const windowMs = deps.accessJustChanged ? IAM_PROPAGATION_WINDOW_MS : 0;
+      let waited = 0;
+      let delay = SETTLE_FIRST_DELAY_MS;
+      for (;;) {
+        const outcome = await baselineOnce();
+        if (outcome.ok) return outcome;
+        if (outcome.refusal === "other" || waited >= windowMs) {
+          return { ok: false, error: explainRefusal(outcome, waited) };
+        }
+        onWait?.(
+          `Waiting for the access the converge step just granted to take effect (${Math.round(waited / 1000)}s of up to ${windowMs / 60_000} min): ${outcome.error.slice(0, 200)}`,
+        );
+        const pause = Math.min(delay, windowMs - waited);
+        await deps.sleep(pause);
+        waited += pause;
+        delay = Math.min(delay * 2, SETTLE_MAX_DELAY_MS);
       }
-      // The serving (old) admin answers its shallow health JSON. Anything
-      // else — IAP's login redirect, a 401/403, an HTML error page — means
-      // the probe would not see the new revision either.
-      const body = parseJson<{ version?: string }>(admin.text);
-      if (!body?.version) {
-        return {
-          ok: false,
-          error: `admin at ${origins.admin} did not answer its health check (HTTP ${admin.status}); IAP may not admit ${signer}`,
-        };
-      }
-      const gateway = await get(`${origins.gateway}/api/_caelo/health`, {});
-      if (!gateway.reached) {
-        return { ok: false, error: `gateway unreachable at ${origins.gateway}: ${gateway.error}` };
-      }
-      return { ok: true, detail: `admin ${body.version} and gateway reachable` };
     },
 
     async deep(expect) {
@@ -288,6 +412,20 @@ export function createIapHealthProbe(target: HealthTarget, deps: HealthProbeDeps
     },
   };
 
+  /**
+   * The baseline's final error: a refusal that outlived the wait says the
+   * access was granted and has not applied yet (re-run later), apart from
+   * a refusal with no fresh grant behind it (missing access).
+   */
+  function explainRefusal(
+    outcome: Extract<BaselineOutcome, { ok: false }>,
+    waitedMs: number,
+  ): string {
+    if (outcome.refusal === "sign-denied") return `${outcome.error}\n${signDeniedHelp(waitedMs)}`;
+    if (outcome.refusal === "other" || !deps.accessJustChanged) return outcome.error;
+    return `${outcome.error}\nThe upgrade's converge step changed this access just now and it had not applied after ${Math.round(waitedMs / 60_000)} minutes of waiting. Nothing was changed: re-run upgrade in a few minutes.`;
+  }
+
   async function deepOnce(expectedVersion: string | null): Promise<HealthOutcome> {
     const token = await iapToken();
     if (!token.ok) return token;

package/src/upgrade/release-images.test.ts

@@ -4,6 +4,7 @@ import { describe, expect, it } from "bun:test";
 import {
   canonicalDigest,
   canonicalReleaseTag,
+  cosignInstallHint,
   cosignVerifyArgv,
   parseBearerChallenge,
   RELEASE_SIGNER_IDENTITY_REGEXP,
@@ -117,3 +118,19 @@ describe("canonical values from the network", () => {
     expect(canonicalReleaseTag("latest")).toBeNull();
   });
 });
+
+describe("cosignInstallHint (#816)", () => {
+  it("never offers --skip-verify to the Owner", () => {
+    for (const platform of ["darwin", "linux", "win32"] as const) {
+      expect(cosignInstallHint(platform)).not.toContain("skip-verify");
+    }
+  });
+
+  it("is not brew-only: Cloud Shell and Linux get a verified path", () => {
+    const linux = cosignInstallHint("linux");
+    expect(linux).not.toContain("brew");
+    expect(linux).toContain("./run.sh upgrade");
+    expect(linux).toContain("docs.sigstore.dev");
+    expect(cosignInstallHint("darwin")).toContain("brew install cosign");
+  });
+});

package/src/upgrade/release-images.ts

@@ -300,23 +300,40 @@ function probeBinary(argv: string[]): boolean {
 }
 
 /**
- * Ensure cosign is on PATH. If not — and the operator is on macOS with
- * brew available — offer to `brew install cosign` interactively. Falls
- * back to printing the platform install URL + the --skip-verify
- * escape hatch when auto-install isn't possible.
+ * How to get cosign onto PATH on `platform`. Owner-facing, so it names
+ * only verified installs and never the unverified escape hatch: an Owner
+ * who cannot install cosign must not be steered to roll unsigned images
+ * (#816). Cloud Shell (Linux) is where the break-glass upgrade runs; its
+ * `./run.sh upgrade` installs a checksum-verified cosign itself.
+ */
+export function cosignInstallHint(platform: NodeJS.Platform): string {
+  const lines = ["Install cosign, then run this command again:"];
+  if (platform === "darwin") lines.push("  • brew install cosign");
+  lines.push(
+    "  • Cloud Shell: run the upgrade with `./run.sh upgrade` from https://github.com/caelo-cms/install — it installs a verified cosign first",
+    "  • Linux / Windows: the signed release binary, https://docs.sigstore.dev/cosign/system_config/installation/",
+  );
+  return lines.join("\n");
+}
+
+/**
+ * Ensure cosign is on PATH; the upgrade's pre-flight runs this before it
+ * plans or changes anything, so a missing cosign never stops an upgrade
+ * half-way (#816). On macOS with brew it offers `brew install cosign`;
+ * elsewhere it prints {@link cosignInstallHint}.
  *
- * Returns true if cosign is callable after this function returns.
+ * @returns whether cosign is callable afterwards
  */
-async function ensureCosignAvailable(): Promise<boolean> {
+export async function ensureCosignAvailable(): Promise<boolean> {
   if (probeBinary(["cosign", "version"])) return true;
 
-  log.error(red("cosign not found on PATH — required for image-signature verification."));
+  log.error(red("cosign not found on PATH — required to verify the release images' signatures."));
 
   // macOS auto-install offer. Only when `brew` is itself on PATH —
   // otherwise the operator is on a stripped-down install where
   // homebrew was never set up, and the manual instructions are more
   // useful than a failing brew shell-out.
-  if (process.platform === "darwin" && probeBinary(["brew", "--version"])) {
+  if (process.platform === "darwin" && process.stdin.isTTY && probeBinary(["brew", "--version"])) {
     const proceed = await confirm({
       message: "Install cosign via brew now? (one-shot: brew install cosign)",
       initialValue: true,
@@ -338,12 +355,7 @@ async function ensureCosignAvailable(): Promise<boolean> {
     }
   }
 
-  log.warn(
-    "Install cosign:\n" +
-      "  • brew install cosign           (macOS)\n" +
-      "  • https://docs.sigstore.dev/cosign/installation/  (other)\n" +
-      "Or pass --skip-verify to roll without signature checks (NOT recommended for production).",
-  );
+  log.warn(cosignInstallHint(process.platform));
   return false;
 }
 

package/src/upgrade/reporter.ts

@@ -16,6 +16,8 @@ export interface ReportStep {
   failed(message: string): void;
   /** Ended without success, but the upgrade continues. */
   warned(message: string): void;
+  /** Still running: say what it is waiting for now. */
+  progress(message: string): void;
 }
 
 export interface UpgradeReporter {
@@ -36,6 +38,7 @@ export function clackReporter(): UpgradeReporter {
         done: (m) => s.stop(green(m)),
         failed: (m) => s.stop(red(m)),
         warned: (m) => s.stop(yellow(m)),
+        progress: (m) => s.message(m),
       };
     },
     info: (m) => log.info(m),

package/src/upgrade/upgrade.test.ts

@@ -21,7 +21,7 @@ import {
 } from "./app-roll.js";
 import { restoreCommand, takeSqlBackup } from "./backup.js";
 import { needsConverge, startConverge } from "./converge.js";
-import type { HealthOutcome, HealthProbe } from "./health.js";
+import { createIapHealthProbe, type HealthOutcome, type HealthProbe } from "./health.js";
 import type { UpgradeReporter } from "./reporter.js";
 import {
   type GcpUpgradeDeps,
@@ -48,9 +48,13 @@ const secretRef = (name: string, secret: string) => ({
   valueFrom: { secretKeyRef: { name: `caelo-production-${secret}`, key: "latest" } },
 });
 
-/** A described service, as `gcloud run services describe --format=json` returns it. */
-const serviceJson = (image: string, env: unknown[]) =>
+/**
+ * A described service, as `gcloud run services describe --format=json`
+ * returns it. `ingress`: the service-level setting (#732).
+ */
+const serviceJson = (image: string, env: unknown[], ingress = "all") =>
   JSON.stringify({
+    metadata: { annotations: { "run.googleapis.com/ingress": ingress } },
     spec: {
       template: {
         metadata: {
@@ -150,6 +154,7 @@ function recordingReporter() {
         done: (m) => lines.push(`done: ${m}`),
         failed: (m) => lines.push(`failed: ${m}`),
         warned: (m) => lines.push(`warned: ${m}`),
+        progress: (m) => lines.push(`progress: ${m}`),
       };
     },
     info: (m) => lines.push(`info: ${m}`),
@@ -188,6 +193,7 @@ const TARGET: GcpUpgradeTarget = {
   infraContractVersion: 1,
   skipVerify: false,
   skipHealthCheck: false,
+  credentials: "operator",
 };
 
 /** The same install on the `gcp` provider (load balancer + URL map). */
@@ -788,6 +794,88 @@ describe("upgradeGcpInstall — converge, then app-roll", () => {
     expect(calls.some((c) => APP_ROLL_WRITE.test(c))).toBe(false);
   });
 
+  it("#732 — opens a load-balancer-only gcp-firebase gateway in converge, before the baseline and migrations", async () => {
+    const events: string[] = [];
+    const internalGateway = serviceJson(
+      "registry/gateway@sha256:old",
+      JSON.parse(GATEWAY_JSON).spec.template.spec.containers[0].env,
+      "internal-and-cloud-load-balancing",
+    );
+    const { deps, lines } = upgradeDeps(events, {
+      overrides: [[new RegExp(`describe ${GATEWAY} .*--format=json`), ok(internalGateway)]],
+    });
+    expect(await upgradeGcpInstall(TARGET, deps)).toBe(true);
+    const ingress = indexOf(
+      events,
+      new RegExp(`^gcloud run services update ${GATEWAY} .*--ingress=all`),
+    );
+    expect(ingress).toBeLessThan(indexOf(events, /^health baseline/));
+    expect(ingress).toBeLessThan(indexOf(events, /^migrate /));
+    // Its own update: the image roll does not carry it (no new revision).
+    expect(events.filter((e) => e.includes("--ingress=all"))).toHaveLength(1);
+    expect(lines).toContain(
+      `info:   applied: ingress internal-and-cloud-load-balancing → all on ${GATEWAY} (the gcp-firebase stack's gateway ingress (#650); the health check, the staging domain and remote MCP reach its run.app URL)`,
+    );
+  });
+
+  it("#732 — leaves a gateway that is already open alone, and never touches the gcp gateway's ingress", async () => {
+    for (const target of [TARGET, GCP_TARGET]) {
+      const events: string[] = [];
+      const { deps } = upgradeDeps(events);
+      await upgradeGcpInstall(target, deps);
+      expect(events.some((e) => e.includes("--ingress"))).toBe(false);
+    }
+  });
+
+  it("#734 — tells the health probe when converge just changed access, so its baseline waits", async () => {
+    const seen: boolean[] = [];
+    const events: string[] = [];
+    const { deps } = upgradeDeps(events);
+    await upgradeGcpInstall(TARGET, {
+      ...deps,
+      healthProbe: async (_plan, context) => {
+        seen.push(context.accessJustChanged);
+        return { ok: true, probe: fakeHealth(events) };
+      },
+    });
+    // The fake install's empty IAM policies make converge add every binding.
+    expect(seen).toEqual([true]);
+  });
+
+  it("#733 — a gateway answering Google's HTML 404 stops the upgrade before migrations and any roll", async () => {
+    const events: string[] = [];
+    const { deps, calls, lines } = upgradeDeps(events);
+    const htmlGateway: HttpFetch = async (url) => {
+      const [status, text] = url.includes(":signJwt")
+        ? [200, JSON.stringify({ signedJwt: "signed.jwt" })]
+        : url.includes("admin")
+          ? [200, JSON.stringify({ ok: true, version: "0.10.20" })]
+          : [404, "<html><title>Error 404 (Not Found)!!1</title></html>"];
+      return { ok: status < 400, status, text: async () => text };
+    };
+    expect(
+      await upgradeGcpInstall(TARGET, {
+        ...deps,
+        healthProbe: async () => ({
+          ok: true,
+          probe: createIapHealthProbe(
+            {
+              provider: "gcp-firebase",
+              projectId: "acme",
+              domain: "acme.com",
+              adminServiceUrl: `https://${ADMIN}-xyz.a.run.app`,
+              gatewayServiceUrl: `https://${GATEWAY}-xyz.a.run.app`,
+            },
+            { run: deps.run, http: htmlGateway, sleep: noSleep, accessJustChanged: false },
+          ),
+        }),
+      }),
+    ).toBe(false);
+    expect(events.some((e) => e.startsWith("migrate"))).toBe(false);
+    expect(calls.some((c) => APP_ROLL_WRITE.test(c))).toBe(false);
+    expect(lines.some((l) => l.includes("answered HTTP 404 without JSON"))).toBe(true);
+  });
+
   it("--skip-health-check rolls without a probe", async () => {
     const events: string[] = [];
     const { deps } = upgradeDeps(events);
@@ -814,6 +902,7 @@ const INSTALL: AppRollInstall = {
   env: "production",
   domain: "acme.com",
   stagingHost: "staging.acme.com",
+  credentials: "operator",
 };
 
 async function plannedRoll(run: GcloudRunner): Promise<AppRollPlan> {
@@ -942,7 +1031,7 @@ describe("takeSqlBackup", () => {
     const { run, calls } = fakeInstall(events);
     const r = await takeSqlBackup(
       run,
-      { projectId: "acme", env: "production" },
+      { projectId: "acme", env: "production", credentials: "operator" },
       "backup",
       () => new Date("2026-10-13T10:00:00Z"),
     );
@@ -959,7 +1048,11 @@ describe("takeSqlBackup", () => {
 
   it("fails loudly when the backup's id can't be found", async () => {
     const { run } = fakeInstall([], [[/^sql backups list/, ok("")]]);
-    const r = await takeSqlBackup(run, { projectId: "acme", env: "production" }, "backup");
+    const r = await takeSqlBackup(
+      run,
+      { projectId: "acme", env: "production", credentials: "operator" },
+      "backup",
+    );
     expect(r.ok).toBe(false);
     if (!r.ok)
       expect(r.error).toContain("gcloud sql backups list --instance=caelo-production-pg-abc");
@@ -967,11 +1060,47 @@ describe("takeSqlBackup", () => {
 
   it("fails when the install has no Cloud SQL instance", async () => {
     const { run } = fakeInstall([], [[/^sql instances list/, ok("")]]);
-    const r = await takeSqlBackup(run, { projectId: "acme", env: "production" }, "backup");
+    const r = await takeSqlBackup(
+      run,
+      { projectId: "acme", env: "production", credentials: "operator" },
+      "backup",
+    );
     expect(r).toEqual({
       ok: false,
-      error:
-        "no caelo-production-pg* Cloud SQL instance in project acme (or listing it was denied)",
+      error: "Could not find the caelo-production-pg* Cloud SQL instance in project acme",
     });
   });
+
+  it("reports expired credentials instead of a missing instance (#630)", async () => {
+    const { run } = fakeInstall(
+      [],
+      [
+        [
+          /^sql instances list/,
+          fail(
+            "ERROR: (gcloud.sql.instances.list) Reauthentication failed. cannot prompt during non-interactive execution.",
+          ),
+        ],
+      ],
+    );
+    const operator = await takeSqlBackup(
+      run,
+      { projectId: "acme", env: "production", credentials: "operator" },
+      "backup",
+    );
+    expect(operator.ok).toBe(false);
+    if (operator.ok) return;
+    expect(operator.error).toContain("gcloud credentials expired");
+    expect(operator.error).toContain("gcloud auth login");
+    expect(operator.error).not.toContain("Could not find");
+    const job = await takeSqlBackup(
+      run,
+      { projectId: "acme", env: "production", credentials: "job" },
+      "backup",
+    );
+    expect(job.ok).toBe(false);
+    if (job.ok) return;
+    expect(job.error).toContain("rejected this job's credentials");
+    expect(job.error).not.toContain("gcloud auth login");
+  });
 });

package/src/upgrade/upgrade.ts

@@ -40,6 +40,7 @@ import {
 } from "./app-roll.js";
 import { backupFailureHelp, restoreCommand, type SqlBackup, takeSqlBackup } from "./backup.js";
 import { startConverge } from "./converge.js";
+import type { GcloudCredentials } from "./gcp-lookup.js";
 import type { HealthProbe } from "./health.js";
 import type { UpgradeReporter } from "./reporter.js";
 
@@ -63,6 +64,8 @@ export interface GcpUpgradeTarget {
    * release manifest (the converge job knows it; the terminal does not).
    */
   readonly migrationsContract?: boolean | null;
+  /** Whose gcloud credentials run the sequence: the Owner's, or the converge job's (#630). */
+  readonly credentials: GcloudCredentials;
 }
 
 /**
@@ -91,8 +94,14 @@ export interface GcpUpgradeDeps {
   readonly migrate: (
     imageRef: string,
   ) => Promise<{ readonly ok: boolean; readonly error?: string }>;
+  /**
+   * The health probe for `plan`'s install. `accessJustChanged`: the
+   * converge before it changed access the probe relies on, so its baseline
+   * waits for that to take effect (health.ts, #734).
+   */
   readonly healthProbe: (
     plan: AppRollPlan,
+    context: { readonly accessJustChanged: boolean },
   ) => Promise<
     | { readonly ok: true; readonly probe: HealthProbe }
     | { readonly ok: false; readonly error: string }
@@ -164,6 +173,7 @@ export async function upgradeGcpInstallOutcome(
       env: target.env,
       domain: target.domain,
       stagingHost: target.stagingHost,
+      credentials: target.credentials,
     },
     images.images,
     run,
@@ -198,7 +208,7 @@ export async function upgradeGcpInstallOutcome(
     const sBak = r.step("Taking a pre-upgrade Cloud SQL backup...");
     const taken = await takeSqlBackup(
       run,
-      { projectId, env: target.env },
+      { projectId, env: target.env, credentials: target.credentials },
       `pre-upgrade to ${target.targetTag}`,
       deps.now,
     );
@@ -229,7 +239,9 @@ export async function upgradeGcpInstallOutcome(
   // 5. App-roll, with the converge steps that sit between its own.
   let health: HealthProbe | null = null;
   if (!target.skipHealthCheck) {
-    const probe = await deps.healthProbe(plan);
+    const probe = await deps.healthProbe(plan, {
+      accessJustChanged: converge.accessJustChanged(),
+    });
     if (!probe.ok) {
       r.error(probe.error);
       r.warn("No migrations ran and no traffic moved.");