@caelo-cms/provisioning 0.10.61 → 0.10.64

package/src/upgrade/app-roll.ts

@@ -57,6 +57,8 @@ import {
   describeServiceJson,
   findCurrentRevision,
   findServiceUrl,
+  type GcloudCredentials,
+  lookupFailure,
   pinTraffic,
   resolveGcpResourceName,
 } from "./gcp-lookup.js";
@@ -78,6 +80,8 @@ export interface AppRollInstall {
   readonly domain: string;
   /** The protected staging host the admin is told about (#650). */
   readonly stagingHost: string;
+  /** Whose gcloud credentials the lookups run with; names the fix for an auth failure. */
+  readonly credentials: GcloudCredentials;
 }
 
 /** The release image one service rolls to. */
@@ -147,15 +151,20 @@ export async function planAppRoll(
   const services: ServicePlan[] = [];
   for (const slug of ROLL_ORDER) {
     const prefix = `caelo-${install.env}-${slug}`;
-    const serviceName = await resolveGcpResourceName(
+    const lookup = await resolveGcpResourceName(
       run,
       "run-service",
       prefix,
       install.projectId,
       install.region,
     );
-    if (!serviceName)
-      return { ok: false, error: `Could not find the ${prefix}* Cloud Run service` };
+    if (lookup.status !== "found") {
+      return {
+        ok: false,
+        error: lookupFailure(lookup, `${prefix}* Cloud Run service`, install.credentials),
+      };
+    }
+    const serviceName = lookup.name;
     const priorRevision = await findCurrentRevision(
       run,
       install.projectId,
@@ -404,10 +413,12 @@ export async function executeAppRoll(
   // migrations, and roll a healthy release back.
   if (deps.health) {
     const s = r.step("Checking the health check can reach admin + gateway...");
-    const base = await deps.health.baseline().catch((e: unknown) => ({
-      ok: false as const,
-      error: e instanceof Error ? e.message : String(e),
-    }));
+    const base = await deps.health
+      .baseline((reason) => s.progress(reason))
+      .catch((e: unknown) => ({
+        ok: false as const,
+        error: e instanceof Error ? e.message : String(e),
+      }));
     if (!base.ok) {
       s.failed(`Health check can't reach the install: ${base.error}`);
       r.warn(

package/src/upgrade/backup.ts

@@ -13,7 +13,7 @@
 
 import type { GcloudRunner } from "../gcloud-retry.js";
 import { gcpNamePrefix } from "../gcp-names.js";
-import { resolveGcpResourceName } from "./gcp-lookup.js";
+import { type GcloudCredentials, lookupFailure, resolveGcpResourceName } from "./gcp-lookup.js";
 
 /** A finished on-demand backup. */
 export interface SqlBackup {
@@ -38,22 +38,31 @@ export type SqlBackupResult =
  */
 export async function takeSqlBackup(
   run: GcloudRunner,
-  install: { readonly projectId: string; readonly env: string },
+  install: {
+    readonly projectId: string;
+    readonly env: string;
+    readonly credentials: GcloudCredentials;
+  },
   purpose: string,
   now: () => Date = () => new Date(),
 ): Promise<SqlBackupResult> {
-  const instance = await resolveGcpResourceName(
+  const lookup = await resolveGcpResourceName(
     run,
     "sql-instance",
     `${gcpNamePrefix(install.env)}-pg`,
     install.projectId,
   );
-  if (!instance) {
+  if (lookup.status !== "found") {
     return {
       ok: false,
-      error: `no ${gcpNamePrefix(install.env)}-pg* Cloud SQL instance in project ${install.projectId} (or listing it was denied)`,
+      error: lookupFailure(
+        lookup,
+        `${gcpNamePrefix(install.env)}-pg* Cloud SQL instance in project ${install.projectId}`,
+        install.credentials,
+      ),
     };
   }
+  const instance = lookup.name;
   const description = `caelo-cms ${purpose} ${now().toISOString()}`;
   const project = ["--project", install.projectId];
   const create = await run([

package/src/upgrade/converge.ts

@@ -65,6 +65,7 @@ import {
 import { ensureUpgraderJob, resolveUpgraderTarget } from "../upgrader-setup.js";
 import { ensureUrlMapRedirectsAccess } from "../url-map-redirects-access.js";
 import type { AppRollHooks, AppRollPlan, RollPlan, RollSlug } from "./app-roll.js";
+import { lookupFailure } from "./gcp-lookup.js";
 import type { UpgradeReporter } from "./reporter.js";
 
 /**
@@ -116,6 +117,13 @@ export interface ConvergeSession {
    * infrastructure contract version be recorded.
    */
   complete(): boolean;
+  /**
+   * Whether the part before the app-roll changed access the health check
+   * relies on (an IAM binding, MCP access, the gateway's ingress). Google
+   * applies such changes only after a while, so the health baseline then
+   * waits for a refusal to clear (health.ts, #734).
+   */
+  accessJustChanged(): boolean;
 }
 
 /** Start a converge for the install `plan` rolls. */
@@ -137,6 +145,7 @@ export function startConverge(
     services,
   };
   let incomplete = false;
+  let accessChanged = false;
   // Env vars the rolls must leave as the services have them, because the
   // thing they point at could not be set up.
   const leaveUntouched: string[] = [];
@@ -179,10 +188,12 @@ export function startConverge(
 
   async function stackInvariants(): Promise<boolean> {
     // IAM bindings + CDN policy added to the stacks after this install was
-    // provisioned (stack-contract.ts), and GCP's SSH/RDP-from-anywhere
-    // firewall rules removed (default-firewall.ts). Additive + idempotent.
+    // provisioned (stack-contract.ts), a gcp-firebase gateway's ingress
+    // (#732), and GCP's SSH/RDP-from-anywhere firewall rules removed
+    // (default-firewall.ts). Before the app-roll: its health baseline
+    // reaches the gateway through that ingress. Idempotent.
     const s = r.step(
-      "Ensuring the IAM bindings + CDN settings the stack declares, removing the default network's pre-populated firewall rules...",
+      "Ensuring the IAM bindings, CDN settings and gateway ingress the stack declares, removing the default network's pre-populated firewall rules...",
     );
     const report = await ensureStackInvariants(target, runDeps);
     const applied = report.outcomes.filter((o) => o.status === "applied");
@@ -193,6 +204,7 @@ export function startConverge(
         `Stack invariants ok (${report.outcomes.length - failed.length}/${report.outcomes.length}, ${applied.length} applied)`,
       );
     }
+    if (applied.length > 0) accessChanged = true;
     for (const o of applied) r.info(`  applied: ${o.id} (${o.why})`);
     for (const o of failed) {
       const line = `  ${o.onFailure === "abort" ? "FAILED" : "not applied"}: ${o.id} — ${o.why}\n    ${o.error ?? ""}`;
@@ -213,7 +225,7 @@ export function startConverge(
     // audit is what verifies it and fails the upgrade while it remains.
     const s = r.step("Removing public_role from the Cloud SQL superuser group...");
     const outcome = await revokePublicRoleMemberships(
-      { projectId: install.projectId, env: install.env },
+      { projectId: install.projectId, env: install.env, credentials: install.credentials },
       runDeps,
     );
     if (outcome.status === "failed") {
@@ -250,6 +262,8 @@ export function startConverge(
         : { ok: false as const, error: "admin IAP backend service not found" };
     if (mcp.ok) {
       s.done(`MCP access ready (${mcp.serviceAccount}; ${mcp.operators.length} operator(s))`);
+      if (mcp.granted.length > 0) accessChanged = true;
+      for (const g of mcp.granted) r.info(`  granted: ${g}`);
       return;
     }
     s.warned(`MCP access not configured: ${mcp.error}`);
@@ -408,12 +422,15 @@ export function startConverge(
     // #613 — public_role moves onto its own password right before the
     // gateway's roll. The gateway has no other database login (#635).
     const s = r.step("Preparing the gateway's database login (public_role)...");
-    const sqlInstance = await findSqlInstance(install, { run });
-    if (!sqlInstance) {
-      s.failed("Could not find the Cloud SQL instance. Aborting upgrade.");
+    const lookup = await findSqlInstance(install, { run });
+    if (lookup.status !== "found") {
+      s.failed(
+        `${lookupFailure(lookup, "Cloud SQL instance", install.credentials)}. Aborting upgrade.`,
+      );
       r.warn("Migrations ran; no traffic was shifted. Re-run upgrade once the instance is found.");
       return false;
     }
+    const sqlInstance = lookup.name;
     const roleTarget = { projectId: install.projectId, env: install.env, sqlInstance };
     const roleDeps = { ...runDeps, ...(deps.http ? { http: deps.http } : {}) };
     s.done("public_role ready to switch");
@@ -645,5 +662,6 @@ export function startConverge(
     },
 
     complete: () => !incomplete,
+    accessJustChanged: () => accessChanged,
   };
 }

package/src/upgrade/gcp-lookup.test.ts

@@ -10,13 +10,92 @@
 import { describe, expect, it } from "bun:test";
 import type { GcloudResult } from "../gcloud.js";
 import type { GcloudRunner } from "../gcloud-retry.js";
-import { findServingImage, servingImageDigests } from "./gcp-lookup.js";
+import {
+  findServingImage,
+  lookupFailure,
+  resolveGcpResourceName,
+  servingImageDigests,
+} from "./gcp-lookup.js";
 
 const INDEX = `sha256:${"a".repeat(64)}`;
 const PLATFORM = `sha256:${"b".repeat(64)}`;
 const REPO = "europe-west1-docker.pkg.dev/caelo-website/caelo-cms-images/admin";
 
 const ok = (stdout: string): GcloudResult => ({ ok: true, stdout, stderr: "", exitCode: 0 });
+const fail = (stderr: string): GcloudResult => ({ ok: false, stdout: "", stderr, exitCode: 1 });
+const REAUTH =
+  "ERROR: (gcloud.run.services.list) There was a problem refreshing your current auth tokens: Reauthentication failed. cannot prompt during non-interactive execution.";
+
+describe("resolveGcpResourceName (#630)", () => {
+  const lookup = (r: GcloudResult) =>
+    resolveGcpResourceName(async () => r, "run-service", "caelo-production-admin", "acme", "eu");
+
+  it("finds the first listed name", async () => {
+    expect(await lookup(ok("\ncaelo-production-admin-3efcfea\nother\n"))).toEqual({
+      status: "found",
+      name: "caelo-production-admin-3efcfea",
+    });
+  });
+
+  it("an empty listing is not-found", async () => {
+    expect(await lookup(ok("\n"))).toEqual({ status: "not-found" });
+  });
+
+  it("a failed listing is an error with gcloud's stderr, never not-found", async () => {
+    expect(await lookup(fail(REAUTH))).toEqual({
+      status: "error",
+      stderr: REAUTH,
+      authExpired: true,
+    });
+    expect(await lookup(fail("ERROR: PERMISSION_DENIED: run.services.list"))).toEqual({
+      status: "error",
+      stderr: "ERROR: PERMISSION_DENIED: run.services.list",
+      authExpired: false,
+    });
+  });
+});
+
+describe("lookupFailure", () => {
+  const what = "caelo-production-admin* Cloud Run service";
+
+  it("not-found reads as before", () => {
+    expect(lookupFailure({ status: "not-found" }, what, "operator")).toBe(
+      "Could not find the caelo-production-admin* Cloud Run service",
+    );
+  });
+
+  it("expired operator credentials name gcloud auth login and what gcloud said", () => {
+    const m = lookupFailure(
+      { status: "error", stderr: REAUTH, authExpired: true },
+      what,
+      "operator",
+    );
+    expect(m).toContain("gcloud credentials expired");
+    expect(m).toContain("gcloud auth login");
+    expect(m).toContain("gcloud auth application-default login");
+    expect(m).toContain("Reauthentication failed");
+    expect(m).not.toContain("Could not find");
+  });
+
+  it("a job never tells anyone to run gcloud auth login", () => {
+    const m = lookupFailure({ status: "error", stderr: REAUTH, authExpired: true }, what, "job");
+    expect(m).not.toContain("gcloud auth login");
+    expect(m).toContain("Nothing was changed");
+    expect(m).toContain("Cloud Shell");
+  });
+
+  it("any other failure surfaces gcloud's first error line", () => {
+    expect(
+      lookupFailure(
+        { status: "error", stderr: "ERROR: PERMISSION_DENIED: nope\nmore", authExpired: false },
+        what,
+        "operator",
+      ),
+    ).toBe(
+      "Could not list the caelo-production-admin* Cloud Run service: gcloud failed (PERMISSION_DENIED: nope)",
+    );
+  });
+});
 
 function gcloud(revision: GcloudResult): GcloudRunner {
   return async (args) => {

package/src/upgrade/gcp-lookup.ts

@@ -8,13 +8,70 @@
  */
 
 import { z } from "zod";
+import {
+  GCLOUD_REAUTH_HINT,
+  type GcloudResult,
+  gcloudErrorLine,
+  isGcloudAuthError,
+} from "../gcloud.js";
 import type { GcloudRunner } from "../gcloud-retry.js";
 
+/**
+ * Whose gcloud credentials a lookup ran with. It decides the fix an auth
+ * failure names: the Owner logs in again; a job in the install's project
+ * cannot, so it points at the break-glass run instead (#630).
+ */
+export type GcloudCredentials = "operator" | "job";
+
+/**
+ * The outcome of finding a deployed resource by name prefix. A failed
+ * listing is NOT "not found": expired credentials used to read as
+ * "Could not find caelo-production-admin*" while the service was there (#630).
+ */
+export type GcpLookup =
+  | { readonly status: "found"; readonly name: string }
+  | { readonly status: "not-found" }
+  | { readonly status: "error"; readonly stderr: string; readonly authExpired: boolean };
+
+/** A lookup that did not find its resource. */
+export type GcpLookupMiss = Exclude<GcpLookup, { readonly status: "found" }>;
+
+/** The first name a `--format=value(name)` listing printed, as a {@link GcpLookup}. */
+export function lookupFromListing(r: GcloudResult): GcpLookup {
+  if (!r.ok) {
+    return { status: "error", stderr: r.stderr, authExpired: isGcloudAuthError(r.stderr) };
+  }
+  const name = r.stdout
+    .split("\n")
+    .map((s) => s.trim())
+    .find((s) => s.length > 0);
+  return name ? { status: "found", name } : { status: "not-found" };
+}
+
+/**
+ * The error for a lookup that found nothing, for the operator or the job
+ * that ran it.
+ *
+ * @param what the resource, as the operator knows it, e.g. "caelo-production-admin* Cloud Run service"
+ */
+export function lookupFailure(
+  lookup: GcpLookupMiss,
+  what: string,
+  credentials: GcloudCredentials,
+): string {
+  if (lookup.status === "not-found") return `Could not find the ${what}`;
+  const said = gcloudErrorLine(lookup.stderr);
+  if (!lookup.authExpired) return `Could not list the ${what}: gcloud failed (${said})`;
+  return credentials === "operator"
+    ? `Could not look up the ${what}: ${GCLOUD_REAUTH_HINT} (gcloud: ${said})`
+    : `Could not look up the ${what}: Google rejected this job's credentials (${said}). Nothing was changed. Start the update again; if it fails the same way, run the break-glass upgrade from Cloud Shell, which repairs the job's access.`;
+}
+
 /**
  * Pulumi auto-naming appends a random 7-char suffix to every resource
  * (e.g. `caelo-production-admin-3efcfea`), so lifecycle commands find the
- * deployed name by prefix. Returns the first match, or null when the listing
- * fails or finds nothing.
+ * deployed name by prefix: the first match, "not-found" when the listing is
+ * empty, "error" with gcloud's stderr when the listing itself failed.
  */
 export async function resolveGcpResourceName(
   run: GcloudRunner,
@@ -22,7 +79,7 @@ export async function resolveGcpResourceName(
   prefix: string,
   projectId: string,
   region?: string,
-): Promise<string | null> {
+): Promise<GcpLookup> {
   const args =
     kind === "run-service"
       ? [
@@ -47,14 +104,7 @@ export async function resolveGcpResourceName(
           `name~^${prefix}`,
           "--format=value(name)",
         ];
-  const r = await run(args);
-  if (!r.ok) return null;
-  return (
-    r.stdout
-      .split("\n")
-      .map((s) => s.trim())
-      .find((s) => s.length > 0) ?? null
-  );
+  return lookupFromListing(await run(args));
 }
 
 /** A Cloud Run lookup is regional; there is no default region to guess. */

package/src/upgrade/health.test.ts

@@ -9,6 +9,7 @@ import {
   evaluateGatewayDeepHealth,
   type HealthTarget,
   healthOrigins,
+  IAM_PROPAGATION_WINDOW_MS,
 } from "./health.js";
 
 const migrations = (adminPending = 0, publicPending = 0) => ({
@@ -153,6 +154,7 @@ describe("createIapHealthProbe", () => {
       run: gcloudOk,
       http: net.http,
       sleep: async () => {},
+      accessJustChanged: false,
       now: () => 1000,
     });
     expect(await probe.deep({ version: "0.11.0" })).toEqual({
@@ -198,6 +200,7 @@ describe("createIapHealthProbe", () => {
       run: gcloudOk,
       http: net.http,
       sleep: async () => {},
+      accessJustChanged: false,
       attempts: 5,
     });
     expect((await probe.deep({ version: null })).ok).toBe(true);
@@ -211,6 +214,7 @@ describe("createIapHealthProbe", () => {
       run: gcloudOk,
       http: stuck.http,
       sleep: async () => {},
+      accessJustChanged: false,
       attempts: 2,
     });
     const r = await failing.deep({ version: null });
@@ -227,6 +231,7 @@ describe("createIapHealthProbe", () => {
       run: gcloudOk,
       http: net.http,
       sleep: async () => {},
+      accessJustChanged: false,
     });
     const r = await probe.baseline();
     expect(r.ok).toBe(false);
@@ -234,7 +239,7 @@ describe("createIapHealthProbe", () => {
       expect(r.error).toContain("IAP may not admit caelo-mcp@acme.iam.gserviceaccount.com");
   });
 
-  it("baseline succeeds when the old admin answers its shallow check and the gateway answers at all", async () => {
+  it("#733 — baseline succeeds when the old admin answers its shallow check and an old gateway its own JSON 404", async () => {
     const net = network({
       admin: () => [200, JSON.stringify({ ok: true, version: "0.10.43" })],
       gateway: () => [404, '{"ok":false}'],
@@ -243,6 +248,7 @@ describe("createIapHealthProbe", () => {
       run: gcloudOk,
       http: net.http,
       sleep: async () => {},
+      accessJustChanged: false,
     });
     expect(await probe.baseline()).toEqual({
       ok: true,
@@ -250,16 +256,133 @@ describe("createIapHealthProbe", () => {
     });
   });
 
-  it("baseline fails with the cause when the operator can't sign as the MCP account", async () => {
-    const http: HttpFetch = async () => ({
-      ok: false,
-      status: 403,
-      text: async () => "Permission iam.serviceAccounts.signJwt denied",
+  it("#733 — baseline fails when the gateway answers Google's HTML 404, naming the ingress", async () => {
+    const net = network({
+      admin: () => [200, JSON.stringify({ ok: true, version: "0.10.20" })],
+      gateway: () => [404, "<!DOCTYPE html><html><title>Error 404 (Not Found)!!1</title>"],
+    });
+    const probe = createIapHealthProbe(FIREBASE, {
+      run: gcloudOk,
+      http: net.http,
+      sleep: async () => {},
+      accessJustChanged: false,
     });
-    const probe = createIapHealthProbe(FIREBASE, { run: gcloudOk, http, sleep: async () => {} });
     const r = await probe.baseline();
     expect(r.ok).toBe(false);
-    if (!r.ok) expect(r.error).toContain("roles/iam.serviceAccountTokenCreator");
+    if (!r.ok) {
+      expect(r.error).toContain("answered HTTP 404 without JSON");
+      expect(r.error).toContain('ingress "all"');
+    }
+    const lb = createIapHealthProbe(
+      { ...FIREBASE, provider: "gcp" },
+      { run: gcloudOk, http: net.http, sleep: async () => {}, accessJustChanged: false },
+    );
+    const viaLb = await lb.baseline();
+    if (!viaLb.ok) expect(viaLb.error).toContain("route /api/* to the gateway backend");
+    expect(viaLb.ok).toBe(false);
+  });
+
+  describe("#734 — a refused signJwt right after converge granted it", () => {
+    /** signJwt answers 403 `denials` times, then signs. */
+    function propagating(denials: number) {
+      let signs = 0;
+      const http: HttpFetch = async (url) => {
+        if (url.includes(":signJwt")) {
+          signs++;
+          return signs <= denials
+            ? {
+                ok: false,
+                status: 403,
+                text: async () => "Permission 'iam.serviceAccounts.signJwt' denied",
+              }
+            : { ok: true, status: 200, text: async () => JSON.stringify({ signedJwt: "s.j" }) };
+        }
+        return url.includes("admin")
+          ? { ok: true, status: 200, text: async () => JSON.stringify({ version: "0.10.20" }) }
+          : { ok: false, status: 404, text: async () => '{"ok":false}' };
+      };
+      return { http, signs: () => signs };
+    }
+
+    it("waits for the grant to apply, saying why, then passes", async () => {
+      const net = propagating(4);
+      const slept: number[] = [];
+      const waits: string[] = [];
+      const probe = createIapHealthProbe(FIREBASE, {
+        run: gcloudOk,
+        http: net.http,
+        sleep: async (ms) => {
+          slept.push(ms);
+        },
+        accessJustChanged: true,
+      });
+      expect(await probe.baseline((reason) => waits.push(reason))).toEqual({
+        ok: true,
+        detail: "admin 0.10.20 and gateway reachable",
+      });
+      expect(net.signs()).toBe(5);
+      expect(slept).toEqual([5_000, 10_000, 20_000, 30_000]);
+      expect(waits).toHaveLength(4);
+      expect(waits[0]).toContain("Waiting for the access the converge step just granted");
+    });
+
+    it("gives up after the propagation window: the grant exists, re-run in a few minutes", async () => {
+      const net = propagating(Number.POSITIVE_INFINITY);
+      let slept = 0;
+      const probe = createIapHealthProbe(FIREBASE, {
+        run: gcloudOk,
+        http: net.http,
+        sleep: async (ms) => {
+          slept += ms;
+        },
+        accessJustChanged: true,
+      });
+      const r = await probe.baseline();
+      expect(slept).toBe(IAM_PROPAGATION_WINDOW_MS);
+      expect(r.ok).toBe(false);
+      if (!r.ok) {
+        expect(r.error).toContain("The grant exists; nothing was changed: re-run upgrade");
+        expect(r.error).toContain("after 7 minutes");
+      }
+    });
+
+    it("fails at once when nothing was just granted: the grant is missing", async () => {
+      const net = propagating(Number.POSITIVE_INFINITY);
+      let slept = 0;
+      const probe = createIapHealthProbe(FIREBASE, {
+        run: gcloudOk,
+        http: net.http,
+        sleep: async (ms) => {
+          slept += ms;
+        },
+        accessJustChanged: false,
+      });
+      const r = await probe.baseline();
+      expect(slept).toBe(0);
+      expect(net.signs()).toBe(1);
+      expect(r.ok).toBe(false);
+      if (!r.ok) {
+        expect(r.error).toContain("it lacks roles/iam.serviceAccountTokenCreator");
+        expect(r.error).toContain("run upgrade as an allowlisted account");
+      }
+    });
+
+    it("never waits out a failure that is not a refusal", async () => {
+      const http: HttpFetch = async () => {
+        throw new Error("ENOTFOUND");
+      };
+      let slept = 0;
+      const probe = createIapHealthProbe(FIREBASE, {
+        run: gcloudOk,
+        http,
+        sleep: async (ms) => {
+          slept += ms;
+        },
+        accessJustChanged: true,
+      });
+      expect((await probe.baseline()).ok).toBe(false);
+      expect(slept).toBe(0);
+    });
   });
 
   it("a transport failure while signing is a failed check, not a throw", async () => {
@@ -270,6 +393,7 @@ describe("createIapHealthProbe", () => {
       run: gcloudOk,
       http,
       sleep: async () => {},
+      accessJustChanged: false,
       attempts: 2,
     });
     const r = await probe.deep({ version: null });