@caelo-cms/provisioning 0.11.11 → 0.11.12

package/src/converge-job.ts

@@ -16,7 +16,9 @@
  *     verifies its signature (release-manifest.ts), and decides
  *     (upgrader-assess.ts, `route: "converge"`): no downgrade, nothing
  *     below minFromVersion, no automatic converge unless the policy allows
- *     it; then it checks its own IAM access (converge-job-preflight.ts).
+ *     it; then it checks its own IAM access (converge-job-preflight.ts) and
+ *     stops on anything missing — except a role of the self-grant
+ *     allowlist, which it leaves to stage 2.
  *     Then it verifies the target's ops image, points this job at it BY DIGEST and starts a new
  *     execution, handing the request over to it. It changes nothing else:
  *     the converge logic always comes from the release it installs.
@@ -25,7 +27,9 @@
  *     version is the request's). It verifies the manifest again, checks
  *     that the image it runs is the one the manifest signs, takes the
  *     install's state lock (#815; the upgrader job and the terminal
- *     `upgrade` take the same one, so they exclude each other), and runs
+ *     `upgrade` take the same one, so they exclude each other), grants its
+ *     own account the allowlisted roles the self-check found missing (each
+ *     a timeline event; IAM is asked again before it goes on), and runs
  *     the very upgrade the terminal runs (upgrade/upgrade.ts): backup →
  *     converge + app-roll → deep health check, rollback on failure →
  *     install.json + the infrastructure contract → the request's end.
@@ -46,10 +50,16 @@
  */
 
 import type { ConvergeCheckpoint } from "./converge-checkpoint.js";
-import { CONVERGE_PREFLIGHT_VOICE, convergeAccessChecks } from "./converge-job-preflight.js";
+import {
+  CONVERGE_PREFLIGHT_VOICE,
+  convergeAccessChecks,
+  INSTALL_ROLE_USES,
+  selfGrantPlan,
+} from "./converge-job-preflight.js";
 import { GCP_STACK_ENV } from "./gcp-names.js";
-import { decideInstallJob, type ExecutionState } from "./install-job.js";
+import { decideInstallJob, type ExecutionState, type InstallJobRole } from "./install-job.js";
 import type { InstallMetadata } from "./install-state.js";
+import { provisionerServiceAccountEmail } from "./provisioner-identity.js";
 import {
   compareSemver,
   type ReleaseManifest,
@@ -60,6 +70,7 @@ import type { UpgradeCheckpoint } from "./upgrade/upgrade.js";
 import { assessUpgradeRequest } from "./upgrader-assess.js";
 import {
   type IamTester,
+  missingAccessText,
   preflightErrorReason,
   preflightMissingReason,
   runIamPreflight,
@@ -116,6 +127,14 @@ export type ConvergeUpgradeOutcome =
       readonly unrecorded: boolean;
     };
 
+/**
+ * How long stage 2 waits for IAM to confirm the roles it granted itself:
+ * Google documents that a policy change usually applies within two minutes
+ * and can take seven (cloud.google.com/iam/docs/access-change-propagation).
+ */
+export const SELF_GRANT_RECHECK_INTERVAL_MS = 15_000;
+export const SELF_GRANT_RECHECK_ATTEMPTS = 32;
+
 /** How {@link ConvergeJobDeps.secureDatabase} ended. */
 export type SecureDatabaseResult =
   | { readonly ok: true; readonly host: string }
@@ -138,6 +157,16 @@ export interface ConvergeJobDeps {
   readonly region: string;
   /** Asks IAM as this job's own identity (the IAM self-check, #884). */
   readonly iam: IamTester;
+  /**
+   * Grant this job's own account (the install's provisioner) `role` on the
+   * project. Stage 2 only, and only for roles of the self-grant allowlist
+   * (converge-job-preflight.ts); the real one refuses any other role.
+   */
+  readonly grantOwnRole: (
+    role: InstallJobRole,
+  ) => Promise<{ readonly ok: true } | { readonly ok: false; readonly error: string }>;
+  /** Waits between the IAM re-checks after a self-grant. */
+  readonly sleep: (ms: number) => Promise<void>;
   /** This Cloud Run execution; null outside Cloud Run. */
   readonly execution: string | null;
   /** The Caelo release this code belongs to (CAELO_VERSION of the image). */
@@ -326,18 +355,11 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
 
     // #884 — decided to run: ask IAM for what the rest needs before the
     // lock, the hand-over or the backup (converge-job-preflight.ts). Both
-    // stages ask; stage 2's list is the target release's own.
-    const access = await runIamPreflight(
-      convergeAccessChecks({ projectId: deps.projectId, region: deps.region, env: GCP_STACK_ENV }),
-      deps.iam,
-    );
-    if (access.kind === "missing") {
-      return stopWith(
-        "needs_cloud_shell",
-        ["iam-preflight"],
-        preflightMissingReason(access.missing, CONVERGE_PREFLIGHT_VOICE),
-      );
-    }
+    // stages ask; stage 2's list is the target release's own. A missing
+    // role of the self-grant allowlist does not stop the run: stage 2
+    // grants it once it has verified its own image and holds the lock.
+    const stage1 = compareSemver(deps.ownVersion, m.version) !== 0;
+    const access = await askIam();
     if (access.kind === "error") {
       return stopWith(
         "failed",
@@ -345,12 +367,36 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
         preflightErrorReason(access.errors, CONVERGE_PREFLIGHT_VOICE),
       );
     }
-    r.info("The converge job has every permission this update needs.");
+    let toGrant: readonly InstallJobRole[] = [];
+    if (access.kind === "missing") {
+      const plan = selfGrantPlan(access.missing);
+      if (plan.blocking.length > 0) {
+        return stopWith(
+          "needs_cloud_shell",
+          ["iam-preflight"],
+          preflightMissingReason(access.missing, CONVERGE_PREFLIGHT_VOICE),
+        );
+      }
+      toGrant = plan.grant;
+    }
+    if (toGrant.length === 0) {
+      r.info("The converge job has every permission this update needs.");
+    } else {
+      r.info(
+        `The converge job's account lacks ${toGrant.join(", ")}, which this update needs. ${stage1 ? `The installer of ${m.version} grants them to it` : "It grants them to itself"} before it changes anything, once its own installer image is verified.`,
+      );
+    }
 
-    if (compareSemver(deps.ownVersion, m.version) !== 0) {
+    if (stage1) {
       return await handOver(m);
     }
-    return await convergeHere(m, fromVersion ?? (meta.releaseVersion as string), checkpoint, meta);
+    return await convergeHere(
+      m,
+      fromVersion ?? (meta.releaseVersion as string),
+      checkpoint,
+      meta,
+      toGrant,
+    );
   } catch (e) {
     return stopWith(
       "failed",
@@ -414,6 +460,7 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
     fromVersion: string,
     checkpoint: ConvergeCheckpoint | null,
     judged: InstallMetadata,
+    toGrant: readonly InstallJobRole[],
   ): Promise<UpgraderRunResult> {
     const installId = judged.installId;
     const own = await deps.ownImageDigest();
@@ -453,6 +500,13 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
     const stale = recordsChangedSince(judged, changed);
     if (stale) return stopWith(stale.status, stale.codes, stale.reason);
 
+    // After the image check and under the lock: only the target release's
+    // verified installer grants, and only roles of its own allowlist.
+    if (toGrant.length > 0) {
+      const granted = await grantOwnRoles(m, toGrant, changed);
+      if (granted) return stopWith(granted.status, granted.codes, granted.reason);
+    }
+
     const startedAt = deps.now().toISOString();
     let current: ConvergeCheckpoint = {
       statusVersion: 1,
@@ -550,6 +604,76 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
     );
   }
 
+  /** The converge self-check against IAM, as this job's identity. */
+  function askIam() {
+    return runIamPreflight(
+      convergeAccessChecks({ projectId: deps.projectId, region: deps.region, env: GCP_STACK_ENV }),
+      deps.iam,
+    );
+  }
+
+  /**
+   * Stage 2: grant this job's account each role in `roles` (the self-check
+   * found them missing; all are on the allowlist), then wait until IAM
+   * confirms the whole self-check. Null when it does; how the request ends
+   * otherwise.
+   */
+  async function grantOwnRoles(
+    m: ReleaseManifest,
+    roles: readonly InstallJobRole[],
+    changed: boolean,
+  ): Promise<{ status: UpdateEndStatus; codes: string[]; reason: string } | null> {
+    const account = provisionerServiceAccountEmail(deps.projectId);
+    const granted: InstallJobRole[] = [];
+    const before = () =>
+      changed
+        ? "This update already started changing the site; "
+        : granted.length > 0
+          ? `Nothing on the site was changed (the converge job's account got ${granted.join(", ")}): `
+          : "Nothing was changed: ";
+    for (const role of roles) {
+      const result = await deps.grantOwnRole(role);
+      if (!result.ok) {
+        return {
+          status: "needs_cloud_shell",
+          codes: ["iam-self-grant"],
+          reason: `${before()}the converge job could not grant its own account (${account}) ${role}, which Caelo ${m.version} needs to ${INSTALL_ROLE_USES[role].need} (${result.error}). ${CONVERGE_PREFLIGHT_VOICE.repair}`,
+        };
+      }
+      granted.push(role);
+      r.success(
+        `Granted ${role} to the converge job's account (${account}), needed by Caelo ${m.version} to ${INSTALL_ROLE_USES[role].need}.`,
+      );
+    }
+    await r.flush();
+    let last = await askIam();
+    for (let attempt = 1; last.kind !== "ok" && attempt < SELF_GRANT_RECHECK_ATTEMPTS; attempt++) {
+      await deps.sleep(SELF_GRANT_RECHECK_INTERVAL_MS);
+      last = await askIam();
+    }
+    if (last.kind === "ok") {
+      r.info(
+        "IAM confirms the granted roles; the converge job has every permission this update needs.",
+      );
+      return null;
+    }
+    const minutes = Math.round(
+      (SELF_GRANT_RECHECK_INTERVAL_MS * (SELF_GRANT_RECHECK_ATTEMPTS - 1)) / 60_000,
+    );
+    if (last.kind === "error") {
+      return {
+        status: "failed",
+        codes: ["iam-self-grant"],
+        reason: `${before()}after granting its own account ${granted.join(", ")}, the converge job could not check its access again (${last.errors.join("; ")}). Start the update again from Settings → Updates in a few minutes.`,
+      };
+    }
+    return {
+      status: "needs_cloud_shell",
+      codes: ["iam-self-grant"],
+      reason: `${before()}the converge job granted its own account ${granted.join(", ")}, but ${minutes} minutes later Google Cloud still reports permissions missing — ${missingAccessText(last.missing)}. ${CONVERGE_PREFLIGHT_VOICE.repair}`,
+    };
+  }
+
   /**
    * Whether install.json, re-read under the lock, is still what the run was
    * judged on; how the request ends otherwise.

package/src/install-job.ts

@@ -70,6 +70,12 @@ export const OPS_IMAGE_REPOSITORY =
  * the job's own output has to reach Cloud Logging so the terminal can
  * tail it. Granted only to installs that use the job. Not an escalation: the
  * provisioner already holds `resourcemanager.projectIamAdmin`.
+ *
+ * This list is also the converge job's self-grant allowlist
+ * (converge-job-preflight.ts `CONVERGE_SELF_GRANTABLE_ROLES`): a role added
+ * here is granted by the converge job of the release that adds it to an
+ * install that lacks it, with no terminal run. Never add an ownership or
+ * IAM-policy role (the tests refuse `SELF_GRANT_FORBIDDEN_ROLES`).
  */
 export const INSTALL_JOB_ROLES = [
   "roles/iam.roleAdmin",

package/src/mcp-auth-proxy-setup.test.ts

@@ -190,7 +190,10 @@ describe("ensureReleaseServices", () => {
     let cleaned = false;
     const { run, calls } = fakeGcloud({
       ...accountAndGrants,
-      "run services describe caelo-production-data-gateway": fail("NOT_FOUND: could not be found"),
+      // gcloud's real wording: no NOT_FOUND code (searchVIU, 2026-11-03).
+      "run services describe caelo-production-data-gateway": fail(
+        "ERROR: (gcloud.run.services.describe) Cannot find service [caelo-production-data-gateway]",
+      ),
       "run services list": ok("caelo-production-gateway-bbb\n"),
       "run services describe caelo-production-gateway-bbb": ok(JSON.stringify(OLD_GATEWAY)),
       "run services replace": ok(),
@@ -293,6 +296,35 @@ describe("ensureReleaseServices", () => {
     expect(calls.some((c) => c.startsWith("run "))).toBe(false);
   });
 
+  it("retries a grant while the account it just created has not propagated", async () => {
+    const { run: base, calls } = fakeGcloud({
+      ...accountAndGrants,
+      "run services describe caelo-production-mcp-auth-proxy": ok(
+        "caelo-production-mcp-auth-proxy\n",
+      ),
+    });
+    let refusals = 2;
+    const run: GcloudRunner = async (args, opts) => {
+      if (args[0] === "secrets" && refusals > 0) {
+        refusals -= 1;
+        return fail(
+          `ERROR: (gcloud.secrets.add-iam-policy-binding) Status code: 400. Service account ${PROXY_SA} does not exist..`,
+        );
+      }
+      return base(args, opts);
+    };
+    const pauses: number[] = [];
+    const outcomes = await ensureReleaseServices(gcp, {
+      run,
+      sleep: async (ms) => {
+        pauses.push(ms);
+      },
+    });
+    expect(outcomes.map((o) => o.status)).toEqual(["applied", "present", "present", "present"]);
+    expect(pauses.length).toBe(2);
+    expect(calls.filter((c) => c.startsWith("secrets add-iam-policy-binding"))).toHaveLength(2);
+  });
+
   it("stops at once when the account cannot be read", async () => {
     const { run, calls } = fakeGcloud({
       [`iam service-accounts describe ${PROXY_SA}`]: fail("PERMISSION_DENIED"),

package/src/mcp-auth-proxy-setup.ts

@@ -31,7 +31,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { gcloud as defaultGcloud } from "./gcloud.js";
-import type { GcloudRunner } from "./gcloud-retry.js";
+import { type GcloudRunner, realSleep, runWithRetry, type Sleep } from "./gcloud-retry.js";
 import { gcpSecretId, mcpAuthProxyServiceAccountEmail } from "./gcp-names.js";
 import { type EnsureOutcome, ensureMcpAuthProxyServiceAccount } from "./runtime-secrets.js";
 import {
@@ -70,9 +70,13 @@ export interface ReleaseServicesDeps {
     readonly path: string;
     readonly cleanup: () => void;
   };
+  /** Waits between retries of a grant on the account created a moment ago. */
+  readonly sleep?: Sleep;
 }
 
-const NOT_FOUND = /NOT_FOUND|not found|does not exist|could not be found/i;
+// gcloud words a missing Cloud Run service "Cannot find service [<name>]."
+// with no NOT_FOUND code (searchVIU upgrade to v0.11.10, 2026-11-03).
+const NOT_FOUND = /NOT_FOUND|not found|cannot find|does not exist|could not be found/i;
 
 /** Ensure everything {@link ReleaseServicesTarget} needs; outcomes in order, failures included. */
 export async function ensureReleaseServices(
@@ -84,7 +88,7 @@ export async function ensureReleaseServices(
   const account = await ensureMcpAuthProxyServiceAccount(target, { run });
   outcomes.push(account);
   if (account.status === "failed") return outcomes;
-  outcomes.push(...(await ensureMcpAuthProxySecretAccess(target, run)));
+  outcomes.push(...(await ensureMcpAuthProxySecretAccess(target, run, deps.sleep ?? realSleep)));
   if (outcomes.some((o) => o.status === "failed")) return outcomes;
   outcomes.push(
     target.provider === "gcp-firebase"
@@ -103,12 +107,15 @@ export async function ensureReleaseServices(
 async function ensureMcpAuthProxySecretAccess(
   target: Pick<ReleaseServicesTarget, "projectId" | "env">,
   run: GcloudRunner,
+  sleep: Sleep,
 ): Promise<EnsureOutcome[]> {
   const member = `serviceAccount:${mcpAuthProxyServiceAccountEmail(target.projectId, target.env)}`;
   const outcomes: EnsureOutcome[] = [];
   for (const secret of serviceSecrets("mcp-auth-proxy")) {
     const id = `secretAccessor → MCP-Auth-Proxy on ${secret}`;
-    const r = await run([
+    // The account was usually created a moment ago, and IAM answers "does
+    // not exist" until it has propagated: retried, not failed.
+    const r = await runWithRetry(run, sleep, [
       "secrets",
       "add-iam-policy-binding",
       gcpSecretId(target.env, secret),

package/src/upgrader-iam-preflight.ts

@@ -92,9 +92,9 @@ export interface AccessCheck {
   readonly permissions: readonly string[];
 }
 
-/** A check with the permissions it lacks. */
-export interface MissingAccess {
-  readonly check: AccessCheck;
+/** A check with the permissions it lacks (`C`: the caller's own check type, handed back as given). */
+export interface MissingAccess<C extends AccessCheck = AccessCheck> {
+  readonly check: C;
   readonly missing: readonly string[];
   /**
    * Permissions that could not be judged because another one is missing
@@ -114,9 +114,9 @@ export interface IamTester {
 }
 
 /** How the self-check came out. */
-export type PreflightOutcome =
+export type PreflightOutcome<C extends AccessCheck = AccessCheck> =
   | { readonly kind: "ok" }
-  | { readonly kind: "missing"; readonly missing: readonly MissingAccess[] }
+  | { readonly kind: "missing"; readonly missing: readonly MissingAccess<C>[] }
   /** IAM could not be asked about at least one resource; nothing was judged missing. */
   | { readonly kind: "error"; readonly errors: readonly string[] };
 
@@ -250,10 +250,10 @@ export function upgraderAccessChecks(target: {
 }
 
 /** Ask IAM about every check (one call per resource); never throws. */
-export async function runIamPreflight(
-  checks: readonly AccessCheck[],
+export async function runIamPreflight<C extends AccessCheck>(
+  checks: readonly C[],
   tester: IamTester,
-): Promise<PreflightOutcome> {
+): Promise<PreflightOutcome<C>> {
   // One question per resource: the project alone carries four checks.
   const byResource = new Map<string, { resource: IamResource; permissions: Set<string> }>();
   for (const c of checks) {
@@ -300,7 +300,7 @@ export async function runIamPreflight(
       }),
   );
   if (errors.length > 0) return { kind: "error", errors: errors.sort() };
-  const missing: MissingAccess[] = [];
+  const missing: MissingAccess<C>[] = [];
   for (const check of checks) {
     const key = resourceName(check.resource);
     const held = granted.get(key) ?? new Set<string>();
@@ -343,6 +343,11 @@ export function preflightMissingReason(
   missing: readonly MissingAccess[],
   voice: PreflightVoice,
 ): string {
+  return `Nothing was changed: ${voice.subject} checked its access before starting and is missing permissions this update needs — ${missingAccessText(missing)}. ${voice.repair}`;
+}
+
+/** Every missing permission with what it is for, as one clause list ("it cannot …; it cannot …"). */
+export function missingAccessText(missing: readonly MissingAccess[]): string {
   const lines = missing.map((m) => {
     const where = resourceName(m.check.resource);
     const parts = [
@@ -355,7 +360,7 @@ export function preflightMissingReason(
     ];
     return `it ${m.missing.length > 0 ? "cannot" : "may not be able to"} ${m.check.need} (${parts.join("; ")})`;
   });
-  return `Nothing was changed: ${voice.subject} checked its access before starting and is missing permissions this update needs — ${lines.join("; ")}. ${voice.repair}`;
+  return lines.join("; ");
 }
 
 /** The request's end reason when IAM could not be asked. */

package/src/wizards/gcp-cost.ts

@@ -192,7 +192,7 @@ export function estimateGcpCost(inputs: CostEstimateInputs): {
       name: "Quality checks (Cloud Run audit job)",
       monthlyUsd: 2,
       notes:
-        "one container per Lighthouse run, all in parallel (up to 5 pages x 3 runs, ~90 s at 2 vCPU each): ~$0.05 per audited Stage, partly in the free tier; results bucket ~$0",
+        "one container per Lighthouse run, all in parallel (up to 5 pages x 3 runs, ~90 s at 2 vCPU each; a page whose runs disagree gets 2 more): ~$0.05 per audited Stage, at most ~$0.08, partly in the free tier; results bucket ~$0",
     },
     {
       name: "Screenshots (Cloud Run capture job)",