@caelo-cms/provisioning 0.11.11 → 0.11.13

package/src/converge-job.test.ts

@@ -19,6 +19,7 @@ import {
   type ConvergeQueue,
   type ConvergeUpgradeOutcome,
   runConvergeJob,
+  SELF_GRANT_RECHECK_ATTEMPTS,
   type SecureDatabaseResult,
 } from "./converge-job.js";
 import type { ExecutionState } from "./install-job.js";
@@ -150,6 +151,8 @@ function harness(
     iam?: IamTester;
     /** #1373 — the database TLS step's answer; null: the queue dials the DNS name already. */
     secureDatabase?: SecureDatabaseResult | null;
+    /** What a self-grant answers; ok by default. */
+    grant?: (role: string) => { ok: true } | { ok: false; error: string };
   } = {},
 ): Harness {
   const calls: string[] = [];
@@ -171,6 +174,13 @@ function harness(
     iam: opts.iam ?? {
       granted: async (_resource, permissions) => ({ ok: true, granted: permissions }),
     },
+    grantOwnRole: async (role) => {
+      calls.push(`grant ${role}`);
+      return opts.grant ? opts.grant(role) : { ok: true };
+    },
+    sleep: async () => {
+      calls.push("sleep");
+    },
     execution: "caelo-production-converge-run1",
     ownVersion: opts.ownVersion ?? "0.11.0",
     fetchManifest: async (v) => {
@@ -465,6 +475,139 @@ describe("runConvergeJob — stage 2 (the target release's own installer)", () =
   });
 });
 
+/**
+ * An IAM that lacks the workflows.* permissions (an install set up before
+ * 0.11.11 added roles/workflows.editor, the searchviu-website case) until
+ * the job granted the role, and confirms it `lag` asks later.
+ */
+function lacksWorkflows(lag = 0, extraMissing: (p: string) => boolean = () => false) {
+  const state = { granted: false, asksSinceGrant: 0 };
+  const iam: IamTester = {
+    granted: async (_resource, permissions) => {
+      if (state.granted) state.asksSinceGrant += 1;
+      const workflowsHeld = state.granted && state.asksSinceGrant > lag;
+      return {
+        ok: true,
+        granted: permissions.filter(
+          (p) => !extraMissing(p) && (workflowsHeld || !p.startsWith("workflows.")),
+        ),
+      };
+    },
+  };
+  const grant = (role: string) => {
+    if (role === "roles/workflows.editor") state.granted = true;
+    return { ok: true } as const;
+  };
+  return { iam, grant };
+}
+
+describe("runConvergeJob — the job grants itself the roles its release names (2026-11-03)", () => {
+  it("stage 1 leaves a missing allowlisted role to stage 2 and hands over without granting", async () => {
+    const { iam, grant } = lacksWorkflows();
+    const h = harness({ ownVersion: "0.10.53", iam, grant });
+    const r = await runConvergeJob(h.deps);
+    expect(r.status).toBeNull();
+    expect(h.calls.some((c) => c.startsWith("grant"))).toBe(false);
+    expect(h.calls).toContain("start");
+    expect(h.q.ends).toEqual([]);
+    expect(
+      h.q.events.some(
+        (e) =>
+          e.message.includes("lacks roles/workflows.editor") &&
+          e.message.includes("The installer of 0.11.0 grants them"),
+      ),
+    ).toBe(true);
+  });
+
+  it("stage 2 grants the role after the image check and under the lock, records it, re-checks, converges", async () => {
+    const { iam, grant } = lacksWorkflows();
+    const h = harness({ iam, grant });
+    const r = await runConvergeJob(h.deps);
+    expect(r.status).toBe("succeeded");
+    const g = h.calls.indexOf("grant roles/workflows.editor");
+    expect(g).toBeGreaterThan(h.calls.indexOf("lock"));
+    expect(g).toBeLessThan(h.calls.indexOf("checkpoint running"));
+    expect(g).toBeLessThan(h.calls.indexOf("upgrade"));
+    // Only the missing role, nothing else of the list.
+    expect(h.calls.filter((c) => c.startsWith("grant"))).toEqual(["grant roles/workflows.editor"]);
+    const event = h.q.events.find((e) => e.message.startsWith("Granted roles/workflows.editor"));
+    expect(event?.level).toBe("success");
+    expect(event?.message).toBe(
+      "Granted roles/workflows.editor to the converge job's account (caelo-provisioner@acme.iam.gserviceaccount.com), needed by Caelo 0.11.0 to deploy and update the workflow the scheduler tick runs (#1140).",
+    );
+    expect(h.q.events.some((e) => e.message.startsWith("IAM confirms the granted roles"))).toBe(
+      true,
+    );
+  });
+
+  it("waits for IAM to confirm a fresh grant (propagation) before it goes on", async () => {
+    const { iam, grant } = lacksWorkflows(3);
+    const h = harness({ iam, grant });
+    const r = await runConvergeJob(h.deps);
+    expect(r.status).toBe("succeeded");
+    const sleeps = h.calls.filter((c) => c === "sleep").length;
+    expect(sleeps).toBeGreaterThan(0);
+    expect(h.calls.lastIndexOf("sleep")).toBeLessThan(h.calls.indexOf("upgrade"));
+  });
+
+  it("a grant that fails (org policy) ends the request with the Cloud Shell step, before any change", async () => {
+    const { iam } = lacksWorkflows();
+    const h = harness({
+      iam,
+      grant: () => ({
+        ok: false,
+        error: "ERROR: (gcloud.projects.add-iam-policy-binding) FAILED_PRECONDITION: org policy",
+      }),
+    });
+    const r = await runConvergeJob(h.deps);
+    expect(r.status).toBe("needs_cloud_shell");
+    expect(h.q.ends[0]?.codes).toEqual(["iam-self-grant"]);
+    const reason = h.q.ends[0]?.reason ?? "";
+    expect(reason).toStartWith(
+      "Nothing was changed: the converge job could not grant its own account",
+    );
+    expect(reason).toContain("roles/workflows.editor");
+    expect(reason).toContain("org policy");
+    expect(reason).toContain("Run the update once from Cloud Shell");
+    expect(h.upgrades).toEqual([]);
+    expect(h.calls.some((c) => c.startsWith("checkpoint"))).toBe(false);
+    expect(h.calls.at(-1)).toBe("unlock");
+  });
+
+  it("a grant IAM never confirms ends with the Cloud Shell step after the wait", async () => {
+    const { iam, grant } = lacksWorkflows(10_000);
+    const h = harness({ iam, grant });
+    const r = await runConvergeJob(h.deps);
+    expect(r.status).toBe("needs_cloud_shell");
+    expect(h.q.ends[0]?.codes).toEqual(["iam-self-grant"]);
+    expect(h.q.ends[0]?.reason).toContain("workflows.workflows.create");
+    expect(h.calls.filter((c) => c === "sleep").length).toBe(SELF_GRANT_RECHECK_ATTEMPTS - 1);
+    expect(h.upgrades).toEqual([]);
+  });
+
+  it("a missing permission no allowlisted role carries stops either stage without granting anything", async () => {
+    // workflows.* is repairable, actAs on its own account is not.
+    const { iam, grant } = lacksWorkflows(0, (p) => p === "iam.serviceAccounts.actAs");
+    for (const stage of ["0.10.53", "0.11.0"]) {
+      const h = harness({ ownVersion: stage, iam, grant });
+      const r = await runConvergeJob(h.deps);
+      expect(r.status).toBe("needs_cloud_shell");
+      expect(h.q.ends[0]?.codes).toEqual(["iam-preflight"]);
+      expect(h.calls.some((c) => /^(grant|lock|point|start|upgrade)/.test(c))).toBe(false);
+      expect(h.q.ends[0]?.reason).toContain("workflows.workflows.create");
+      expect(h.q.ends[0]?.reason).toContain("iam.serviceAccounts.actAs");
+    }
+  });
+
+  it("stage 2 refuses an image the manifest does not sign before it grants anything", async () => {
+    const { iam, grant } = lacksWorkflows();
+    const h = harness({ iam, grant, ownDigest: D("9") });
+    const r = await runConvergeJob(h.deps);
+    expect(r.status).toBe("refused");
+    expect(h.calls.some((c) => c.startsWith("grant"))).toBe(false);
+  });
+});
+
 describe("runConvergeJob — resume after a killed run", () => {
   it("a run that died is adopted once Cloud Run says it ended, and resumes from the checkpoint", async () => {
     const h = harness({

package/src/converge-job.ts

@@ -16,7 +16,9 @@
  *     verifies its signature (release-manifest.ts), and decides
  *     (upgrader-assess.ts, `route: "converge"`): no downgrade, nothing
  *     below minFromVersion, no automatic converge unless the policy allows
- *     it; then it checks its own IAM access (converge-job-preflight.ts).
+ *     it; then it checks its own IAM access (converge-job-preflight.ts) and
+ *     stops on anything missing — except a role of the self-grant
+ *     allowlist, which it leaves to stage 2.
  *     Then it verifies the target's ops image, points this job at it BY DIGEST and starts a new
  *     execution, handing the request over to it. It changes nothing else:
  *     the converge logic always comes from the release it installs.
@@ -25,7 +27,9 @@
  *     version is the request's). It verifies the manifest again, checks
  *     that the image it runs is the one the manifest signs, takes the
  *     install's state lock (#815; the upgrader job and the terminal
- *     `upgrade` take the same one, so they exclude each other), and runs
+ *     `upgrade` take the same one, so they exclude each other), grants its
+ *     own account the allowlisted roles the self-check found missing (each
+ *     a timeline event; IAM is asked again before it goes on), and runs
  *     the very upgrade the terminal runs (upgrade/upgrade.ts): backup →
  *     converge + app-roll → deep health check, rollback on failure →
  *     install.json + the infrastructure contract → the request's end.
@@ -46,10 +50,16 @@
  */
 
 import type { ConvergeCheckpoint } from "./converge-checkpoint.js";
-import { CONVERGE_PREFLIGHT_VOICE, convergeAccessChecks } from "./converge-job-preflight.js";
+import {
+  CONVERGE_PREFLIGHT_VOICE,
+  convergeAccessChecks,
+  INSTALL_ROLE_USES,
+  selfGrantPlan,
+} from "./converge-job-preflight.js";
 import { GCP_STACK_ENV } from "./gcp-names.js";
-import { decideInstallJob, type ExecutionState } from "./install-job.js";
+import { decideInstallJob, type ExecutionState, type InstallJobRole } from "./install-job.js";
 import type { InstallMetadata } from "./install-state.js";
+import { provisionerServiceAccountEmail } from "./provisioner-identity.js";
 import {
   compareSemver,
   type ReleaseManifest,
@@ -60,6 +70,7 @@ import type { UpgradeCheckpoint } from "./upgrade/upgrade.js";
 import { assessUpgradeRequest } from "./upgrader-assess.js";
 import {
   type IamTester,
+  missingAccessText,
   preflightErrorReason,
   preflightMissingReason,
   runIamPreflight,
@@ -116,6 +127,14 @@ export type ConvergeUpgradeOutcome =
       readonly unrecorded: boolean;
     };
 
+/**
+ * How long stage 2 waits for IAM to confirm the roles it granted itself:
+ * Google documents that a policy change usually applies within two minutes
+ * and can take seven (cloud.google.com/iam/docs/access-change-propagation).
+ */
+export const SELF_GRANT_RECHECK_INTERVAL_MS = 15_000;
+export const SELF_GRANT_RECHECK_ATTEMPTS = 32;
+
 /** How {@link ConvergeJobDeps.secureDatabase} ended. */
 export type SecureDatabaseResult =
   | { readonly ok: true; readonly host: string }
@@ -138,6 +157,16 @@ export interface ConvergeJobDeps {
   readonly region: string;
   /** Asks IAM as this job's own identity (the IAM self-check, #884). */
   readonly iam: IamTester;
+  /**
+   * Grant this job's own account (the install's provisioner) `role` on the
+   * project. Stage 2 only, and only for roles of the self-grant allowlist
+   * (converge-job-preflight.ts); the real one refuses any other role.
+   */
+  readonly grantOwnRole: (
+    role: InstallJobRole,
+  ) => Promise<{ readonly ok: true } | { readonly ok: false; readonly error: string }>;
+  /** Waits between the IAM re-checks after a self-grant. */
+  readonly sleep: (ms: number) => Promise<void>;
   /** This Cloud Run execution; null outside Cloud Run. */
   readonly execution: string | null;
   /** The Caelo release this code belongs to (CAELO_VERSION of the image). */
@@ -326,18 +355,11 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
 
     // #884 — decided to run: ask IAM for what the rest needs before the
     // lock, the hand-over or the backup (converge-job-preflight.ts). Both
-    // stages ask; stage 2's list is the target release's own.
-    const access = await runIamPreflight(
-      convergeAccessChecks({ projectId: deps.projectId, region: deps.region, env: GCP_STACK_ENV }),
-      deps.iam,
-    );
-    if (access.kind === "missing") {
-      return stopWith(
-        "needs_cloud_shell",
-        ["iam-preflight"],
-        preflightMissingReason(access.missing, CONVERGE_PREFLIGHT_VOICE),
-      );
-    }
+    // stages ask; stage 2's list is the target release's own. A missing
+    // role of the self-grant allowlist does not stop the run: stage 2
+    // grants it once it has verified its own image and holds the lock.
+    const stage1 = compareSemver(deps.ownVersion, m.version) !== 0;
+    const access = await askIam();
     if (access.kind === "error") {
       return stopWith(
         "failed",
@@ -345,12 +367,36 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
         preflightErrorReason(access.errors, CONVERGE_PREFLIGHT_VOICE),
       );
     }
-    r.info("The converge job has every permission this update needs.");
+    let toGrant: readonly InstallJobRole[] = [];
+    if (access.kind === "missing") {
+      const plan = selfGrantPlan(access.missing);
+      if (plan.blocking.length > 0) {
+        return stopWith(
+          "needs_cloud_shell",
+          ["iam-preflight"],
+          preflightMissingReason(access.missing, CONVERGE_PREFLIGHT_VOICE),
+        );
+      }
+      toGrant = plan.grant;
+    }
+    if (toGrant.length === 0) {
+      r.info("The converge job has every permission this update needs.");
+    } else {
+      r.info(
+        `The converge job's account lacks ${toGrant.join(", ")}, which this update needs. ${stage1 ? `The installer of ${m.version} grants them to it` : "It grants them to itself"} before it changes anything, once its own installer image is verified.`,
+      );
+    }
 
-    if (compareSemver(deps.ownVersion, m.version) !== 0) {
+    if (stage1) {
       return await handOver(m);
     }
-    return await convergeHere(m, fromVersion ?? (meta.releaseVersion as string), checkpoint, meta);
+    return await convergeHere(
+      m,
+      fromVersion ?? (meta.releaseVersion as string),
+      checkpoint,
+      meta,
+      toGrant,
+    );
   } catch (e) {
     return stopWith(
       "failed",
@@ -414,6 +460,7 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
     fromVersion: string,
     checkpoint: ConvergeCheckpoint | null,
     judged: InstallMetadata,
+    toGrant: readonly InstallJobRole[],
   ): Promise<UpgraderRunResult> {
     const installId = judged.installId;
     const own = await deps.ownImageDigest();
@@ -453,6 +500,13 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
     const stale = recordsChangedSince(judged, changed);
     if (stale) return stopWith(stale.status, stale.codes, stale.reason);
 
+    // After the image check and under the lock: only the target release's
+    // verified installer grants, and only roles of its own allowlist.
+    if (toGrant.length > 0) {
+      const granted = await grantOwnRoles(m, toGrant, changed);
+      if (granted) return stopWith(granted.status, granted.codes, granted.reason);
+    }
+
     const startedAt = deps.now().toISOString();
     let current: ConvergeCheckpoint = {
       statusVersion: 1,
@@ -550,6 +604,76 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
     );
   }
 
+  /** The converge self-check against IAM, as this job's identity. */
+  function askIam() {
+    return runIamPreflight(
+      convergeAccessChecks({ projectId: deps.projectId, region: deps.region, env: GCP_STACK_ENV }),
+      deps.iam,
+    );
+  }
+
+  /**
+   * Stage 2: grant this job's account each role in `roles` (the self-check
+   * found them missing; all are on the allowlist), then wait until IAM
+   * confirms the whole self-check. Null when it does; how the request ends
+   * otherwise.
+   */
+  async function grantOwnRoles(
+    m: ReleaseManifest,
+    roles: readonly InstallJobRole[],
+    changed: boolean,
+  ): Promise<{ status: UpdateEndStatus; codes: string[]; reason: string } | null> {
+    const account = provisionerServiceAccountEmail(deps.projectId);
+    const granted: InstallJobRole[] = [];
+    const before = () =>
+      changed
+        ? "This update already started changing the site; "
+        : granted.length > 0
+          ? `Nothing on the site was changed (the converge job's account got ${granted.join(", ")}): `
+          : "Nothing was changed: ";
+    for (const role of roles) {
+      const result = await deps.grantOwnRole(role);
+      if (!result.ok) {
+        return {
+          status: "needs_cloud_shell",
+          codes: ["iam-self-grant"],
+          reason: `${before()}the converge job could not grant its own account (${account}) ${role}, which Caelo ${m.version} needs to ${INSTALL_ROLE_USES[role].need} (${result.error}). ${CONVERGE_PREFLIGHT_VOICE.repair}`,
+        };
+      }
+      granted.push(role);
+      r.success(
+        `Granted ${role} to the converge job's account (${account}), needed by Caelo ${m.version} to ${INSTALL_ROLE_USES[role].need}.`,
+      );
+    }
+    await r.flush();
+    let last = await askIam();
+    for (let attempt = 1; last.kind !== "ok" && attempt < SELF_GRANT_RECHECK_ATTEMPTS; attempt++) {
+      await deps.sleep(SELF_GRANT_RECHECK_INTERVAL_MS);
+      last = await askIam();
+    }
+    if (last.kind === "ok") {
+      r.info(
+        "IAM confirms the granted roles; the converge job has every permission this update needs.",
+      );
+      return null;
+    }
+    const minutes = Math.round(
+      (SELF_GRANT_RECHECK_INTERVAL_MS * (SELF_GRANT_RECHECK_ATTEMPTS - 1)) / 60_000,
+    );
+    if (last.kind === "error") {
+      return {
+        status: "failed",
+        codes: ["iam-self-grant"],
+        reason: `${before()}after granting its own account ${granted.join(", ")}, the converge job could not check its access again (${last.errors.join("; ")}). Start the update again from Settings → Updates in a few minutes.`,
+      };
+    }
+    return {
+      status: "needs_cloud_shell",
+      codes: ["iam-self-grant"],
+      reason: `${before()}the converge job granted its own account ${granted.join(", ")}, but ${minutes} minutes later Google Cloud still reports permissions missing — ${missingAccessText(last.missing)}. ${CONVERGE_PREFLIGHT_VOICE.repair}`,
+    };
+  }
+
   /**
    * Whether install.json, re-read under the lock, is still what the run was
    * judged on; how the request ends otherwise.

package/src/infra-contract.test.ts

@@ -20,8 +20,8 @@ describe("infrastructure contract steps", () => {
     );
   });
 
-  it("is at 18: public_role revoke (3), URL-map redirects role (4), no gateway cms_admin login (5), ops-bucket list (6), gateway proxy hops (7), quality-audit job (8), converge job (9), remote MCP (10), Google sign-in (11), project-level sign-in client (12), edge security headers (13), capture job (14), database verify-full TLS (15), MCP-Auth-Proxy (16), staging database (17), scheduler tick (18)", () => {
-    expect(INFRA_CONTRACT_VERSION).toBe(18);
+  it("is at 19: public_role revoke (3), URL-map redirects role (4), no gateway cms_admin login (5), ops-bucket list (6), gateway proxy hops (7), quality-audit job (8), converge job (9), remote MCP (10), Google sign-in (11), project-level sign-in client (12), edge security headers (13), capture job (14), database verify-full TLS (15), MCP-Auth-Proxy (16), staging database (17), scheduler tick (18), scheduler tick as JSON (19)", () => {
+    expect(INFRA_CONTRACT_VERSION).toBe(19);
     expect(INFRA_CONTRACT_STEPS[3]).toContain("#633");
     expect(INFRA_CONTRACT_STEPS[4]).toContain("#587");
     expect(INFRA_CONTRACT_STEPS[5]).toContain("#635");
@@ -38,6 +38,7 @@ describe("infrastructure contract steps", () => {
     expect(INFRA_CONTRACT_STEPS[16]).toContain("#1328");
     expect(INFRA_CONTRACT_STEPS[17]).toContain("#1329");
     expect(INFRA_CONTRACT_STEPS[18]).toContain("#1140");
+    expect(INFRA_CONTRACT_STEPS[19]).toContain("JSON");
   });
 
   it("lists the pending steps oldest first", () => {
@@ -48,6 +49,6 @@ describe("infrastructure contract steps", () => {
   });
 
   it("names a version a newer release added by number", () => {
-    expect(infraStepsText(18, 19)).toBe("infrastructure step 19 (see that release's notes)");
+    expect(infraStepsText(19, 20)).toBe("infrastructure step 20 (see that release's notes)");
   });
 });

package/src/infra-contract.ts

@@ -22,7 +22,7 @@
  * whose app-roll references infrastructure an install does not have yet fails
  * at roll time, after migrations ran.
  */
-export const INFRA_CONTRACT_VERSION = 18;
+export const INFRA_CONTRACT_VERSION = 19;
 
 // History (why each bump happened — an install below a number needs the
 // converge that introduced it):
@@ -150,6 +150,9 @@ export const INFRA_CONTRACT_VERSION = 18;
 //       on the admin. The new admin runs its periodic work and plugin workers
 //       only on that tick, so an in-cloud upgrader must not roll an install
 //       onto it before the tick exists.
+//  19 — the scheduler workflow's tick is a JSON POST. Without a content type
+//       SvelteKit refused every tick of contract 18 with 403, so no periodic
+//       work ran. The workflow source is only deployed by a converge.
 
 /**
  * What each contract version adds, in operator terms — the upgrader's
@@ -175,6 +178,7 @@ export const INFRA_CONTRACT_STEPS: Readonly<Record<number, string>> = {
   16: "the MCP sign-in service as its own Cloud Run service with its own account (no database access), the visitor gateway as its own service on Firebase installs, and on Google Cloud load-balancer installs the sign-in service's backend and Cloud Armor rule (#1328)",
   17: "the staging site's own database and its gateway login, kept apart from the live data (#1329)",
   18: "the scheduler tick that runs the admin's periodic work and plugin workers every few minutes: its service account, workflow and Cloud Scheduler job, and its IAP access (#1140)",
+  19: "the scheduler tick sent as a JSON request, because the admin refused every tick before (#1140)",
 };
 
 /**

package/src/install-job.ts

@@ -70,6 +70,12 @@ export const OPS_IMAGE_REPOSITORY =
  * the job's own output has to reach Cloud Logging so the terminal can
  * tail it. Granted only to installs that use the job. Not an escalation: the
  * provisioner already holds `resourcemanager.projectIamAdmin`.
+ *
+ * This list is also the converge job's self-grant allowlist
+ * (converge-job-preflight.ts `CONVERGE_SELF_GRANTABLE_ROLES`): a role added
+ * here is granted by the converge job of the release that adds it to an
+ * install that lacks it, with no terminal run. Never add an ownership or
+ * IAM-policy role (the tests refuse `SELF_GRANT_FORBIDDEN_ROLES`).
  */
 export const INSTALL_JOB_ROLES = [
   "roles/iam.roleAdmin",

package/src/lifecycle.ts

@@ -20,6 +20,7 @@ import { join } from "node:path";
 import { provisioningUpgradeCommand } from "@caelo-cms/shared";
 import { cancel, confirm, isCancel, log, note, select, spinner } from "@clack/prompts";
 import { bold, cyan, dim, green, red, yellow } from "kleur/colors";
+import { stagingDomainState } from "./admin-domain.js";
 import { activeAccount, checkGcloudAuth, gcloud } from "./gcloud.js";
 import { type GcloudRunner, realSleep } from "./gcloud-retry.js";
 import { GCP_STACK_ENV } from "./gcp-names.js";
@@ -686,12 +687,21 @@ async function upgradeLockedInstall(
   const stateNotice = localStateNotice(meta);
   if (stateNotice) note(stateNotice, "Pulumi state is on this machine");
   if (ok && provider === "gcp-firebase") {
-    // #650 — staging builds now render the staging host; the gateway
-    // serves it once it is mapped. The command is idempotent, so this
-    // reminder is safe to repeat on every upgrade.
-    log.info(
-      `Protected staging site: run ${bold("cms-provision staging-domain enable")} once to serve the staging host (it maps the host the admin uses — ${bold(stagingHost)} unless changed there —, turns on password protection, prints the login and the DNS record). Host and access: admin → Settings → Domains / Staging access.`,
-    );
+    // #650 — staging builds render the staging host; the gateway serves it
+    // once it is mapped. Only an install without the mapping is told to run
+    // `staging-domain enable`.
+    const staging = await stagingDomainState({ projectId, region, stagingHost, run });
+    if (staging.status === "mapped") {
+      log.info(`Protected staging site: https://${stagingHost}/`);
+    } else {
+      const unknown =
+        staging.status === "unknown"
+          ? ` (could not check whether ${bold(stagingHost)} is mapped: ${staging.reason})`
+          : "";
+      log.info(
+        `Protected staging site: run ${bold("cms-provision staging-domain enable")} once to serve the staging host${unknown} (it maps the host the admin uses — ${bold(stagingHost)} unless changed there —, turns on password protection, prints the login and the DNS record). Host and access: admin → Settings → Domains / Staging access.`,
+      );
+    }
   }
   return ok;
 }

package/src/mcp-auth-proxy-setup.test.ts

@@ -190,7 +190,10 @@ describe("ensureReleaseServices", () => {
     let cleaned = false;
     const { run, calls } = fakeGcloud({
       ...accountAndGrants,
-      "run services describe caelo-production-data-gateway": fail("NOT_FOUND: could not be found"),
+      // gcloud's real wording: no NOT_FOUND code (searchVIU, 2026-11-03).
+      "run services describe caelo-production-data-gateway": fail(
+        "ERROR: (gcloud.run.services.describe) Cannot find service [caelo-production-data-gateway]",
+      ),
       "run services list": ok("caelo-production-gateway-bbb\n"),
       "run services describe caelo-production-gateway-bbb": ok(JSON.stringify(OLD_GATEWAY)),
       "run services replace": ok(),
@@ -293,6 +296,35 @@ describe("ensureReleaseServices", () => {
     expect(calls.some((c) => c.startsWith("run "))).toBe(false);
   });
 
+  it("retries a grant while the account it just created has not propagated", async () => {
+    const { run: base, calls } = fakeGcloud({
+      ...accountAndGrants,
+      "run services describe caelo-production-mcp-auth-proxy": ok(
+        "caelo-production-mcp-auth-proxy\n",
+      ),
+    });
+    let refusals = 2;
+    const run: GcloudRunner = async (args, opts) => {
+      if (args[0] === "secrets" && refusals > 0) {
+        refusals -= 1;
+        return fail(
+          `ERROR: (gcloud.secrets.add-iam-policy-binding) Status code: 400. Service account ${PROXY_SA} does not exist..`,
+        );
+      }
+      return base(args, opts);
+    };
+    const pauses: number[] = [];
+    const outcomes = await ensureReleaseServices(gcp, {
+      run,
+      sleep: async (ms) => {
+        pauses.push(ms);
+      },
+    });
+    expect(outcomes.map((o) => o.status)).toEqual(["applied", "present", "present", "present"]);
+    expect(pauses.length).toBe(2);
+    expect(calls.filter((c) => c.startsWith("secrets add-iam-policy-binding"))).toHaveLength(2);
+  });
+
   it("stops at once when the account cannot be read", async () => {
     const { run, calls } = fakeGcloud({
       [`iam service-accounts describe ${PROXY_SA}`]: fail("PERMISSION_DENIED"),

package/src/mcp-auth-proxy-setup.ts

@@ -31,7 +31,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { gcloud as defaultGcloud } from "./gcloud.js";
-import type { GcloudRunner } from "./gcloud-retry.js";
+import { type GcloudRunner, realSleep, runWithRetry, type Sleep } from "./gcloud-retry.js";
 import { gcpSecretId, mcpAuthProxyServiceAccountEmail } from "./gcp-names.js";
 import { type EnsureOutcome, ensureMcpAuthProxyServiceAccount } from "./runtime-secrets.js";
 import {
@@ -70,9 +70,13 @@ export interface ReleaseServicesDeps {
     readonly path: string;
     readonly cleanup: () => void;
   };
+  /** Waits between retries of a grant on the account created a moment ago. */
+  readonly sleep?: Sleep;
 }
 
-const NOT_FOUND = /NOT_FOUND|not found|does not exist|could not be found/i;
+// gcloud words a missing Cloud Run service "Cannot find service [<name>]."
+// with no NOT_FOUND code (searchVIU upgrade to v0.11.10, 2026-11-03).
+const NOT_FOUND = /NOT_FOUND|not found|cannot find|does not exist|could not be found/i;
 
 /** Ensure everything {@link ReleaseServicesTarget} needs; outcomes in order, failures included. */
 export async function ensureReleaseServices(
@@ -84,7 +88,7 @@ export async function ensureReleaseServices(
   const account = await ensureMcpAuthProxyServiceAccount(target, { run });
   outcomes.push(account);
   if (account.status === "failed") return outcomes;
-  outcomes.push(...(await ensureMcpAuthProxySecretAccess(target, run)));
+  outcomes.push(...(await ensureMcpAuthProxySecretAccess(target, run, deps.sleep ?? realSleep)));
   if (outcomes.some((o) => o.status === "failed")) return outcomes;
   outcomes.push(
     target.provider === "gcp-firebase"
@@ -103,12 +107,15 @@ export async function ensureReleaseServices(
 async function ensureMcpAuthProxySecretAccess(
   target: Pick<ReleaseServicesTarget, "projectId" | "env">,
   run: GcloudRunner,
+  sleep: Sleep,
 ): Promise<EnsureOutcome[]> {
   const member = `serviceAccount:${mcpAuthProxyServiceAccountEmail(target.projectId, target.env)}`;
   const outcomes: EnsureOutcome[] = [];
   for (const secret of serviceSecrets("mcp-auth-proxy")) {
     const id = `secretAccessor → MCP-Auth-Proxy on ${secret}`;
-    const r = await run([
+    // The account was usually created a moment ago, and IAM answers "does
+    // not exist" until it has propagated: retried, not failed.
+    const r = await runWithRetry(run, sleep, [
       "secrets",
       "add-iam-policy-binding",
       gcpSecretId(target.env, secret),

package/src/scheduler-setup.test.ts

@@ -177,6 +177,12 @@ describe("schedulerWorkflowSource", () => {
     expect(src).toContain(`exp: \${now + 600}`);
   });
 
+  it("declares the tick as JSON, or SvelteKit refuses it as a cross-site form post", () => {
+    const tick = src.slice(src.indexOf("- tick:"), src.indexOf("- done:"));
+    expect(tick).toContain("Content-Type: application/json");
+    expect(tick).toContain("body: {}");
+  });
+
   it("posts the tick with the signed JWT and a bounded timeout", () => {
     expect(src).toContain('url: "https://admin.acme.com/api/internal/scheduler/tick"');
     expect(src).toContain(`Authorization: \${"Bearer " + signed.body.signedJwt}`);

package/src/scheduler-setup.ts

@@ -165,6 +165,11 @@ export function tickOrigin(
  * Credentials API, authenticated with the workflow's own OAuth token), then
  * POST the tick with it. A refused or failed tick (HTTP ≥ 400) fails the
  * execution, so it shows in the workflow's execution history and logs.
+ *
+ * The tick carries `Content-Type: application/json` and a JSON body, like the
+ * self-hosted clock (admin-core scheduler/clock.ts), because the admin's
+ * SvelteKit treats a POST with no content type and no `Origin` as a
+ * cross-site form submission: it answers 403 before the route runs.
  */
 export function schedulerWorkflowSource(opts: {
   readonly serviceAccount: string;
@@ -198,6 +203,8 @@ main:
           url: "${origin}${SCHEDULER_TICK_PATH}"
           headers:
             Authorization: \${"Bearer " + signed.body.signedJwt}
+            Content-Type: application/json
+          body: {}
           timeout: ${SCHEDULER_TICK_TIMEOUT_SECONDS}
         result: answer
     - done: