@caelo-cms/provisioning 0.11.9 → 0.11.11

package/src/upgrade/app-roll.ts

@@ -25,15 +25,23 @@
  */
 
 import type { GcloudRunner, Sleep } from "../gcloud-retry.js";
-import { gatewayServiceAccountEmail, runServiceAccountEmail } from "../gcp-names.js";
+import {
+  gatewayServiceAccountEmail,
+  mcpAuthProxyServiceAccountEmail,
+  runServiceAccountEmail,
+} from "../gcp-names.js";
 import type { ImageDigests } from "../install-state.js";
 import {
   adminMediaVolume,
+  CLOUD_RUN_SLUGS,
   type CloudRunEnvVar,
-  GATEWAY_TIMEOUT_SECONDS,
+  type CloudRunSlug,
+  cloudRunServicePrefix,
+  MCP_AUTH_PROXY_TIMEOUT_SECONDS,
   RETIRED_SECRET_FILES,
   SERVICE_CPU_IDLE,
   SERVICE_SECRET_FILES,
+  SERVICE_VPC_EGRESS,
 } from "../stack-contract.js";
 import {
   type DeployedService,
@@ -44,12 +52,14 @@ import {
   liveContainerEnv,
   liveContainerMemory,
   liveCpuIdle,
+  liveDirectVpc,
   liveTimeoutSeconds,
   liveVolumes,
   planAdminMemory,
   planContractEnv,
   planCpuIdle,
   planMediaVolume,
+  planNoVpc,
   planRequestTimeout,
   planSecretFiles,
   rollService,
@@ -69,10 +79,27 @@ import {
 import type { HealthProbe } from "./health.js";
 import type { UpgradeReporter } from "./reporter.js";
 
-export type RollSlug = "admin" | "gateway";
+export type RollSlug = CloudRunSlug;
 
-/** The roll order: the gateway follows the admin, never the other way round. */
-export const ROLL_ORDER: readonly RollSlug[] = ["admin", "gateway"];
+/**
+ * The roll order: the gateway follows the admin, never the other way round,
+ * and the MCP-Auth-Proxy comes last (#1328) — on gcp-firebase it forwards
+ * the visitor traffic still addressed to it to the gateway, which must
+ * already serve the new release by then.
+ */
+export const ROLL_ORDER: readonly RollSlug[] = CLOUD_RUN_SLUGS;
+
+/** The service account each service's new revision runs as. */
+export function rollServiceAccount(slug: RollSlug, projectId: string, env: string): string {
+  switch (slug) {
+    case "admin":
+      return runServiceAccountEmail(projectId, env);
+    case "gateway":
+      return gatewayServiceAccountEmail(projectId, env);
+    case "mcp-auth-proxy":
+      return mcpAuthProxyServiceAccountEmail(projectId, env);
+  }
+}
 
 /** The install an app-roll acts on. */
 export interface AppRollInstall {
@@ -110,6 +137,8 @@ export interface ServicePlan {
   readonly liveTimeout: number | null;
   /** Whether the service gets CPU only during requests now. */
   readonly liveCpuIdle: boolean;
+  /** Whether the service egresses into the install's VPC now. */
+  readonly liveDirectVpc: boolean;
 }
 
 /** A service plan plus the changes its roll applies. */
@@ -139,6 +168,13 @@ export interface AppRollPlan {
   readonly services: readonly RollPlan[];
   /** Notes for the operator (e.g. a memory limit left as is). */
   readonly warnings: readonly string[];
+  /**
+   * #1338 — the services dial the Cloud SQL instance's IP, which no
+   * verify-full connection can use, so their database URLs are not planned.
+   * A converge moves them onto the instance's DNS name
+   * ({@link EnvAdjustments.databaseHost}); a roll refuses while this holds.
+   */
+  readonly databaseHostPending?: boolean;
 }
 
 export type PlanResult =
@@ -166,7 +202,7 @@ export async function planAppRoll(
 ): Promise<PlanResult> {
   const services: ServicePlan[] = [];
   for (const slug of ROLL_ORDER) {
-    const prefix = `caelo-${install.env}-${slug}`;
+    const prefix = cloudRunServicePrefix(install.provider, install.env, slug);
     const lookup = await resolveGcpResourceName(
       run,
       "run-service",
@@ -210,6 +246,7 @@ export async function planAppRoll(
       liveVolumes: liveVolumes(json),
       liveTimeout: liveTimeoutSeconds(json),
       liveCpuIdle: liveCpuIdle(json),
+      liveDirectVpc: liveDirectVpc(json),
     });
   }
   const env = planEnv(install, services, {});
@@ -265,29 +302,45 @@ export async function planAppRoll(
     // stack's memory default (never lowered; an unparsable value is kept).
     const memory = s.slug === "admin" ? planAdminMemory(s.liveMemory) : null;
     if (memory && !memory.ok) warnings.push(`admin memory left as is: ${memory.error}`);
-    // #806 — the gateway streams remote-MCP chat turns (stack-contract.ts
-    // GATEWAY_TIMEOUT_SECONDS). The admin's timeout is the stacks' alone.
+    // #806 — the MCP-Auth-Proxy streams remote-MCP chat turns
+    // (stack-contract.ts MCP_AUTH_PROXY_TIMEOUT_SECONDS). The other services'
+    // timeouts are the stacks' alone.
     const timeout =
-      s.slug === "gateway" ? planRequestTimeout(s.liveTimeout, GATEWAY_TIMEOUT_SECONDS) : null;
+      s.slug === "mcp-auth-proxy"
+        ? planRequestTimeout(s.liveTimeout, MCP_AUTH_PROXY_TIMEOUT_SECONDS)
+        : null;
     const memoryFlags = memory?.ok ? memory.flags : [];
     const timeoutFlags = timeout?.ok ? timeout.flags : [];
     const cpu = planCpuIdle(s.liveCpuIdle, SERVICE_CPU_IDLE[s.slug]);
     const cpuFlags = cpu.ok ? cpu.flags : [];
+    // #1328 — the gcp-firebase proxy is the former gateway service: it
+    // leaves the VPC in the same revision that drops its database env.
+    const vpc = SERVICE_VPC_EGRESS[s.slug] ? null : planNoVpc(s.liveDirectVpc);
+    const vpcFlags = vpc?.ok ? vpc.flags : [];
     return {
       ...s,
       envFlags: env.services[s.slug].flags,
       envChanges: env.services[s.slug].changes,
-      resourceFlags: [...memoryFlags, ...timeoutFlags, ...cpuFlags],
+      resourceFlags: [...memoryFlags, ...timeoutFlags, ...cpuFlags, ...vpcFlags],
       resourceChanges: [
         ...memoryFlags.map((f) => `${s.liveMemory ?? "(default)"} → ${f}`),
         ...(timeout?.ok ? timeout.changes : []),
         ...(cpu.ok ? cpu.changes : []),
+        ...(vpc?.ok ? vpc.changes : []),
       ],
       volumeFlags: [...(s.slug === "admin" ? media.flags : []), ...secretFiles[s.slug].flags],
       volumeChanges: [...(s.slug === "admin" ? media.changes : []), ...secretFiles[s.slug].changes],
     };
   });
-  return { ok: true, plan: { install, services: rolls, warnings } };
+  if (env.databaseHostPending) {
+    warnings.push(
+      "the services still dial the Cloud SQL instance's IP; the converge moves them onto its DNS name for verify-full TLS (#1338) before anything rolls",
+    );
+  }
+  return {
+    ok: true,
+    plan: { install, services: rolls, warnings, databaseHostPending: env.databaseHostPending },
+  };
 }
 
 /** Env the converge half adds to, or keeps out of, the rolls ({@link AppRollHooks.afterMigrations}). */
@@ -299,6 +352,11 @@ export interface EnvAdjustments {
    * identity env, which only the converge (operator identity) can resolve.
    */
   readonly extraEnv?: Readonly<Partial<Record<RollSlug, readonly CloudRunEnvVar[]>>>;
+  /**
+   * #1338 — the database host the converge moved the services onto (the
+   * Cloud SQL instance's DNS name, cloudsql-tls.ts).
+   */
+  readonly databaseHost?: string;
 }
 
 function planEnv(
@@ -322,6 +380,7 @@ function planEnv(
     {
       leaveUntouched: adjust.leaveUntouched ?? [],
       ...(adjust.extraEnv ? { extraEnv: adjust.extraEnv } : {}),
+      ...(adjust.databaseHost ? { databaseHost: adjust.databaseHost } : {}),
     },
   );
 }
@@ -333,11 +392,12 @@ function planEnv(
  */
 export function adjustingEnv(plan: AppRollPlan, adjust: EnvAdjustments): AppRollPlan {
   const extra = Object.values(adjust.extraEnv ?? {}).some((v) => (v?.length ?? 0) > 0);
-  if ((adjust.leaveUntouched?.length ?? 0) === 0 && !extra) return plan;
+  if ((adjust.leaveUntouched?.length ?? 0) === 0 && !extra && !adjust.databaseHost) return plan;
   const env = planEnv(plan.install, plan.services, adjust);
   if (!env.ok) return plan;
   return {
     ...plan,
+    databaseHostPending: env.databaseHostPending,
     services: plan.services.map((s) => ({
       ...s,
       envFlags: env.services[s.slug].flags,
@@ -370,7 +430,7 @@ export interface AppRollHooks {
   afterMigrations?(): Promise<
     | ({ readonly ok: true; readonly leaveUntouched: readonly string[] } & Pick<
         EnvAdjustments,
-        "extraEnv"
+        "extraEnv" | "databaseHost"
       >)
     | { readonly ok: false }
   >;
@@ -470,6 +530,17 @@ export async function executeAppRoll(
     r.warn("--skip-health-check set — the new revisions get Cloud Run's readiness check only.");
   }
 
+  // #1338 — the new images refuse a database URL without verify-full, and
+  // only a converge can move the services off the instance's IP. Without
+  // one (the in-cloud upgrader) the roll stops before it changes anything.
+  if (plan.databaseHostPending && !hooks.afterMigrations) {
+    r.error(
+      "The services still dial the Cloud SQL instance's IP address. This release connects with verify-full TLS, which needs the instance's DNS name: run the converge first (Settings → Updates installs it as a cloud-setup release, or `cms-provision upgrade` from a terminal).",
+    );
+    r.warn("No migrations ran and no traffic moved.");
+    return fail("migrations", false);
+  }
+
   // Migrations BEFORE any traffic moves, with the NEW admin image: the old
   // image does not carry the release's migration files (v0.9.2).
   const admin = plan.services.find((s) => s.slug === "admin") as RollPlan;
@@ -490,8 +561,15 @@ export async function executeAppRoll(
     plan = adjustingEnv(plan, {
       leaveUntouched: converged.leaveUntouched,
       ...(converged.extraEnv ? { extraEnv: converged.extraEnv } : {}),
+      ...(converged.databaseHost ? { databaseHost: converged.databaseHost } : {}),
     });
   }
+  if (plan.databaseHostPending) {
+    r.error(
+      "The converge did not move the services onto the Cloud SQL instance's DNS name, so the new revisions could not reach the database with verify-full TLS (#1338). Nothing rolled; fix the database TLS step above and re-run.",
+    );
+    return fail("converge", true);
+  }
 
   const rolled: RollPlan[] = [];
   /** Jobs this roll moved onto the admin's image (#875, #1217), with their prior image. */
@@ -540,10 +618,7 @@ export async function executeAppRoll(
         region: install.region,
         projectId: install.projectId,
         imageRef: p.imageRef,
-        serviceAccount:
-          p.slug === "admin"
-            ? runServiceAccountEmail(install.projectId, install.env)
-            : gatewayServiceAccountEmail(install.projectId, install.env),
+        serviceAccount: rollServiceAccount(p.slug, install.projectId, install.env),
         envFlags: p.envFlags,
         resourceFlags: p.resourceFlags,
         volumeFlags: p.volumeFlags,
@@ -637,9 +712,12 @@ export async function executeAppRoll(
     s.done(`Healthy: ${deep.detail}`);
   }
 
+  const digestOf = (slug: RollSlug) =>
+    (plan.services.find((s) => s.slug === slug) as RollPlan).digest;
   const digests: ImageDigests = {
-    admin: (plan.services.find((s) => s.slug === "admin") as RollPlan).digest,
-    gateway: (plan.services.find((s) => s.slug === "gateway") as RollPlan).digest,
+    admin: digestOf("admin"),
+    gateway: digestOf("gateway"),
+    "mcp-auth-proxy": digestOf("mcp-auth-proxy"),
   };
   await deps.recordDigests(digests);
   return { ok: true, digests };

package/src/upgrade/converge.ts

@@ -5,7 +5,8 @@
  * service accounts, secrets or other cloud resources so the install matches
  * what the release's stacks declare — the gateway's run SA and the
  * CLI-generated runtime secrets, the stack's IAM/CDN/firewall invariants,
- * MCP access through IAP, public_role's Cloud SQL role memberships (#633),
+ * MCP access through IAP, the staging database and its gateway login
+ * (#1329), the gateway logins' Cloud SQL role memberships (#633),
  * the gateway's and public_role's database logins, the operator-access sync
  * job, the quality-audit job (#875), the remote-MCP load-balancer rules and
  * identity env (#806), and dropping bindings nothing reads any more.
@@ -30,10 +31,16 @@
 
 import { ensureAuditJob, resolveAdminImage } from "../audit-job-setup.js";
 import { ensureCaptureJob } from "../capture-job-setup.js";
+import {
+  dialsIpAddress,
+  ensureCloudSqlVerifyFull,
+  liveDatabaseUrls,
+  moveServicesToDnsHost,
+} from "../cloudsql-tls.js";
 import { ensureConvergeJob } from "../converge-job-setup.js";
 import { findSqlInstance, planPublicRoleSwitch } from "../gateway-credentials.js";
 import type { GcloudRunner, Sleep } from "../gcloud-retry.js";
-import { upgraderServiceAccountEmail } from "../gcp-names.js";
+import { schedulerServiceAccountEmail, upgraderServiceAccountEmail } from "../gcp-names.js";
 import type { InstallMetadata } from "../install-state.js";
 import { ensureMcpIapAccess, type IapResource } from "../mcp-iap.js";
 import { ensureOperatorAccessSync, resolveOperatorAccessTarget } from "../operator-access.js";
@@ -46,6 +53,7 @@ import {
   plainGeneratedSecretSeed,
   readSecretReplication,
 } from "../runtime-secrets.js";
+import { ensureSchedulerTick } from "../scheduler-setup.js";
 import { selfCommand } from "../self-command.js";
 import {
   AUDIT_JOB_ENV_VAR,
@@ -54,9 +62,13 @@ import {
   CAPTURE_JOB_ENV_VAR,
   type CloudRunEnvVar,
   CONVERGE_JOB_ENV_VAR,
+  DATABASE_URL_VARS,
+  INTERIM_FORWARD_ORIGIN_ENV,
   MCP_ENV_VAR,
   OPERATOR_ACCESS_JOB_ENV_VAR,
+  parseSchedulerIntervalMinutes,
   remoteMcpIdentityEnv,
+  SCHEDULER_SERVICE_ACCOUNT_ENV_VAR,
   UPGRADER_JOB_ENV_VAR,
 } from "../stack-contract.js";
 import {
@@ -65,10 +77,11 @@ import {
   liveEnvHasInlinePassword,
   removeRetiredIamBindings,
 } from "../stack-converge.js";
+import { ensureStagingDatabase } from "../staging-database.js";
 import { ensureUpgraderJob, resolveUpgraderTarget } from "../upgrader-setup.js";
 import { ensureUrlMapRedirectsAccess } from "../url-map-redirects-access.js";
 import type { AppRollHooks, AppRollPlan, RollPlan, RollSlug } from "./app-roll.js";
-import { lookupFailure } from "./gcp-lookup.js";
+import { findServiceUrl, lookupFailure } from "./gcp-lookup.js";
 import type { UpgradeReporter } from "./reporter.js";
 
 /**
@@ -102,6 +115,11 @@ export interface ConvergeDeps {
    * then fails and the converge stays incomplete.
    */
   readonly opsImageRef?: string | null;
+  /**
+   * #1140 — the install's `schedulerIntervalMinutes` knob (install.json);
+   * absent: the default. The scheduler step refuses an out-of-range value.
+   */
+  readonly schedulerIntervalMinutes?: number;
 }
 
 export interface ConvergeSession {
@@ -139,7 +157,11 @@ export function startConverge(
   const { reporter: r, run } = deps;
   const runDeps = { run, ...(deps.sleep ? { sleep: deps.sleep } : {}) };
   const service = (slug: RollSlug) => plan.services.find((s) => s.slug === slug) as RollPlan;
-  const services = { admin: service("admin").serviceName, gateway: service("gateway").serviceName };
+  const services = {
+    admin: service("admin").serviceName,
+    gateway: service("gateway").serviceName,
+    "mcp-auth-proxy": service("mcp-auth-proxy").serviceName,
+  };
   const target = {
     provider: install.provider,
     projectId: install.projectId,
@@ -222,22 +244,48 @@ export function startConverge(
     return true;
   }
 
+  async function stagingDatabase(): Promise<boolean> {
+    // #1329 — staging's own database and its gateway login, on an install
+    // from before them. Before migrations: the cms_public_staging run and
+    // migration 0530 need both, and the membership revoke below needs the
+    // user. A failure aborts — the new admin refuses to start without the
+    // staging database.
+    const s = r.step("Ensuring the staging database (cms_public_staging) and its gateway login...");
+    const outcomes = await ensureStagingDatabase(
+      { projectId: install.projectId, env: install.env, credentials: install.credentials },
+      { ...runDeps, ...(deps.http ? { http: deps.http } : {}) },
+    );
+    const failed = outcomes.filter((o) => o.status === "failed");
+    if (failed.length > 0) {
+      s.failed("The staging database could not be set up. Aborting upgrade.");
+      for (const o of failed) r.error(`  FAILED: ${o.id}\n    ${o.error ?? ""}`);
+      r.warn("No traffic was shifted and no migrations ran. Fix the above and re-run.");
+      return false;
+    }
+    s.done("Staging database and its gateway login ready");
+    for (const o of outcomes.filter((o) => o.status === "applied")) {
+      r.info(`  ${o.id}: ${o.detail ?? "created"}`);
+    }
+    return true;
+  }
+
   async function publicRoleIsolation(): Promise<boolean> {
-    // #633 — the Admin API made public_role a member of cloudsqlsuperuser.
-    // Revoked here, before migrations, because the migration job's role
-    // audit is what verifies it and fails the upgrade while it remains.
-    const s = r.step("Removing public_role from the Cloud SQL superuser group...");
+    // #633 — the Admin API made public_role (and, #1329, public_staging_role)
+    // a member of cloudsqlsuperuser. Revoked here, before migrations,
+    // because the migration job's role audit is what verifies it and fails
+    // the upgrade while it remains.
+    const s = r.step("Removing the gateway logins from the Cloud SQL superuser group...");
     const outcome = await revokePublicRoleMemberships(
       { projectId: install.projectId, env: install.env, credentials: install.credentials },
       runDeps,
     );
     if (outcome.status === "failed") {
-      s.failed("public_role's role memberships could not be revoked. Aborting upgrade.");
+      s.failed("The gateway logins' role memberships could not be revoked. Aborting upgrade.");
       r.error(`  ${outcome.error ?? ""}`);
       r.warn("No traffic was shifted and no migrations ran. Fix the above and re-run.");
       return false;
     }
-    s.done(`public_role holds no database roles (${outcome.detail ?? "revoked"})`);
+    s.done(`The gateway logins hold no database roles (${outcome.detail ?? "revoked"})`);
     return true;
   }
 
@@ -371,33 +419,53 @@ export function startConverge(
   }
 
   async function remoteMcpEdge(): Promise<void> {
-    // #806 — on `gcp` the load balancer routes the remote-MCP discovery
-    // paths to the gateway and Cloud Armor lets agent traffic through on
-    // /api/mcp/* (remote-mcp-edge.ts). Only remote MCP depends on it, so a
-    // failure warns.
+    // #806 / #1328 — on `gcp` the load balancer routes the remote-MCP paths
+    // to the MCP-Auth-Proxy's own backend, whose Cloud Armor policy lets
+    // agent traffic through on /api/mcp/* (remote-mcp-edge.ts). Only remote
+    // MCP depends on it, so a failure warns. The gateway being rolled in
+    // this run no longer answers those paths, so until this succeeds remote
+    // MCP is down on the site's domain — said below.
     if (install.provider !== "gcp") return;
-    const s = r.step("Ensuring the load balancer's remote-MCP routes and Cloud Armor rule...");
+    const s = r.step("Ensuring the MCP-Auth-Proxy's load-balancer route and Cloud Armor rule...");
     const edge = await ensureRemoteMcpEdge(
       {
         provider: install.provider,
         projectId: install.projectId,
+        region: install.region,
         env: install.env,
         domain: install.domain,
+        mcpAuthProxyImageRef: service("mcp-auth-proxy").imageRef,
       },
       { ...runDeps, ...(deps.http ? { http: deps.http } : {}) },
     );
     if (edge.ok) {
       for (const line of edge.done) r.info(`  ${line}`);
-      s.done("Remote-MCP load-balancer rules ready");
+      s.done("Remote-MCP load-balancer route ready");
       return;
     }
-    s.warned(`Remote-MCP load-balancer rules not set up: ${edge.error}`);
+    s.warned(`Remote-MCP load-balancer route not set up: ${edge.error}`);
     r.warn(
       "The upgrade continues; remote MCP (Settings → MCP → Remote) cannot be reached through the site's domain until this succeeds. Re-run upgrade once the cause is fixed.",
     );
     incomplete = true;
   }
 
+  async function interimForwardEnv(): Promise<CloudRunEnvVar[]> {
+    // #1328 — on gcp-firebase the proxy is the service the current live
+    // release's Firebase rewrite and the staging domain mapping still
+    // point at; it forwards that traffic to the data gateway, whose URL is
+    // its origin (stack-contract.ts INTERIM_FORWARD_ORIGIN_ENV). Without it
+    // the proxy answers the live site's /api/** with 404 until the next
+    // Publish live, and the staging host with 404 — so this aborts.
+    if (install.provider !== "gcp-firebase") return [];
+    const url = await findServiceUrl(run, install.projectId, install.region, services.gateway);
+    if (!url) throw new Error(`could not read the URL of ${services.gateway}`);
+    return [{ name: INTERIM_FORWARD_ORIGIN_ENV, value: new URL(url).origin }];
+  }
+
+  /** The admin's origin behind IAP, once {@link remoteMcpIdentity} read it. */
+  let resolvedAdminUrl: string | null = null;
+
   async function remoteMcpIdentity(): Promise<Record<RollSlug, CloudRunEnvVar[]> | null> {
     // #806 — CAELO_ADMIN_URL / CAELO_MCP_PUBLIC_URL / CAELO_IAP_AUDIENCE
     // name identities the env contract cannot (stack-contract.ts
@@ -420,6 +488,7 @@ export function startConverge(
       runDeps,
     );
     if (resolved.ok) {
+      resolvedAdminUrl = resolved.identity.adminUrl;
       s.done(
         `Google sign-in and remote MCP can be set up under Settings → Google sign-in (${resolved.identity.mcpPublicUrl})`,
       );
@@ -459,6 +528,71 @@ export function startConverge(
     return name ? { kind: "backend-services", service: name } : null;
   }
 
+  /** #1338 — the host the database TLS step moved the services onto. */
+  let databaseHost: string | null = null;
+
+  async function databaseTls(): Promise<boolean> {
+    // #1338 — verify-full TLS needs the shared server CA, the instance's
+    // DNS name resolving in the VPC, and the services dialling that name
+    // (cloudsql-tls.ts). Before migrations: the migration job reads its host
+    // from the admin's live URL, and the new images refuse anything less
+    // than verify-full. A failure aborts — the new revisions could not
+    // reach the database.
+    const s = r.step("Setting up verify-full TLS to the database (shared CA, private DNS name)...");
+    const liveUrls = plan.services.map((p) => ({
+      serviceName: p.serviceName,
+      liveUrls: liveDatabaseUrls(p.liveEnv, DATABASE_URL_VARS[p.slug]),
+    }));
+    // Only this step moves the services off the instance's IP, and only
+    // after the instance, its certificate and the DNS name are in shape —
+    // so services that dial a host name already went through it.
+    const onIp = liveUrls.some((u) => Object.values(u.liveUrls).some((v) => dialsIpAddress(v)));
+    if (!onIp) {
+      s.done("Database TLS ready: the services already dial the instance's DNS name");
+      return true;
+    }
+    const lookup = await findSqlInstance(install, { run });
+    if (lookup.status !== "found") {
+      s.failed(
+        `${lookupFailure(lookup, "Cloud SQL instance", install.credentials)}. Aborting upgrade.`,
+      );
+      r.warn(
+        "No traffic was shifted and no migrations ran. Re-run upgrade once the instance is found.",
+      );
+      return false;
+    }
+    const tlsDeps = { ...runDeps, ...(deps.http ? { http: deps.http } : {}) };
+    const tls = await ensureCloudSqlVerifyFull(
+      { projectId: install.projectId, env: install.env, sqlInstance: lookup.name },
+      tlsDeps,
+    );
+    for (const line of tls.done) r.info(`  ${line}`);
+    if (!tls.ok) {
+      s.failed(`Database TLS could not be set up: ${tls.error}. Aborting upgrade.`);
+      r.warn("No traffic was shifted and no migrations ran. Fix the above and re-run.");
+      return false;
+    }
+    const moved = await moveServicesToDnsHost(
+      { projectId: install.projectId, region: install.region },
+      liveUrls,
+      tls.host,
+      tlsDeps,
+    );
+    for (const line of moved.done) r.info(`  ${line}`);
+    if (!moved.ok) {
+      s.failed(
+        `The services could not be moved onto ${tls.host}: ${moved.error}. Aborting upgrade.`,
+      );
+      r.warn("No migrations ran. Fix the above and re-run.");
+      return false;
+    }
+    databaseHost = tls.host;
+    s.done(
+      `Database TLS ready: the services dial ${tls.host} (verify-full from the new revisions)`,
+    );
+    return true;
+  }
+
   async function databaseLogins(): Promise<boolean> {
     // #613 — public_role moves onto its own password right before the
     // gateway's roll. The gateway has no other database login (#635).
@@ -578,6 +712,77 @@ export function startConverge(
     leaveUntouched.push(CONVERGE_JOB_ENV_VAR);
   }
 
+  let schedulerReady = false;
+  /** The scheduler's SA exists from an earlier run though this one failed. */
+  let schedulerAccountExists = false;
+
+  async function setUpSchedulerTick(): Promise<
+    | { ok: true; done: string[]; intervalMinutes: number }
+    | { ok: false; done: string[]; error: string }
+  > {
+    let intervalMinutes: number;
+    try {
+      intervalMinutes = parseSchedulerIntervalMinutes(
+        deps.schedulerIntervalMinutes,
+        "install.json schedulerIntervalMinutes",
+      );
+    } catch (e) {
+      return { ok: false, done: [], error: (e as Error).message };
+    }
+    const resource = await adminIapResource();
+    if (!resource) return { ok: false, done: [], error: "the admin's IAP resource was not found" };
+    if (!resolvedAdminUrl)
+      return { ok: false, done: [], error: "the admin's URL could not be read" };
+    const st = await ensureSchedulerTick(
+      {
+        projectId: install.projectId,
+        region: install.region,
+        env: install.env,
+        adminUrl: resolvedAdminUrl,
+        resource,
+        intervalMinutes,
+      },
+      runDeps,
+    );
+    return st.ok ? { ok: true, done: st.done, intervalMinutes } : st;
+  }
+
+  async function schedulerTick(): Promise<void> {
+    // #1140 — the one Cloud Scheduler job that calls the admin's tick
+    // (scheduler-setup.ts). After migrations and before the operator-access
+    // step, which must keep the scheduler's account on IAP. The schedule
+    // starts right away: until the roll below finishes, a tick reaches the
+    // previous admin, which has no such endpoint (404) — that execution
+    // fails and nothing runs; the first tick after the roll does the work.
+    // A failure costs the periodic work (plugin workers, GC, release check),
+    // not the site, so it warns and the admin keeps its current env.
+    const s = r.step("Ensuring the scheduler tick (periodic work and plugin workers)...");
+    const st = await setUpSchedulerTick();
+    for (const step of st.done) r.info(`  ${step}`);
+    if (st.ok) {
+      schedulerReady = true;
+      s.done(`Scheduler tick ready (every ${st.intervalMinutes} min)`);
+      return;
+    }
+    s.warned(`Scheduler tick not set up: ${st.error}`);
+    // An account set up by an earlier run keeps its IAP access, or the next
+    // sync would cut off a tick that still works.
+    const exists = await run([
+      "iam",
+      "service-accounts",
+      "describe",
+      schedulerServiceAccountEmail(install.projectId, install.env),
+      `--project=${install.projectId}`,
+      "--format=value(email)",
+    ]);
+    schedulerAccountExists = exists.ok;
+    r.warn(
+      "The upgrade continues; plugin workers, update checks and clean-up run late or not at all until this succeeds. Re-run upgrade once the cause is fixed.",
+    );
+    incomplete = true;
+    leaveUntouched.push(SCHEDULER_SERVICE_ACCOUNT_ENV_VAR);
+  }
+
   async function operatorAccess(): Promise<void> {
     // The job that keeps IAP in step with the user list (operator-access.ts).
     // After migrations: its database user gets the role migration 0239
@@ -605,6 +810,13 @@ export function startConverge(
                   ],
                 }
               : {}),
+            ...(schedulerReady || schedulerAccountExists
+              ? {
+                  iapServiceAccounts: [
+                    schedulerServiceAccountEmail(install.projectId, install.env),
+                  ],
+                }
+              : {}),
           },
           runDeps,
         )
@@ -627,7 +839,9 @@ export function startConverge(
       if (deps.provisionerIdentity && !(await deps.provisionerIdentity())) return false;
       if (!(await runtimeIdentities())) return false;
       if (!(await stackInvariants())) return false;
+      if (!(await stagingDatabase())) return false;
       if (!(await publicRoleIsolation())) return false;
+      if (!(await databaseTls())) return false;
       await mcpAccess();
       await urlMapRedirectsAccess();
       await auditJob();
@@ -639,14 +853,30 @@ export function startConverge(
     hooks: {
       async afterMigrations() {
         if (!(await databaseLogins())) return { ok: false };
+        // Read-only, first: the scheduler tick needs the admin's URL.
+        const identity = await remoteMcpIdentity();
         await upgraderJob();
         await convergeJob();
+        await schedulerTick();
         await operatorAccess();
-        const extraEnv = await remoteMcpIdentity();
+        let forward: CloudRunEnvVar[];
+        try {
+          forward = await interimForwardEnv();
+        } catch (e) {
+          r.error(
+            `The MCP-Auth-Proxy cannot be pointed at the data gateway: ${e instanceof Error ? e.message : String(e)}. Nothing was rolled; re-run upgrade once it is fixed.`,
+          );
+          return { ok: false };
+        }
+        const extraEnv: Partial<Record<RollSlug, CloudRunEnvVar[]>> = {
+          ...(identity ?? {}),
+          "mcp-auth-proxy": [...(identity?.["mcp-auth-proxy"] ?? []), ...forward],
+        };
         return {
           ok: true,
           leaveUntouched: [...leaveUntouched],
-          ...(extraEnv ? { extraEnv } : {}),
+          extraEnv,
+          ...(databaseHost ? { databaseHost } : {}),
         };
       },
       async beforeRoll(slug) {