@caelo-cms/provisioning 0.11.8 → 0.11.10

package/src/stack-converge.ts

@@ -23,12 +23,13 @@
  *     Storage volume + mount), added in the same `services update` when
  *     missing.
  *   - {@link planSecretFiles} / {@link planRequestTimeout} (#806): the
- *     gateway's secret file (the remote-MCP OAuth client) and its request
+ *     MCP-Auth-Proxy.s secret file (the remote-MCP OAuth client) and its request
  *     timeout, in the same `services update`.
  *   - {@link planCpuIdle}: request-based CPU where the contract wants it
  *     (`SERVICE_CPU_IDLE`), in the same `services update`.
  */
 
+import { isIpLiteral } from "@caelo-cms/shared";
 import { removeDefaultFirewallRules } from "./default-firewall.js";
 import { gcloud as defaultGcloud, type GcloudResult } from "./gcloud.js";
 import { type GcloudRunner, realSleep, runWithRetry, type Sleep } from "./gcloud-retry.js";
@@ -38,8 +39,10 @@ import {
   ADMIN_MEMORY_DEFAULT,
   type AdminEnvInputs,
   adminEnvContract,
+  CLOUD_RUN_SLUGS,
   type CloudRunEnvVar,
   type CloudRunSlug,
+  DATABASE_URL_VARS,
   databaseUrls,
   type GcpProvider,
   gatewayEnvContract,
@@ -47,6 +50,7 @@ import {
   type IamTarget,
   iamMember,
   type MediaVolumeContract,
+  mcpAuthProxyEnvContract,
   memoryQuantityMiB,
   RETIRED_IAM_BINDINGS,
   RETIRED_RUNTIME_SECRETS,
@@ -167,7 +171,7 @@ function maskUrlPassword(value: string): string {
  * gcloud's list flags split on `,`; a value containing one needs the
  * `^<delim>^` escape syntax (`gcloud topic escaping`).
  */
-function listFlag(flag: string, pairs: string[]): string {
+export function listFlag(flag: string, pairs: string[]): string {
   const joined = pairs.join(",");
   if (!pairs.some((p) => p.includes(","))) return `${flag}=${joined}`;
   const delim = ["@", "|", "#", "~", ";"].find((d) => !pairs.some((p) => p.includes(d)));
@@ -306,21 +310,42 @@ export function planContractEnv(
      * refused rather than silently doubled.
      */
     readonly extraEnv?: Readonly<Partial<Record<CloudRunSlug, readonly CloudRunEnvVar[]>>>;
+    /**
+     * #1338 — the database host the converge just set up (the instance's
+     * DNS name, cloudsql-tls.ts), in place of the one the services dialled
+     * when the plan was read.
+     */
+    readonly databaseHost?: string;
   } = {},
 ):
   | {
       readonly ok: true;
       readonly services: Record<CloudRunSlug, { flags: string[]; changes: EnvChange[] }>;
+      /**
+       * #1338 — the services still dial the instance's IP, which no
+       * verify-full connection can use; their database URLs were left as
+       * they are. Only a converge moves them (cloudsql-tls.ts); a roll must
+       * not start while this is true.
+       */
+      readonly databaseHostPending: boolean;
     }
   | { readonly ok: false; readonly error: string } {
-  const host = liveDatabaseHost(services.admin.liveEnv);
-  if (!host.ok) {
+  const live = liveDatabaseHost(services.admin.liveEnv);
+  if (!opts.databaseHost && !live.ok) {
     return {
       ok: false,
-      error: `the Cloud SQL host is unknown: the admin service (${services.admin.serviceName}) ${host.error}`,
+      error: `the Cloud SQL host is unknown: the admin service (${services.admin.serviceName}) ${live.error}`,
     };
   }
-  const runtime: RuntimeEnvInputs = { ...install, databaseUrls: databaseUrls(host.host) };
+  const host = opts.databaseHost ?? (live.ok ? live.host : "");
+  const databaseHostPending = isIpLiteral(host);
+  // An IP cannot carry a verify-full URL (databaseUrls refuses it): the URLs
+  // are planned from a placeholder name and then left untouched, so the
+  // rest of the env still plans and the caller can see what is pending.
+  const runtime: RuntimeEnvInputs = {
+    ...install,
+    databaseUrls: databaseUrls(databaseHostPending ? "pending.invalid" : host),
+  };
   let inputs: AdminEnvInputs;
   if (install.provider === "gcp-firebase") {
     // The site id carries a random suffix Pulumi generated at install time.
@@ -340,36 +365,33 @@ export function planContractEnv(
   } else {
     inputs = { ...runtime, provider: "gcp" };
   }
-  const skip = new Set(opts.leaveUntouched ?? []);
+  const skip = new Set([
+    ...(opts.leaveUntouched ?? []),
+    ...(databaseHostPending ? [...DATABASE_URL_VARS.admin, ...DATABASE_URL_VARS.gateway] : []),
+  ]);
   const desired = (slug: CloudRunSlug, contract: readonly CloudRunEnvVar[]) => {
     const extra = opts.extraEnv?.[slug] ?? [];
     const clash = extra.find((e) => contract.some((c) => c.name === e.name));
     if (clash) throw new Error(`${slug}: ${clash.name} is both in the contract and extra env`);
     return [...contract, ...extra].filter((e) => !skip.has(e.name));
   };
-  let adminDesired: CloudRunEnvVar[];
-  let gatewayDesired: CloudRunEnvVar[];
+  let contracts: Record<CloudRunSlug, CloudRunEnvVar[]>;
   try {
-    adminDesired = desired("admin", adminEnvContract(inputs));
-    gatewayDesired = desired("gateway", gatewayEnvContract(runtime));
+    contracts = {
+      admin: desired("admin", adminEnvContract(inputs)),
+      gateway: desired("gateway", gatewayEnvContract(runtime)),
+      "mcp-auth-proxy": desired("mcp-auth-proxy", mcpAuthProxyEnvContract(install)),
+    };
   } catch (e) {
     return { ok: false, error: e instanceof Error ? e.message : String(e) };
   }
-  const admin = planEnvUpdate(services.admin.liveEnv, adminDesired, RETIRED_SERVICE_ENV.admin);
-  if (!admin.ok) return { ok: false, error: `admin: ${admin.error}` };
-  const gateway = planEnvUpdate(
-    services.gateway.liveEnv,
-    gatewayDesired,
-    RETIRED_SERVICE_ENV.gateway,
-  );
-  if (!gateway.ok) return { ok: false, error: `gateway: ${gateway.error}` };
-  return {
-    ok: true,
-    services: {
-      admin: { flags: admin.flags, changes: admin.changes },
-      gateway: { flags: gateway.flags, changes: gateway.changes },
-    },
-  };
+  const out = {} as Record<CloudRunSlug, { flags: string[]; changes: EnvChange[] }>;
+  for (const slug of CLOUD_RUN_SLUGS) {
+    const plan = planEnvUpdate(services[slug].liveEnv, contracts[slug], RETIRED_SERVICE_ENV[slug]);
+    if (!plan.ok) return { ok: false, error: `${slug}: ${plan.error}` };
+    out[slug] = { flags: plan.flags, changes: plan.changes };
+  }
+  return { ok: true, services: out, databaseHostPending };
 }
 
 // ===========================================================================
@@ -618,16 +640,24 @@ export function liveIngress(serviceJson: string): string | null {
  * the serving one and the app-roll's rollback target stay as they are. On
  * `gcp` the stack declares load-balancer-only and `staging-domain enable`
  * (admin-domain.ts) opens it deliberately, so this never runs there.
+ *
+ * #1328 — both public services of a gcp-firebase install: the data gateway
+ * (the Firebase rewrite, the health probe) and the MCP-Auth-Proxy (remote-MCP
+ * clients, and until #1331 the staging mapping it forwards).
  */
-async function ensureGatewayIngress(
+async function ensurePublicIngress(
   install: InstallTarget,
+  slug: "gateway" | "mcp-auth-proxy",
   run: GcloudRunner,
 ): Promise<InvariantOutcome> {
-  const service = install.services.gateway;
+  const service = install.services[slug];
   const base = {
     id: `ingress all on ${service}`,
     onFailure: "abort" as const,
-    why: "the gcp-firebase stack's gateway ingress (#650); the health check, the staging domain and remote MCP reach its run.app URL",
+    why:
+      slug === "gateway"
+        ? "the gcp-firebase stack's gateway ingress (#650); Firebase Hosting and the health check reach its run.app URL"
+        : "the gcp-firebase stack's MCP-Auth-Proxy ingress (#1328); remote-MCP clients, the health check and the staging domain reach its run.app URL",
   };
   const where = [`--region=${install.region}`, `--project=${install.projectId}`];
   const describe = await run(["run", "services", "describe", service, ...where, "--format=json"]);
@@ -697,7 +727,10 @@ export async function ensureStackInvariants(
     outcomes.push(await ensureCdnPolicy(install, run));
     outcomes.push(await ensureSvgDownloads(install, run));
   }
-  if (install.provider === "gcp-firebase") outcomes.push(await ensureGatewayIngress(install, run));
+  if (install.provider === "gcp-firebase") {
+    outcomes.push(await ensurePublicIngress(install, "gateway", run));
+    outcomes.push(await ensurePublicIngress(install, "mcp-auth-proxy", run));
+  }
   outcomes.push(...(await removeDefaultFirewallRules(install.projectId, { run })));
 
   return {
@@ -1183,6 +1216,35 @@ export function planCpuIdle(live: boolean, desired: boolean): RollAdditionsPlan
   };
 }
 
+/**
+ * Whether the service's revision template egresses into a VPC (Direct VPC
+ * egress, the stacks' `vpcAccess.networkInterfaces`).
+ */
+export function liveDirectVpc(serviceJson: string): boolean {
+  const svc = JSON.parse(serviceJson) as {
+    spec?: { template?: { metadata?: { annotations?: Record<string, string> } } };
+  };
+  return Boolean(
+    svc.spec?.template?.metadata?.annotations?.["run.googleapis.com/network-interfaces"],
+  );
+}
+
+/**
+ * #1328 — `--clear-network` when a service the contract keeps off the VPC
+ * still egresses into it. The MCP-Auth-Proxy on gcp-firebase is the service
+ * that was the gateway, which reached Cloud SQL's private IP that way; the
+ * proxy talks to the admin's public URL only, so it loses the route to the
+ * database's network together with its database credentials.
+ */
+export function planNoVpc(live: boolean): RollAdditionsPlan {
+  if (!live) return { ok: true, flags: [], changes: [] };
+  return {
+    ok: true,
+    flags: ["--clear-network"],
+    changes: ["VPC egress (private ranges) → none"],
+  };
+}
+
 /**
  * The single `gcloud run services update` that rolls a service to a new
  * image, its run SA and its env changes, so all land in one new revision.

package/src/state-store.test.ts

@@ -36,7 +36,8 @@ function fakeBucket() {
   return { objects, calls, run };
 }
 
-const PASSWORD_URL = "postgres://admin_role:hunter2hunter2@10.0.0.3:5432/cms_admin";
+const PASSWORD_URL =
+  "postgres://admin_role:hunter2hunter2@db1.p-label.europe-west1.sql-psa.goog:5432/cms_admin";
 
 let root = "";
 beforeEach(() => {

package/src/upgrade/app-roll.ts

@@ -25,15 +25,23 @@
  */
 
 import type { GcloudRunner, Sleep } from "../gcloud-retry.js";
-import { gatewayServiceAccountEmail, runServiceAccountEmail } from "../gcp-names.js";
+import {
+  gatewayServiceAccountEmail,
+  mcpAuthProxyServiceAccountEmail,
+  runServiceAccountEmail,
+} from "../gcp-names.js";
 import type { ImageDigests } from "../install-state.js";
 import {
   adminMediaVolume,
+  CLOUD_RUN_SLUGS,
   type CloudRunEnvVar,
-  GATEWAY_TIMEOUT_SECONDS,
+  type CloudRunSlug,
+  cloudRunServicePrefix,
+  MCP_AUTH_PROXY_TIMEOUT_SECONDS,
   RETIRED_SECRET_FILES,
   SERVICE_CPU_IDLE,
   SERVICE_SECRET_FILES,
+  SERVICE_VPC_EGRESS,
 } from "../stack-contract.js";
 import {
   type DeployedService,
@@ -44,12 +52,14 @@ import {
   liveContainerEnv,
   liveContainerMemory,
   liveCpuIdle,
+  liveDirectVpc,
   liveTimeoutSeconds,
   liveVolumes,
   planAdminMemory,
   planContractEnv,
   planCpuIdle,
   planMediaVolume,
+  planNoVpc,
   planRequestTimeout,
   planSecretFiles,
   rollService,
@@ -69,10 +79,27 @@ import {
 import type { HealthProbe } from "./health.js";
 import type { UpgradeReporter } from "./reporter.js";
 
-export type RollSlug = "admin" | "gateway";
+export type RollSlug = CloudRunSlug;
 
-/** The roll order: the gateway follows the admin, never the other way round. */
-export const ROLL_ORDER: readonly RollSlug[] = ["admin", "gateway"];
+/**
+ * The roll order: the gateway follows the admin, never the other way round,
+ * and the MCP-Auth-Proxy comes last (#1328) — on gcp-firebase it forwards
+ * the visitor traffic still addressed to it to the gateway, which must
+ * already serve the new release by then.
+ */
+export const ROLL_ORDER: readonly RollSlug[] = CLOUD_RUN_SLUGS;
+
+/** The service account each service's new revision runs as. */
+export function rollServiceAccount(slug: RollSlug, projectId: string, env: string): string {
+  switch (slug) {
+    case "admin":
+      return runServiceAccountEmail(projectId, env);
+    case "gateway":
+      return gatewayServiceAccountEmail(projectId, env);
+    case "mcp-auth-proxy":
+      return mcpAuthProxyServiceAccountEmail(projectId, env);
+  }
+}
 
 /** The install an app-roll acts on. */
 export interface AppRollInstall {
@@ -110,6 +137,8 @@ export interface ServicePlan {
   readonly liveTimeout: number | null;
   /** Whether the service gets CPU only during requests now. */
   readonly liveCpuIdle: boolean;
+  /** Whether the service egresses into the install's VPC now. */
+  readonly liveDirectVpc: boolean;
 }
 
 /** A service plan plus the changes its roll applies. */
@@ -139,6 +168,13 @@ export interface AppRollPlan {
   readonly services: readonly RollPlan[];
   /** Notes for the operator (e.g. a memory limit left as is). */
   readonly warnings: readonly string[];
+  /**
+   * #1338 — the services dial the Cloud SQL instance's IP, which no
+   * verify-full connection can use, so their database URLs are not planned.
+   * A converge moves them onto the instance's DNS name
+   * ({@link EnvAdjustments.databaseHost}); a roll refuses while this holds.
+   */
+  readonly databaseHostPending?: boolean;
 }
 
 export type PlanResult =
@@ -166,7 +202,7 @@ export async function planAppRoll(
 ): Promise<PlanResult> {
   const services: ServicePlan[] = [];
   for (const slug of ROLL_ORDER) {
-    const prefix = `caelo-${install.env}-${slug}`;
+    const prefix = cloudRunServicePrefix(install.provider, install.env, slug);
     const lookup = await resolveGcpResourceName(
       run,
       "run-service",
@@ -210,6 +246,7 @@ export async function planAppRoll(
       liveVolumes: liveVolumes(json),
       liveTimeout: liveTimeoutSeconds(json),
       liveCpuIdle: liveCpuIdle(json),
+      liveDirectVpc: liveDirectVpc(json),
     });
   }
   const env = planEnv(install, services, {});
@@ -265,29 +302,45 @@ export async function planAppRoll(
     // stack's memory default (never lowered; an unparsable value is kept).
     const memory = s.slug === "admin" ? planAdminMemory(s.liveMemory) : null;
     if (memory && !memory.ok) warnings.push(`admin memory left as is: ${memory.error}`);
-    // #806 — the gateway streams remote-MCP chat turns (stack-contract.ts
-    // GATEWAY_TIMEOUT_SECONDS). The admin's timeout is the stacks' alone.
+    // #806 — the MCP-Auth-Proxy streams remote-MCP chat turns
+    // (stack-contract.ts MCP_AUTH_PROXY_TIMEOUT_SECONDS). The other services'
+    // timeouts are the stacks' alone.
     const timeout =
-      s.slug === "gateway" ? planRequestTimeout(s.liveTimeout, GATEWAY_TIMEOUT_SECONDS) : null;
+      s.slug === "mcp-auth-proxy"
+        ? planRequestTimeout(s.liveTimeout, MCP_AUTH_PROXY_TIMEOUT_SECONDS)
+        : null;
     const memoryFlags = memory?.ok ? memory.flags : [];
     const timeoutFlags = timeout?.ok ? timeout.flags : [];
     const cpu = planCpuIdle(s.liveCpuIdle, SERVICE_CPU_IDLE[s.slug]);
     const cpuFlags = cpu.ok ? cpu.flags : [];
+    // #1328 — the gcp-firebase proxy is the former gateway service: it
+    // leaves the VPC in the same revision that drops its database env.
+    const vpc = SERVICE_VPC_EGRESS[s.slug] ? null : planNoVpc(s.liveDirectVpc);
+    const vpcFlags = vpc?.ok ? vpc.flags : [];
     return {
       ...s,
       envFlags: env.services[s.slug].flags,
       envChanges: env.services[s.slug].changes,
-      resourceFlags: [...memoryFlags, ...timeoutFlags, ...cpuFlags],
+      resourceFlags: [...memoryFlags, ...timeoutFlags, ...cpuFlags, ...vpcFlags],
       resourceChanges: [
         ...memoryFlags.map((f) => `${s.liveMemory ?? "(default)"} → ${f}`),
         ...(timeout?.ok ? timeout.changes : []),
         ...(cpu.ok ? cpu.changes : []),
+        ...(vpc?.ok ? vpc.changes : []),
       ],
       volumeFlags: [...(s.slug === "admin" ? media.flags : []), ...secretFiles[s.slug].flags],
       volumeChanges: [...(s.slug === "admin" ? media.changes : []), ...secretFiles[s.slug].changes],
     };
   });
-  return { ok: true, plan: { install, services: rolls, warnings } };
+  if (env.databaseHostPending) {
+    warnings.push(
+      "the services still dial the Cloud SQL instance's IP; the converge moves them onto its DNS name for verify-full TLS (#1338) before anything rolls",
+    );
+  }
+  return {
+    ok: true,
+    plan: { install, services: rolls, warnings, databaseHostPending: env.databaseHostPending },
+  };
 }
 
 /** Env the converge half adds to, or keeps out of, the rolls ({@link AppRollHooks.afterMigrations}). */
@@ -299,6 +352,11 @@ export interface EnvAdjustments {
    * identity env, which only the converge (operator identity) can resolve.
    */
   readonly extraEnv?: Readonly<Partial<Record<RollSlug, readonly CloudRunEnvVar[]>>>;
+  /**
+   * #1338 — the database host the converge moved the services onto (the
+   * Cloud SQL instance's DNS name, cloudsql-tls.ts).
+   */
+  readonly databaseHost?: string;
 }
 
 function planEnv(
@@ -322,6 +380,7 @@ function planEnv(
     {
       leaveUntouched: adjust.leaveUntouched ?? [],
       ...(adjust.extraEnv ? { extraEnv: adjust.extraEnv } : {}),
+      ...(adjust.databaseHost ? { databaseHost: adjust.databaseHost } : {}),
     },
   );
 }
@@ -333,11 +392,12 @@ function planEnv(
  */
 export function adjustingEnv(plan: AppRollPlan, adjust: EnvAdjustments): AppRollPlan {
   const extra = Object.values(adjust.extraEnv ?? {}).some((v) => (v?.length ?? 0) > 0);
-  if ((adjust.leaveUntouched?.length ?? 0) === 0 && !extra) return plan;
+  if ((adjust.leaveUntouched?.length ?? 0) === 0 && !extra && !adjust.databaseHost) return plan;
   const env = planEnv(plan.install, plan.services, adjust);
   if (!env.ok) return plan;
   return {
     ...plan,
+    databaseHostPending: env.databaseHostPending,
     services: plan.services.map((s) => ({
       ...s,
       envFlags: env.services[s.slug].flags,
@@ -370,7 +430,7 @@ export interface AppRollHooks {
   afterMigrations?(): Promise<
     | ({ readonly ok: true; readonly leaveUntouched: readonly string[] } & Pick<
         EnvAdjustments,
-        "extraEnv"
+        "extraEnv" | "databaseHost"
       >)
     | { readonly ok: false }
   >;
@@ -470,6 +530,17 @@ export async function executeAppRoll(
     r.warn("--skip-health-check set — the new revisions get Cloud Run's readiness check only.");
   }
 
+  // #1338 — the new images refuse a database URL without verify-full, and
+  // only a converge can move the services off the instance's IP. Without
+  // one (the in-cloud upgrader) the roll stops before it changes anything.
+  if (plan.databaseHostPending && !hooks.afterMigrations) {
+    r.error(
+      "The services still dial the Cloud SQL instance's IP address. This release connects with verify-full TLS, which needs the instance's DNS name: run the converge first (Settings → Updates installs it as a cloud-setup release, or `cms-provision upgrade` from a terminal).",
+    );
+    r.warn("No migrations ran and no traffic moved.");
+    return fail("migrations", false);
+  }
+
   // Migrations BEFORE any traffic moves, with the NEW admin image: the old
   // image does not carry the release's migration files (v0.9.2).
   const admin = plan.services.find((s) => s.slug === "admin") as RollPlan;
@@ -490,8 +561,15 @@ export async function executeAppRoll(
     plan = adjustingEnv(plan, {
       leaveUntouched: converged.leaveUntouched,
       ...(converged.extraEnv ? { extraEnv: converged.extraEnv } : {}),
+      ...(converged.databaseHost ? { databaseHost: converged.databaseHost } : {}),
     });
   }
+  if (plan.databaseHostPending) {
+    r.error(
+      "The converge did not move the services onto the Cloud SQL instance's DNS name, so the new revisions could not reach the database with verify-full TLS (#1338). Nothing rolled; fix the database TLS step above and re-run.",
+    );
+    return fail("converge", true);
+  }
 
   const rolled: RollPlan[] = [];
   /** Jobs this roll moved onto the admin's image (#875, #1217), with their prior image. */
@@ -540,10 +618,7 @@ export async function executeAppRoll(
         region: install.region,
         projectId: install.projectId,
         imageRef: p.imageRef,
-        serviceAccount:
-          p.slug === "admin"
-            ? runServiceAccountEmail(install.projectId, install.env)
-            : gatewayServiceAccountEmail(install.projectId, install.env),
+        serviceAccount: rollServiceAccount(p.slug, install.projectId, install.env),
         envFlags: p.envFlags,
         resourceFlags: p.resourceFlags,
         volumeFlags: p.volumeFlags,
@@ -637,9 +712,12 @@ export async function executeAppRoll(
     s.done(`Healthy: ${deep.detail}`);
   }
 
+  const digestOf = (slug: RollSlug) =>
+    (plan.services.find((s) => s.slug === slug) as RollPlan).digest;
   const digests: ImageDigests = {
-    admin: (plan.services.find((s) => s.slug === "admin") as RollPlan).digest,
-    gateway: (plan.services.find((s) => s.slug === "gateway") as RollPlan).digest,
+    admin: digestOf("admin"),
+    gateway: digestOf("gateway"),
+    "mcp-auth-proxy": digestOf("mcp-auth-proxy"),
   };
   await deps.recordDigests(digests);
   return { ok: true, digests };