@caelo-cms/provisioning 0.11.10 → 0.11.12

package/src/converge-job.test.ts

@@ -19,6 +19,8 @@ import {
   type ConvergeQueue,
   type ConvergeUpgradeOutcome,
   runConvergeJob,
+  SELF_GRANT_RECHECK_ATTEMPTS,
+  type SecureDatabaseResult,
 } from "./converge-job.js";
 import type { ExecutionState } from "./install-job.js";
 import type { InstallMetadata } from "./install-state.js";
@@ -147,6 +149,10 @@ function harness(
     imageSigned?: boolean;
     /** What IAM answers the self-check; every permission asked for by default. */
     iam?: IamTester;
+    /** #1373 — the database TLS step's answer; null: the queue dials the DNS name already. */
+    secureDatabase?: SecureDatabaseResult | null;
+    /** What a self-grant answers; ok by default. */
+    grant?: (role: string) => { ok: true } | { ok: false; error: string };
   } = {},
 ): Harness {
   const calls: string[] = [];
@@ -156,11 +162,25 @@ function harness(
   let stored = opts.checkpoint ?? null;
   const deps: ConvergeJobDeps = {
     queue: q.queue,
+    secureDatabase:
+      opts.secureDatabase === undefined || opts.secureDatabase === null
+        ? null
+        : async () => {
+            calls.push("secure database");
+            return opts.secureDatabase as SecureDatabaseResult;
+          },
     projectId: "acme",
     region: "europe-west1",
     iam: opts.iam ?? {
       granted: async (_resource, permissions) => ({ ok: true, granted: permissions }),
     },
+    grantOwnRole: async (role) => {
+      calls.push(`grant ${role}`);
+      return opts.grant ? opts.grant(role) : { ok: true };
+    },
+    sleep: async () => {
+      calls.push("sleep");
+    },
     execution: "caelo-production-converge-run1",
     ownVersion: opts.ownVersion ?? "0.11.0",
     fetchManifest: async (v) => {
@@ -455,6 +475,139 @@ describe("runConvergeJob — stage 2 (the target release's own installer)", () =
   });
 });
 
+/**
+ * An IAM that lacks the workflows.* permissions (an install set up before
+ * 0.11.11 added roles/workflows.editor, the searchviu-website case) until
+ * the job granted the role, and confirms it `lag` asks later.
+ */
+function lacksWorkflows(lag = 0, extraMissing: (p: string) => boolean = () => false) {
+  const state = { granted: false, asksSinceGrant: 0 };
+  const iam: IamTester = {
+    granted: async (_resource, permissions) => {
+      if (state.granted) state.asksSinceGrant += 1;
+      const workflowsHeld = state.granted && state.asksSinceGrant > lag;
+      return {
+        ok: true,
+        granted: permissions.filter(
+          (p) => !extraMissing(p) && (workflowsHeld || !p.startsWith("workflows.")),
+        ),
+      };
+    },
+  };
+  const grant = (role: string) => {
+    if (role === "roles/workflows.editor") state.granted = true;
+    return { ok: true } as const;
+  };
+  return { iam, grant };
+}
+
+describe("runConvergeJob — the job grants itself the roles its release names (2026-11-03)", () => {
+  it("stage 1 leaves a missing allowlisted role to stage 2 and hands over without granting", async () => {
+    const { iam, grant } = lacksWorkflows();
+    const h = harness({ ownVersion: "0.10.53", iam, grant });
+    const r = await runConvergeJob(h.deps);
+    expect(r.status).toBeNull();
+    expect(h.calls.some((c) => c.startsWith("grant"))).toBe(false);
+    expect(h.calls).toContain("start");
+    expect(h.q.ends).toEqual([]);
+    expect(
+      h.q.events.some(
+        (e) =>
+          e.message.includes("lacks roles/workflows.editor") &&
+          e.message.includes("The installer of 0.11.0 grants them"),
+      ),
+    ).toBe(true);
+  });
+
+  it("stage 2 grants the role after the image check and under the lock, records it, re-checks, converges", async () => {
+    const { iam, grant } = lacksWorkflows();
+    const h = harness({ iam, grant });
+    const r = await runConvergeJob(h.deps);
+    expect(r.status).toBe("succeeded");
+    const g = h.calls.indexOf("grant roles/workflows.editor");
+    expect(g).toBeGreaterThan(h.calls.indexOf("lock"));
+    expect(g).toBeLessThan(h.calls.indexOf("checkpoint running"));
+    expect(g).toBeLessThan(h.calls.indexOf("upgrade"));
+    // Only the missing role, nothing else of the list.
+    expect(h.calls.filter((c) => c.startsWith("grant"))).toEqual(["grant roles/workflows.editor"]);
+    const event = h.q.events.find((e) => e.message.startsWith("Granted roles/workflows.editor"));
+    expect(event?.level).toBe("success");
+    expect(event?.message).toBe(
+      "Granted roles/workflows.editor to the converge job's account (caelo-provisioner@acme.iam.gserviceaccount.com), needed by Caelo 0.11.0 to deploy and update the workflow the scheduler tick runs (#1140).",
+    );
+    expect(h.q.events.some((e) => e.message.startsWith("IAM confirms the granted roles"))).toBe(
+      true,
+    );
+  });
+
+  it("waits for IAM to confirm a fresh grant (propagation) before it goes on", async () => {
+    const { iam, grant } = lacksWorkflows(3);
+    const h = harness({ iam, grant });
+    const r = await runConvergeJob(h.deps);
+    expect(r.status).toBe("succeeded");
+    const sleeps = h.calls.filter((c) => c === "sleep").length;
+    expect(sleeps).toBeGreaterThan(0);
+    expect(h.calls.lastIndexOf("sleep")).toBeLessThan(h.calls.indexOf("upgrade"));
+  });
+
+  it("a grant that fails (org policy) ends the request with the Cloud Shell step, before any change", async () => {
+    const { iam } = lacksWorkflows();
+    const h = harness({
+      iam,
+      grant: () => ({
+        ok: false,
+        error: "ERROR: (gcloud.projects.add-iam-policy-binding) FAILED_PRECONDITION: org policy",
+      }),
+    });
+    const r = await runConvergeJob(h.deps);
+    expect(r.status).toBe("needs_cloud_shell");
+    expect(h.q.ends[0]?.codes).toEqual(["iam-self-grant"]);
+    const reason = h.q.ends[0]?.reason ?? "";
+    expect(reason).toStartWith(
+      "Nothing was changed: the converge job could not grant its own account",
+    );
+    expect(reason).toContain("roles/workflows.editor");
+    expect(reason).toContain("org policy");
+    expect(reason).toContain("Run the update once from Cloud Shell");
+    expect(h.upgrades).toEqual([]);
+    expect(h.calls.some((c) => c.startsWith("checkpoint"))).toBe(false);
+    expect(h.calls.at(-1)).toBe("unlock");
+  });
+
+  it("a grant IAM never confirms ends with the Cloud Shell step after the wait", async () => {
+    const { iam, grant } = lacksWorkflows(10_000);
+    const h = harness({ iam, grant });
+    const r = await runConvergeJob(h.deps);
+    expect(r.status).toBe("needs_cloud_shell");
+    expect(h.q.ends[0]?.codes).toEqual(["iam-self-grant"]);
+    expect(h.q.ends[0]?.reason).toContain("workflows.workflows.create");
+    expect(h.calls.filter((c) => c === "sleep").length).toBe(SELF_GRANT_RECHECK_ATTEMPTS - 1);
+    expect(h.upgrades).toEqual([]);
+  });
+
+  it("a missing permission no allowlisted role carries stops either stage without granting anything", async () => {
+    // workflows.* is repairable, actAs on its own account is not.
+    const { iam, grant } = lacksWorkflows(0, (p) => p === "iam.serviceAccounts.actAs");
+    for (const stage of ["0.10.53", "0.11.0"]) {
+      const h = harness({ ownVersion: stage, iam, grant });
+      const r = await runConvergeJob(h.deps);
+      expect(r.status).toBe("needs_cloud_shell");
+      expect(h.q.ends[0]?.codes).toEqual(["iam-preflight"]);
+      expect(h.calls.some((c) => /^(grant|lock|point|start|upgrade)/.test(c))).toBe(false);
+      expect(h.q.ends[0]?.reason).toContain("workflows.workflows.create");
+      expect(h.q.ends[0]?.reason).toContain("iam.serviceAccounts.actAs");
+    }
+  });
+
+  it("stage 2 refuses an image the manifest does not sign before it grants anything", async () => {
+    const { iam, grant } = lacksWorkflows();
+    const h = harness({ iam, grant, ownDigest: D("9") });
+    const r = await runConvergeJob(h.deps);
+    expect(r.status).toBe("refused");
+    expect(h.calls.some((c) => c.startsWith("grant"))).toBe(false);
+  });
+});
+
 describe("runConvergeJob — resume after a killed run", () => {
   it("a run that died is adopted once Cloud Run says it ended, and resumes from the checkpoint", async () => {
     const h = harness({
@@ -556,6 +709,66 @@ describe("runConvergeJob — resume after a killed run", () => {
   });
 });
 
+describe("runConvergeJob — the database TLS step (#1373)", () => {
+  const DNS = "abc.p-label.europe-west1.sql-psa.goog";
+
+  it("a failed TLS step reaches the timeline and ends the request, before any roll", async () => {
+    const h = harness({
+      secureDatabase: {
+        ok: false,
+        error:
+          "database TLS could not be set up: caelo-production-pg-1 reports no PRIVATE_SERVICES_ACCESS DNS name",
+      },
+    });
+    const r = await runConvergeJob(h.deps);
+    expect(r.exitCode).toBe(1);
+    expect(r.status).toBe("failed");
+    expect(h.upgrades).toEqual([]);
+    // Under the lock, after the resumable checkpoint, released at the end.
+    expect(h.calls).toEqual([
+      "manifest 0.11.0",
+      "lock",
+      "checkpoint running",
+      "secure database",
+      "checkpoint failed",
+      "unlock",
+    ]);
+    const end = h.q.ends[0];
+    expect(end?.codes).toEqual(["database-tls"]);
+    expect(end?.reason).toContain("reports no PRIVATE_SERVICES_ACCESS DNS name");
+    // The step's start and its error are timeline events of the request.
+    const converge = h.q.events.filter((e) => e.phase === "converge");
+    expect(converge[0]?.message).toStartWith("Setting up verified encryption");
+    const error = h.q.events.find((e) => e.level === "error");
+    expect(error?.message).toContain("reports no PRIVATE_SERVICES_ACCESS DNS name");
+  });
+
+  it("a successful TLS step runs right before the upgrade and names the verified host", async () => {
+    const h = harness({ secureDatabase: { ok: true, host: DNS } });
+    const r = await runConvergeJob(h.deps);
+    expect(r.status).toBe("succeeded");
+    const at = h.calls.indexOf("secure database");
+    expect(at).toBeGreaterThan(h.calls.indexOf("lock"));
+    expect(h.calls[at + 1]).toBe("upgrade");
+    expect(h.q.events.some((e) => e.level === "success" && e.message.includes(DNS))).toBe(true);
+  });
+
+  it("stage 1 never runs it: the release's own installer changes the database", async () => {
+    const h = harness({ ownVersion: "0.10.53", secureDatabase: { ok: true, host: DNS } });
+    await runConvergeJob(h.deps);
+    expect(h.calls.includes("secure database")).toBe(false);
+  });
+
+  it("a stop before the lock leaves the database alone", async () => {
+    const h = harness({
+      secureDatabase: { ok: true, host: DNS },
+      manifest: { ok: false, error: { code: "signature", message: "no valid signature" } },
+    });
+    await runConvergeJob(h.deps);
+    expect(h.calls.includes("secure database")).toBe(false);
+  });
+});
+
 describe("runConvergeJob — nothing to do", () => {
   it("exits cleanly without a converge request", async () => {
     const h = harness({ claim: null });

package/src/converge-job.ts

@@ -16,7 +16,9 @@
  *     verifies its signature (release-manifest.ts), and decides
  *     (upgrader-assess.ts, `route: "converge"`): no downgrade, nothing
  *     below minFromVersion, no automatic converge unless the policy allows
- *     it; then it checks its own IAM access (converge-job-preflight.ts).
+ *     it; then it checks its own IAM access (converge-job-preflight.ts) and
+ *     stops on anything missing — except a role of the self-grant
+ *     allowlist, which it leaves to stage 2.
  *     Then it verifies the target's ops image, points this job at it BY DIGEST and starts a new
  *     execution, handing the request over to it. It changes nothing else:
  *     the converge logic always comes from the release it installs.
@@ -25,7 +27,9 @@
  *     version is the request's). It verifies the manifest again, checks
  *     that the image it runs is the one the manifest signs, takes the
  *     install's state lock (#815; the upgrader job and the terminal
- *     `upgrade` take the same one, so they exclude each other), and runs
+ *     `upgrade` take the same one, so they exclude each other), grants its
+ *     own account the allowlisted roles the self-check found missing (each
+ *     a timeline event; IAM is asked again before it goes on), and runs
  *     the very upgrade the terminal runs (upgrade/upgrade.ts): backup →
  *     converge + app-roll → deep health check, rollback on failure →
  *     install.json + the infrastructure contract → the request's end.
@@ -46,10 +50,16 @@
  */
 
 import type { ConvergeCheckpoint } from "./converge-checkpoint.js";
-import { CONVERGE_PREFLIGHT_VOICE, convergeAccessChecks } from "./converge-job-preflight.js";
+import {
+  CONVERGE_PREFLIGHT_VOICE,
+  convergeAccessChecks,
+  INSTALL_ROLE_USES,
+  selfGrantPlan,
+} from "./converge-job-preflight.js";
 import { GCP_STACK_ENV } from "./gcp-names.js";
-import { decideInstallJob, type ExecutionState } from "./install-job.js";
+import { decideInstallJob, type ExecutionState, type InstallJobRole } from "./install-job.js";
 import type { InstallMetadata } from "./install-state.js";
+import { provisionerServiceAccountEmail } from "./provisioner-identity.js";
 import {
   compareSemver,
   type ReleaseManifest,
@@ -60,6 +70,7 @@ import type { UpgradeCheckpoint } from "./upgrade/upgrade.js";
 import { assessUpgradeRequest } from "./upgrader-assess.js";
 import {
   type IamTester,
+  missingAccessText,
   preflightErrorReason,
   preflightMissingReason,
   runIamPreflight,
@@ -116,14 +127,46 @@ export type ConvergeUpgradeOutcome =
       readonly unrecorded: boolean;
     };
 
+/**
+ * How long stage 2 waits for IAM to confirm the roles it granted itself:
+ * Google documents that a policy change usually applies within two minutes
+ * and can take seven (cloud.google.com/iam/docs/access-change-propagation).
+ */
+export const SELF_GRANT_RECHECK_INTERVAL_MS = 15_000;
+export const SELF_GRANT_RECHECK_ATTEMPTS = 32;
+
+/** How {@link ConvergeJobDeps.secureDatabase} ended. */
+export type SecureDatabaseResult =
+  | { readonly ok: true; readonly host: string }
+  | { readonly ok: false; readonly error: string };
+
 export interface ConvergeJobDeps {
   readonly queue: ConvergeQueue;
+  /**
+   * #1373 — the install's database TLS step (#1338, cloudsql-tls.ts), and
+   * then {@link queue} moved onto the database's DNS name with verify-full.
+   * Null when the queue already dials that name. Until it ran the queue
+   * dials the instance's IP (converge-db-trust.ts), so a failure of the
+   * step reaches the timeline and the request's end like any other step;
+   * when it fails the queue is reopened on whatever the install dials then.
+   */
+  readonly secureDatabase: (() => Promise<SecureDatabaseResult>) | null;
   /** From the metadata server — never from env or argv. */
   readonly projectId: string;
   /** The region this job runs in (metadata server). */
   readonly region: string;
   /** Asks IAM as this job's own identity (the IAM self-check, #884). */
   readonly iam: IamTester;
+  /**
+   * Grant this job's own account (the install's provisioner) `role` on the
+   * project. Stage 2 only, and only for roles of the self-grant allowlist
+   * (converge-job-preflight.ts); the real one refuses any other role.
+   */
+  readonly grantOwnRole: (
+    role: InstallJobRole,
+  ) => Promise<{ readonly ok: true } | { readonly ok: false; readonly error: string }>;
+  /** Waits between the IAM re-checks after a self-grant. */
+  readonly sleep: (ms: number) => Promise<void>;
   /** This Cloud Run execution; null outside Cloud Run. */
   readonly execution: string | null;
   /** The Caelo release this code belongs to (CAELO_VERSION of the image). */
@@ -312,18 +355,11 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
 
     // #884 — decided to run: ask IAM for what the rest needs before the
     // lock, the hand-over or the backup (converge-job-preflight.ts). Both
-    // stages ask; stage 2's list is the target release's own.
-    const access = await runIamPreflight(
-      convergeAccessChecks({ projectId: deps.projectId, region: deps.region, env: GCP_STACK_ENV }),
-      deps.iam,
-    );
-    if (access.kind === "missing") {
-      return stopWith(
-        "needs_cloud_shell",
-        ["iam-preflight"],
-        preflightMissingReason(access.missing, CONVERGE_PREFLIGHT_VOICE),
-      );
-    }
+    // stages ask; stage 2's list is the target release's own. A missing
+    // role of the self-grant allowlist does not stop the run: stage 2
+    // grants it once it has verified its own image and holds the lock.
+    const stage1 = compareSemver(deps.ownVersion, m.version) !== 0;
+    const access = await askIam();
     if (access.kind === "error") {
       return stopWith(
         "failed",
@@ -331,12 +367,36 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
         preflightErrorReason(access.errors, CONVERGE_PREFLIGHT_VOICE),
       );
     }
-    r.info("The converge job has every permission this update needs.");
+    let toGrant: readonly InstallJobRole[] = [];
+    if (access.kind === "missing") {
+      const plan = selfGrantPlan(access.missing);
+      if (plan.blocking.length > 0) {
+        return stopWith(
+          "needs_cloud_shell",
+          ["iam-preflight"],
+          preflightMissingReason(access.missing, CONVERGE_PREFLIGHT_VOICE),
+        );
+      }
+      toGrant = plan.grant;
+    }
+    if (toGrant.length === 0) {
+      r.info("The converge job has every permission this update needs.");
+    } else {
+      r.info(
+        `The converge job's account lacks ${toGrant.join(", ")}, which this update needs. ${stage1 ? `The installer of ${m.version} grants them to it` : "It grants them to itself"} before it changes anything, once its own installer image is verified.`,
+      );
+    }
 
-    if (compareSemver(deps.ownVersion, m.version) !== 0) {
+    if (stage1) {
       return await handOver(m);
     }
-    return await convergeHere(m, fromVersion ?? (meta.releaseVersion as string), checkpoint, meta);
+    return await convergeHere(
+      m,
+      fromVersion ?? (meta.releaseVersion as string),
+      checkpoint,
+      meta,
+      toGrant,
+    );
   } catch (e) {
     return stopWith(
       "failed",
@@ -400,6 +460,7 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
     fromVersion: string,
     checkpoint: ConvergeCheckpoint | null,
     judged: InstallMetadata,
+    toGrant: readonly InstallJobRole[],
   ): Promise<UpgraderRunResult> {
     const installId = judged.installId;
     const own = await deps.ownImageDigest();
@@ -439,6 +500,13 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
     const stale = recordsChangedSince(judged, changed);
     if (stale) return stopWith(stale.status, stale.codes, stale.reason);
 
+    // After the image check and under the lock: only the target release's
+    // verified installer grants, and only roles of its own allowlist.
+    if (toGrant.length > 0) {
+      const granted = await grantOwnRoles(m, toGrant, changed);
+      if (granted) return stopWith(granted.status, granted.codes, granted.reason);
+    }
+
     const startedAt = deps.now().toISOString();
     let current: ConvergeCheckpoint = {
       statusVersion: 1,
@@ -463,6 +531,24 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
     save({});
 
     r.phase("converge");
+    // #1373 — stage 2 only, under the lock and after every check: it
+    // changes the database instance, its DNS and the services' URLs.
+    if (deps.secureDatabase) {
+      r.info(
+        "Setting up verified encryption for the database connection (the database's certificate and its private DNS name).",
+      );
+      await r.flush();
+      const secured = await deps.secureDatabase();
+      if (!secured.ok) {
+        save({ state: "failed", phase: "stopped" });
+        return stopWith(
+          "failed",
+          ["database-tls"],
+          `The update to ${m.version} stopped at the database's TLS step, before it rolled anything: ${secured.error}. The previous release keeps serving. Fix what it names and start the update again; it continues where it stopped.`,
+        );
+      }
+      r.success(`The database connection is verified (${secured.host}).`);
+    }
     const outcome = await deps.upgrade({
       manifest: m,
       reporter: r,
@@ -518,6 +604,76 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
     );
   }
 
+  /** The converge self-check against IAM, as this job's identity. */
+  function askIam() {
+    return runIamPreflight(
+      convergeAccessChecks({ projectId: deps.projectId, region: deps.region, env: GCP_STACK_ENV }),
+      deps.iam,
+    );
+  }
+
+  /**
+   * Stage 2: grant this job's account each role in `roles` (the self-check
+   * found them missing; all are on the allowlist), then wait until IAM
+   * confirms the whole self-check. Null when it does; how the request ends
+   * otherwise.
+   */
+  async function grantOwnRoles(
+    m: ReleaseManifest,
+    roles: readonly InstallJobRole[],
+    changed: boolean,
+  ): Promise<{ status: UpdateEndStatus; codes: string[]; reason: string } | null> {
+    const account = provisionerServiceAccountEmail(deps.projectId);
+    const granted: InstallJobRole[] = [];
+    const before = () =>
+      changed
+        ? "This update already started changing the site; "
+        : granted.length > 0
+          ? `Nothing on the site was changed (the converge job's account got ${granted.join(", ")}): `
+          : "Nothing was changed: ";
+    for (const role of roles) {
+      const result = await deps.grantOwnRole(role);
+      if (!result.ok) {
+        return {
+          status: "needs_cloud_shell",
+          codes: ["iam-self-grant"],
+          reason: `${before()}the converge job could not grant its own account (${account}) ${role}, which Caelo ${m.version} needs to ${INSTALL_ROLE_USES[role].need} (${result.error}). ${CONVERGE_PREFLIGHT_VOICE.repair}`,
+        };
+      }
+      granted.push(role);
+      r.success(
+        `Granted ${role} to the converge job's account (${account}), needed by Caelo ${m.version} to ${INSTALL_ROLE_USES[role].need}.`,
+      );
+    }
+    await r.flush();
+    let last = await askIam();
+    for (let attempt = 1; last.kind !== "ok" && attempt < SELF_GRANT_RECHECK_ATTEMPTS; attempt++) {
+      await deps.sleep(SELF_GRANT_RECHECK_INTERVAL_MS);
+      last = await askIam();
+    }
+    if (last.kind === "ok") {
+      r.info(
+        "IAM confirms the granted roles; the converge job has every permission this update needs.",
+      );
+      return null;
+    }
+    const minutes = Math.round(
+      (SELF_GRANT_RECHECK_INTERVAL_MS * (SELF_GRANT_RECHECK_ATTEMPTS - 1)) / 60_000,
+    );
+    if (last.kind === "error") {
+      return {
+        status: "failed",
+        codes: ["iam-self-grant"],
+        reason: `${before()}after granting its own account ${granted.join(", ")}, the converge job could not check its access again (${last.errors.join("; ")}). Start the update again from Settings → Updates in a few minutes.`,
+      };
+    }
+    return {
+      status: "needs_cloud_shell",
+      codes: ["iam-self-grant"],
+      reason: `${before()}the converge job granted its own account ${granted.join(", ")}, but ${minutes} minutes later Google Cloud still reports permissions missing — ${missingAccessText(last.missing)}. ${CONVERGE_PREFLIGHT_VOICE.repair}`,
+    };
+  }
+
   /**
    * Whether install.json, re-read under the lock, is still what the run was
    * judged on; how the request ends otherwise.

package/src/database-tls-builders.test.ts

@@ -28,7 +28,7 @@ describe("database URL builders (#1338)", () => {
     const yaml = generateDockerCompose({
       domain: "example.com",
       postgresPassword: "superpw",
-      rolePasswords: { admin: "adminpw", public: "publicpw" },
+      rolePasswords: { admin: "adminpw", public: "publicpw", publicStaging: "ps" },
       caeloSecretKek: "0".repeat(64),
       dbTlsServerCertSha256: "abababababababababababababababababababababababababababababababab",
       internalSecret: "i".repeat(48),
@@ -42,7 +42,8 @@ describe("database URL builders (#1338)", () => {
       project: "caelo-example-com",
     });
     const urls = [...yaml.matchAll(/_DATABASE_URL: "([^"]+)"/g)].map((m) => m[1] as string);
-    expect(urls).toHaveLength(3);
+    // admin: cms_admin, cms_public, cms_public_staging (#1329); gateway: cms_public.
+    expect(urls).toHaveLength(4);
     for (const url of urls) expect(databaseTlsProblem(url)).toBeNull();
   });
 

package/src/deployed-release.test.ts

@@ -128,6 +128,9 @@ describe("checkDeployedRuntimeEnv", () => {
   const adminOk = svc([
     secret("ADMIN_DATABASE_PASSWORD"),
     secret("PUBLIC_ADMIN_DATABASE_PASSWORD"),
+    // #1329 — the staging database's pool: a release from before it is
+    // refused here too and goes through `upgrade` first.
+    secret("PUBLIC_STAGING_ADMIN_DATABASE_PASSWORD"),
   ]);
   const gatewayOk = svc([secret("PUBLIC_DATABASE_PASSWORD", "public-role-password")]);
   const proxyOk = svc([secret("CAELO_MCP_OAUTH_SEAL_KEY", "mcp-oauth-seal-key")]);

package/src/gateway-env-isolation.test.ts

@@ -16,6 +16,12 @@
  *   self-hosted        the generated compose gateway service: env, secrets,
  *                      mounts
  *
+ * Issue #1329 — the same crosswise between the two public databases: the
+ * live gateway's env names nothing of the staging database
+ * (cms_public_staging, public_staging_role, its password secret), and its
+ * identity cannot read the staging login's password, on every adapter. The
+ * staging gateway (#1330) gets the mirror-image checks with its service.
+ *
  * The gateway itself refuses to boot on such an env
  * (apps/api-gateway `adminDatabaseAccessInEnv`); this catches it at review
  * time, before a release can ship it.
@@ -42,6 +48,12 @@ const ADMIN_ACCESS = /cms_admin|admin_role|gateway_role|ADMIN_DATABASE|GATEWAY_D
 /** Secrets that unlock a cms_admin login. */
 const ADMIN_SECRETS = ["postgres-password", "gateway-role-password"];
 
+/** #1329 — anything in the live gateway's env that reaches the staging database. */
+const STAGING_ACCESS = /cms_public_staging|public_staging_role|PUBLIC_STAGING/;
+
+/** #1329 — the secret that unlocks the staging gateway's login. */
+const STAGING_SECRETS = ["public-staging-role-password"];
+
 /** The secrets the gateway reads on GCP: its own login and the staging gate key. */
 const GCP_GATEWAY_SECRETS = [
   "public-role-password",
@@ -68,11 +80,19 @@ describe("gcp + gcp-firebase: the gateway env contract (#635)", () => {
     databaseUrls: databaseUrls("db1.p-label.europe-west1.sql-psa.goog"),
   });
 
-  it("carries no cms_admin URL, role or credential name", () => {
+  it("carries no cms_admin URL, role or credential name, and nothing of the staging database (#1329)", () => {
     for (const v of env) {
       expect(v.name).not.toMatch(ADMIN_ACCESS);
-      if ("value" in v && typeof v.value === "string") expect(v.value).not.toMatch(ADMIN_ACCESS);
+      expect(v.name).not.toMatch(STAGING_ACCESS);
+      if ("value" in v && typeof v.value === "string") {
+        expect(v.value).not.toMatch(ADMIN_ACCESS);
+        expect(v.value).not.toMatch(STAGING_ACCESS);
+      }
     }
+    expect(env.find((v) => v.name === "CAELO_GATEWAY_ENVIRONMENT")).toEqual({
+      name: "CAELO_GATEWAY_ENVIRONMENT",
+      value: "live",
+    });
     expect(env.find((v) => v.name === "PUBLIC_DATABASE_URL")).toEqual({
       name: "PUBLIC_DATABASE_URL",
       value:
@@ -80,12 +100,16 @@ describe("gcp + gcp-firebase: the gateway env contract (#635)", () => {
     });
   });
 
-  it("references no secret that unlocks a cms_admin login", () => {
+  it("references no secret that unlocks a cms_admin or the staging login", () => {
     for (const secret of Object.values(SERVICE_SECRET_ENV.gateway)) {
       expect(ADMIN_SECRETS).not.toContain(secret);
+      expect(STAGING_SECRETS).not.toContain(secret);
     }
     const gatewayBindings = runtimeSecretBindings().filter((b) => b.service === "gateway");
-    for (const b of gatewayBindings) expect(ADMIN_SECRETS).not.toContain(b.secret);
+    for (const b of gatewayBindings) {
+      expect(ADMIN_SECRETS).not.toContain(b.secret);
+      expect(STAGING_SECRETS).not.toContain(b.secret);
+    }
   });
 
   for (const provider of ["gcp", "gcp-firebase"] as const) {
@@ -134,7 +158,7 @@ describe("self-hosted: the generated compose gateway service (#635)", () => {
     const out = generateDockerCompose({
       domain: "example.com",
       postgresPassword: "superpw",
-      rolePasswords: { admin: "adminpw", public: "publicpw" },
+      rolePasswords: { admin: "adminpw", public: "publicpw", publicStaging: "stagingpw" },
       caeloSecretKek: "0".repeat(64),
       dbTlsServerCertSha256: "abababababababababababababababababababababababababababababababab",
       internalSecret: "i".repeat(48),
@@ -152,7 +176,9 @@ describe("self-hosted: the generated compose gateway service (#635)", () => {
     const gateway = out.slice(start, end === -1 ? undefined : end);
     expect(gateway).toContain("PUBLIC_DATABASE_URL:");
     expect(gateway).not.toMatch(ADMIN_ACCESS);
-    expect(gateway).not.toMatch(/superpw|adminpw|gatewaypw/);
+    expect(gateway).not.toMatch(STAGING_ACCESS);
+    expect(gateway).not.toMatch(/superpw|adminpw|gatewaypw|stagingpw/);
+    expect(gateway).toContain('CAELO_GATEWAY_ENVIRONMENT: "live"');
     // No other way in: no Compose secrets, no env file, and exactly one
     // mount — the database CA certificate (#1338), public, no credential.
     expect(gateway).not.toMatch(/^ {4}(secrets|env_file):/m);
@@ -164,6 +190,8 @@ describe("self-hosted: the generated compose gateway service (#635)", () => {
     expect(keys.sort()).toEqual([
       // #1254 — its cms_public pool size (pool-budget-stack.ts).
       "CAELO_DB_PUBLIC_POOL_MAX",
+      // #1329 — the live site's gateway.
+      "CAELO_GATEWAY_ENVIRONMENT",
       "CAELO_TRUSTED_PROXY_HOPS",
       "GATEWAY_PORT",
       "NODE_ENV",
@@ -181,6 +209,9 @@ describe("azure: the gateway container app (#635)", () => {
     const block = src.slice(start, src.indexOf("});", start));
     expect(block).toContain('{ name: "PUBLIC_DATABASE_URL", value: publicDatabaseUrl }');
     expect(block).not.toMatch(/adminDatabase|gatewayDatabase|ADMIN_DATABASE|GATEWAY_DATABASE/);
+    // #1329 — nothing of the staging database, and it says it is the live one.
+    expect(block).not.toMatch(/publicStaging|PUBLIC_STAGING/);
+    expect(block).toContain('{ name: GATEWAY_ENVIRONMENT_ENV, value: "live" }');
   });
 
   it("builds no cms_admin URL for any role but the admin's", () => {
@@ -206,7 +237,9 @@ describe("aws: the stack (#635)", () => {
 
   it("the gateway's task role reads only the public_role password (review finding)", () => {
     expect(AWS_SERVICE_SECRETS.gateway).toEqual(["public-role-password"]);
-    for (const secret of ADMIN_SECRETS) expect(AWS_SERVICE_SECRETS.gateway).not.toContain(secret);
+    for (const secret of [...ADMIN_SECRETS, ...STAGING_SECRETS]) {
+      expect(AWS_SERVICE_SECRETS.gateway).not.toContain(secret);
+    }
   });
 
   it("each service gets its own task role whose policy is built from that table", () => {