@caelo-cms/provisioning 0.10.68 → 0.10.69

package/src/ops-backend.test.ts

@@ -301,6 +301,62 @@ describe("ensureOpsBackend — new install", () => {
   });
 });
 
+describe("ensureOpsBackend — a just-enabled KMS API (#1060)", () => {
+  const DISABLED =
+    "ERROR: (gcloud.kms.keyrings.describe) PERMISSION_DENIED: Cloud Key Management Service (KMS) API has not been used in project 42 before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/cloudkms.googleapis.com/overview?project=42 then retry. (reason: SERVICE_DISABLED)";
+
+  /** KMS calls answer SERVICE_DISABLED for the first `failures` calls after enabling. */
+  function propagating(failures: number) {
+    const g = fakeGcp();
+    let left = failures;
+    const sleeps: number[] = [];
+    const run = async (args: string[]) => {
+      if (args[0] === "kms" && left > 0) {
+        left--;
+        g.calls.push(args.join(" "));
+        return fail(DISABLED);
+      }
+      return g.run(args);
+    };
+    const sleep = async (ms: number) => {
+      sleeps.push(ms);
+    };
+    return { g, run, sleep, sleeps };
+  }
+
+  it("waits for the enablement to propagate, then creates the key ring and key", async () => {
+    const p = propagating(2);
+    const r = await ensureOpsBackend(TARGET, { run: p.run, sleep: p.sleep });
+    expect(r.ok).toBe(true);
+    expect(p.g.state.ring && p.g.state.key).toBe(true);
+    expect(p.sleeps.length).toBe(2);
+    expect(p.g.calls.filter((c) => c.startsWith("kms keyrings describe")).length).toBe(3);
+  });
+
+  it("fails loudly with the next step when the API never comes up", async () => {
+    const p = propagating(Number.POSITIVE_INFINITY);
+    const r = await ensureOpsBackend(TARGET, { run: p.run, sleep: p.sleep });
+    expect(r.ok).toBe(false);
+    if (r.ok) return;
+    expect(r.error).toContain("Reading key ring caelo failed");
+    expect(r.error).toContain("still reported as disabled");
+    expect(r.error).toContain("gcloud services list --enabled --project=acme");
+    expect(r.error).toContain("re-run the same command");
+    // Waited a few minutes in total, then stopped — no endless loop.
+    const waited = p.sleeps.reduce((a, b) => a + b, 0);
+    expect(waited).toBeGreaterThanOrEqual(3 * 60_000);
+    expect(waited).toBeLessThanOrEqual(5 * 60_000);
+    expect(p.g.state.ring).toBe(false);
+  });
+
+  it("does not wait in a dry run", async () => {
+    const p = propagating(Number.POSITIVE_INFINITY);
+    const r = await ensureOpsBackend(TARGET, { run: p.run, sleep: p.sleep, dryRun: true });
+    expect(r.ok).toBe(true);
+    expect(p.sleeps).toEqual([]);
+  });
+});
+
 describe("cost table (CLAUDE.md §11.C)", () => {
   it("lists the state bucket + KMS key at under $0.10/mo without float noise in the total", () => {
     const est = estimateGcpCost({

package/src/ops-backend.ts

@@ -31,7 +31,15 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { gcloud as defaultGcloud } from "./gcloud.js";
-import { type GcloudRunner, realSleep, runWithRetry, type Sleep } from "./gcloud-retry.js";
+import {
+  type GcloudRunner,
+  realSleep,
+  retryWhileServiceEnabling,
+  runWithRetry,
+  SERVICE_ENABLE_DELAYS_MS,
+  SERVICE_NOT_ENABLED,
+  type Sleep,
+} from "./gcloud-retry.js";
 import { operatorMember } from "./provisioner-identity.js";
 import {
   gcsBackendRecord,
@@ -144,9 +152,16 @@ export async function ensureOpsBackend(
   target: OpsBackendTarget,
   deps: { run?: GcloudRunner; sleep?: Sleep; dryRun?: boolean } = {},
 ): Promise<OpsBackendResult> {
-  const run = deps.run ?? defaultGcloud;
   const sleep = deps.sleep ?? realSleep;
   const dryRun = deps.dryRun ?? false;
+  // `services enable` returns before the enablement has propagated: for a
+  // few minutes the first KMS (or storage) call still answers
+  // SERVICE_DISABLED (#1060). Every call waits that out — on a re-run whose
+  // APIs are long on it costs nothing. A dry run only reads and treats a
+  // disabled API as "not there yet", so it does not wait.
+  const run = dryRun
+    ? (deps.run ?? defaultGcloud)
+    : retryWhileServiceEnabling(deps.run ?? defaultGcloud, sleep);
   const { projectId, region } = target;
   const project = `--project=${projectId}`;
   const bucket = opsBucketName(projectId);
@@ -157,9 +172,12 @@ export async function ensureOpsBackend(
   ];
   const actions: string[] = [];
   const act = (what: string) => actions.push(dryRun ? `would ${what}` : what);
+  const waitedMinutes = Math.round(SERVICE_ENABLE_DELAYS_MS.reduce((a, b) => a + b, 0) / 60_000);
   const fail = (what: string, r: { stderr: string; exitCode: number }): OpsBackendResult => ({
     ok: false,
-    error: `${what} failed: ${reason(r.stderr, r.exitCode)}`,
+    error: SERVICE_NOT_ENABLED.test(r.stderr)
+      ? `${what} failed: the API is still reported as disabled ${waitedMinutes} minutes after it was enabled (${reason(r.stderr, r.exitCode)}). Check that ${OPS_APIS.join(" and ")} show as enabled in project ${projectId} (gcloud services list --enabled --project=${projectId}), then re-run the same command — it picks up where it stopped.`
+      : `${what} failed: ${reason(r.stderr, r.exitCode)}`,
   });
 
   // --- APIs -------------------------------------------------------------

package/src/remote-mcp.test.ts

@@ -284,6 +284,8 @@ describe("planSecretFiles / planRequestTimeout (upgrade)", () => {
     );
     expect(plan.ok && plan.flags).toEqual([
       "--remove-secrets=/secrets/mcp-oauth-client/client.json",
+      "--remove-volume-mount=/secrets/mcp-oauth-client",
+      "--remove-volume=mcp-oauth-client",
       "--update-secrets=/secrets/operator-oauth-client/client.json=caelo-production-operator-oauth-client:latest",
     ]);
     // Converged: nothing left to remove or mount.

package/src/stack-contract.ts

@@ -443,6 +443,12 @@ export const SERVICE_SECRET_FILES: Readonly<Record<CloudRunSlug, readonly Secret
 
 /** A secret file an older contract mounted; upgrade unmounts it in the roll. */
 export interface RetiredSecretFile {
+  /**
+   * The contract name of the secret the file was mounted from (gcp id
+   * `caelo-<env>-<secret>`). Names the volume behind the mount — and a
+   * volume left behind without one (#954) — so the roll removes it too.
+   */
+  readonly secret: string;
   readonly mountDir: string;
   readonly fileName: string;
   readonly why: string;
@@ -450,13 +456,15 @@ export interface RetiredSecretFile {
 
 /**
  * Secret files older contracts mounted that nothing reads any more, per
- * service. The roll removes each mount (`--remove-secrets=<path>`) in the
- * same revision that mounts its replacement.
+ * service. The roll removes each mount (`--remove-secrets=<path>`) and the
+ * volume of that secret (`--remove-volume=<name>`) in the same revision that
+ * mounts its replacement (stack-converge.ts planSecretFiles).
  */
 export const RETIRED_SECRET_FILES: Readonly<Record<CloudRunSlug, readonly RetiredSecretFile[]>> = {
   admin: [],
   gateway: [
     {
+      secret: "mcp-oauth-client",
       mountDir: "/secrets/mcp-oauth-client",
       fileName: "client.json",
       why: "the Google sign-in client is mounted from operator-oauth-client now (#914)",

package/src/stack-converge-secrets.test.ts

@@ -14,13 +14,20 @@
 
 import { describe, expect, it } from "bun:test";
 import { databaseUrlFromEnv } from "@caelo-cms/shared";
-import type { CloudRunSlug } from "./stack-contract.js";
+import type { GcloudResult } from "./gcloud.js";
 import {
+  type CloudRunSlug,
+  OPERATOR_OAUTH_CLIENT_FILE,
+  RETIRED_SECRET_FILES,
+} from "./stack-contract.js";
+import {
+  findVolumesWithMissingSecrets,
   type LiveEnvValue,
   liveContainerEnv,
   liveEnvHasInlinePassword,
   planContractEnv,
   planEnvUpdate,
+  planSecretFiles,
   serviceRollArgs,
 } from "./stack-converge.js";
 
@@ -460,3 +467,132 @@ describe("liveEnvHasInlinePassword", () => {
     expect(liveEnvHasInlinePassword(services.gateway.env)).toBe(false);
   });
 });
+
+describe("retiring a secret file removes its volume too (#954)", () => {
+  // The shape searchviu's gateway was left in: gcloud names a volume it
+  // mints for `--update-secrets` after the secret plus a random suffix.
+  const RETIRED_VOLUME = "caelo-production-mcp-oauth-client-mac-won";
+  const OPERATOR_VOLUME = "caelo-production-operator-oauth-client-puw-tuq";
+  const operatorVolume = {
+    name: OPERATOR_VOLUME,
+    secret: {
+      secretName: "caelo-production-operator-oauth-client",
+      items: [{ key: "latest", path: "client.json" }],
+    },
+  };
+  const retiredVolume = {
+    name: RETIRED_VOLUME,
+    secret: {
+      secretName: "projects/42/secrets/caelo-production-mcp-oauth-client",
+      items: [{ key: "latest", path: "client.json" }],
+    },
+  };
+  const plan = (live: Parameters<typeof planSecretFiles>[0]) =>
+    planSecretFiles(live, [OPERATOR_OAUTH_CLIENT_FILE], "production", RETIRED_SECRET_FILES.gateway);
+
+  it("a retired mount: removes the file, the mount and the volume behind it in the roll", () => {
+    const r = plan({
+      executionEnvironment: null,
+      volumes: [retiredVolume],
+      mounts: [{ name: RETIRED_VOLUME, mountPath: "/secrets/mcp-oauth-client" }],
+    });
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.flags).toContain("--remove-secrets=/secrets/mcp-oauth-client/client.json");
+    expect(r.flags).toContain(`--remove-volume=${RETIRED_VOLUME}`);
+    expect(r.flags).toContain("--remove-volume-mount=/secrets/mcp-oauth-client");
+    expect(r.changes.some((c) => c.startsWith(`volume ${RETIRED_VOLUME}: removed`))).toBe(true);
+  });
+
+  it("an orphaned retired volume (no mount, the searchviu state): removes it and says so", () => {
+    const r = plan({
+      executionEnvironment: null,
+      volumes: [retiredVolume, operatorVolume],
+      mounts: [{ name: OPERATOR_VOLUME, mountPath: "/secrets/operator-oauth-client" }],
+    });
+    expect(r).toEqual({
+      ok: true,
+      flags: [`--remove-volume=${RETIRED_VOLUME}`],
+      changes: [
+        `volume ${RETIRED_VOLUME}: removed (the Google sign-in client is mounted from operator-oauth-client now (#914); nothing mounts it)`,
+      ],
+    });
+  });
+
+  it("a converged service plans nothing", () => {
+    const r = plan({
+      executionEnvironment: null,
+      volumes: [operatorVolume],
+      mounts: [{ name: OPERATOR_VOLUME, mountPath: "/secrets/operator-oauth-client" }],
+    });
+    expect(r).toEqual({ ok: true, flags: [], changes: [] });
+  });
+
+  it("refuses a volume of the retired secret mounted somewhere else instead of pulling it", () => {
+    const r = plan({
+      executionEnvironment: null,
+      volumes: [retiredVolume, operatorVolume],
+      mounts: [
+        { name: OPERATOR_VOLUME, mountPath: "/secrets/operator-oauth-client" },
+        { name: RETIRED_VOLUME, mountPath: "/elsewhere" },
+      ],
+    });
+    expect(r.ok).toBe(false);
+    if (r.ok) return;
+    expect(r.error).toContain(`--remove-volume=${RETIRED_VOLUME}`);
+  });
+
+  describe("preflight: a volume pointing at a deleted secret", () => {
+    const ok = (stdout = ""): GcloudResult => ({ ok: true, stdout, stderr: "", exitCode: 0 });
+    const missing: GcloudResult = {
+      ok: false,
+      stdout: "",
+      stderr:
+        "ERROR: (gcloud.secrets.describe) NOT_FOUND: Secret [projects/42/secrets/x] not found",
+      exitCode: 1,
+    };
+    const where = {
+      serviceName: "caelo-production-gateway-532b35d",
+      projectId: "acme",
+      region: "europe-west1",
+    };
+    const live = {
+      executionEnvironment: null,
+      volumes: [retiredVolume, operatorVolume],
+      mounts: [{ name: OPERATOR_VOLUME, mountPath: "/secrets/operator-oauth-client" }],
+    };
+    const gcloud = (gone: string) => {
+      const calls: string[] = [];
+      const run = async (args: string[]) => {
+        calls.push(args.join(" "));
+        return args[2] === gone ? missing : ok("name");
+      };
+      return { run, calls };
+    };
+
+    it("names the volume and the exact fix", async () => {
+      const g = gcloud("caelo-production-mcp-oauth-client");
+      const problems = await findVolumesWithMissingSecrets(live, where, [], g.run);
+      expect(problems).toHaveLength(1);
+      expect(problems[0]).toContain(`volume "${RETIRED_VOLUME}" (not mounted)`);
+      expect(problems[0]).toContain("caelo-production-mcp-oauth-client, which no longer exists");
+      expect(problems[0]).toContain(
+        `gcloud run services update caelo-production-gateway-532b35d --region=europe-west1 --project=acme --remove-volume=${RETIRED_VOLUME}`,
+      );
+    });
+
+    it("skips a volume the roll removes anyway", async () => {
+      const g = gcloud("caelo-production-mcp-oauth-client");
+      expect(await findVolumesWithMissingSecrets(live, where, [RETIRED_VOLUME], g.run)).toEqual([]);
+      expect(g.calls.some((c) => c.includes("mcp-oauth-client"))).toBe(false);
+    });
+
+    it("a mounted volume's fix removes the mount too", async () => {
+      const g = gcloud("caelo-production-operator-oauth-client");
+      const [problem] = await findVolumesWithMissingSecrets(live, where, [RETIRED_VOLUME], g.run);
+      expect(problem).toContain(
+        `--remove-volume-mount=/secrets/operator-oauth-client --remove-volume=${OPERATOR_VOLUME}`,
+      );
+    });
+  });
+});

package/src/stack-converge.ts

@@ -1023,14 +1023,51 @@ export function planSecretFiles(
   const changes: string[] = [];
   // #914 — mounts an older contract made and nothing reads any more
   // (RETIRED_SECRET_FILES) go in the same revision that mounts their
-  // replacement. `--remove-secrets=<dir>/<file>` drops the file, and the
-  // mount with it when it was the volume's only file (gcloud
-  // `_PruneManagedVolumeMapping`). Repeated `--remove-secrets` /
-  // `--update-secrets` flags merge (gcloud map flags use UpdateAction).
+  // replacement. `--remove-secrets=<dir>/<file>` drops the file and its
+  // mount, but NOT the volume: gcloud's `_PruneManagedVolumeMapping` only
+  // deletes the mount, so the volume stayed behind naming a secret the
+  // upgrade deletes after the roll, and the next revision could not deploy
+  // (#954, searchviu). The plan therefore removes that volume itself, plus
+  // any volume of a retired secret that an earlier roll already orphaned.
+  // Repeated map/list flags merge (gcloud UpdateAction); removing a volume
+  // or mount that is already gone is a no-op in gcloud.
+  const removedVolumes = new Set<string>();
+  const removeVolume = (name: string, why: string) => {
+    if (removedVolumes.has(name)) return;
+    removedVolumes.add(name);
+    flags.push(`--remove-volume=${name}`);
+    changes.push(`volume ${name}: removed (${why})`);
+  };
   for (const old of retired) {
-    if (!live.mounts.some((m) => m.mountPath === old.mountDir)) continue;
-    flags.push(`--remove-secrets=${old.mountDir}/${old.fileName}`);
-    changes.push(`secret file ${old.mountDir}/${old.fileName}: removed (${old.why})`);
+    const retiredId = gcpSecretId(env, old.secret);
+    const mount = live.mounts.find((m) => m.mountPath === old.mountDir);
+    if (mount) {
+      const volume = live.volumes.find((v) => v.name === mount.name);
+      const volumeSecret = bareSecretId(volume?.secret?.secretName);
+      if (volume && volumeSecret !== retiredId) {
+        return {
+          ok: false,
+          error: `the service mounts volume "${mount.name}" (secret ${volumeSecret || "(none)"}) at ${old.mountDir}, where only the retired ${retiredId} was ever mounted. Check what it is; remove it (gcloud run services update <service> --remove-volume-mount=${old.mountDir} --remove-volume=${mount.name}) only if nothing needs it, then re-run upgrade.`,
+        };
+      }
+      flags.push(`--remove-secrets=${old.mountDir}/${old.fileName}`);
+      flags.push(`--remove-volume-mount=${old.mountDir}`);
+      changes.push(`secret file ${old.mountDir}/${old.fileName}: removed (${old.why})`);
+      if (volume) removeVolume(volume.name, old.why);
+    }
+    const mounted = new Set(
+      live.mounts.filter((m) => m.mountPath !== old.mountDir).map((m) => m.name),
+    );
+    for (const v of live.volumes) {
+      if (bareSecretId(v.secret?.secretName) !== retiredId) continue;
+      if (mounted.has(v.name)) {
+        return {
+          ok: false,
+          error: `the service mounts volume "${v.name}" of the retired secret ${retiredId} somewhere other than ${old.mountDir}. Nothing reads that secret any more and upgrade deletes it after the roll; remove the mount and the volume (gcloud run services update <service> --remove-volume-mount=<path> --remove-volume=${v.name}), then re-run upgrade.`,
+        };
+      }
+      removeVolume(v.name, `${old.why}; nothing mounts it`);
+    }
   }
   for (const file of files) {
     const secretId = gcpSecretId(env, file.secret);
@@ -1052,6 +1089,50 @@ export function planSecretFiles(
   return { ok: true, flags, changes };
 }
 
+/**
+ * #954 — the secret volumes of a service whose secret no longer exists in
+ * Secret Manager. Cloud Run refuses every new revision of such a service,
+ * so the upgrade names each one with the exact fix before it starts a roll
+ * instead of failing mid-roll. Volumes in `removing` (the roll's own
+ * `--remove-volume` flags) are skipped: the roll takes them off anyway.
+ * A secret gcloud cannot read for another reason is reported too — the
+ * roll's outcome would be unknowable.
+ */
+export async function findVolumesWithMissingSecrets(
+  live: LiveVolumes,
+  where: { readonly serviceName: string; readonly projectId: string; readonly region: string },
+  removing: readonly string[],
+  run: GcloudRunner,
+): Promise<string[]> {
+  const problems: string[] = [];
+  for (const v of live.volumes) {
+    const secretId = bareSecretId(v.secret?.secretName);
+    if (!secretId || removing.includes(v.name)) continue;
+    const r = await run([
+      "secrets",
+      "describe",
+      secretId,
+      `--project=${where.projectId}`,
+      "--format=value(name)",
+    ]);
+    if (r.ok) continue;
+    const mounts = live.mounts.filter((m) => m.name === v.name).map((m) => m.mountPath);
+    const fix = [
+      `gcloud run services update ${where.serviceName}`,
+      `--region=${where.region}`,
+      `--project=${where.projectId}`,
+      ...mounts.map((p) => `--remove-volume-mount=${p}`),
+      `--remove-volume=${v.name}`,
+    ].join(" ");
+    problems.push(
+      RESOURCE_MISSING.test(r.stderr)
+        ? `${where.serviceName} has volume "${v.name}"${mounts.length > 0 ? ` (mounted at ${mounts.join(", ")})` : " (not mounted)"} pointing at secret ${secretId}, which no longer exists, so Cloud Run would refuse the next revision. If nothing reads it, remove it: ${fix} — then re-run upgrade.`
+        : `could not check secret ${secretId} behind volume "${v.name}" of ${where.serviceName}: ${r.stderr.trim() || `gcloud exited ${r.exitCode}`}`,
+    );
+  }
+  return problems;
+}
+
 /** The request timeout of a described service, in seconds (null: Cloud Run's default, 300). */
 export function liveTimeoutSeconds(serviceJson: string): number | null {
   const svc = JSON.parse(serviceJson) as {

package/src/upgrade/app-roll.ts

@@ -37,6 +37,7 @@ import {
 import {
   type DeployedService,
   type EnvChange,
+  findVolumesWithMissingSecrets,
   type LiveEnvValue,
   type LiveVolumes,
   liveContainerEnv,
@@ -147,6 +148,15 @@ export async function planAppRoll(
   install: AppRollInstall,
   images: Readonly<Record<RollSlug, ReleaseImage>>,
   run: GcloudRunner,
+  checks: {
+    /**
+     * #954 — check that every secret volume's secret still exists. The
+     * operator / converge identity can read secret metadata; the upgrader
+     * job's narrow identity holds no Secret Manager permission at all
+     * (upgrader-setup.ts), so it passes false.
+     */
+    readonly secretVolumes: boolean;
+  },
 ): Promise<PlanResult> {
   const services: ServicePlan[] = [];
   for (const slug of ROLL_ORDER) {
@@ -223,6 +233,25 @@ export async function planAppRoll(
     }
     secretFiles[s.slug] = files;
   }
+  // #954 — a volume pointing at a deleted secret makes Cloud Run refuse
+  // every new revision: name it with its fix now, before any roll starts.
+  const orphaned: string[] = [];
+  for (const s of checks.secretVolumes ? services : []) {
+    const removing = secretFiles[s.slug].flags
+      .filter((f) => f.startsWith("--remove-volume="))
+      .map((f) => f.slice("--remove-volume=".length));
+    orphaned.push(
+      ...(await findVolumesWithMissingSecrets(
+        s.liveVolumes,
+        { serviceName: s.serviceName, projectId: install.projectId, region: install.region },
+        removing,
+        run,
+      )),
+    );
+  }
+  if (orphaned.length > 0) {
+    return { ok: false, error: `a service can't take a new revision: ${orphaned.join(" ")}` };
+  }
   const warnings: string[] = [];
   const rolls = services.map((s): RollPlan => {
     // #553 — the admin runs the Lighthouse quality audit; raise it to the

package/src/upgrade/upgrade.test.ts

@@ -906,7 +906,7 @@ const INSTALL: AppRollInstall = {
 };
 
 async function plannedRoll(run: GcloudRunner): Promise<AppRollPlan> {
-  const planned = await planAppRoll(INSTALL, IMAGES, run);
+  const planned = await planAppRoll(INSTALL, IMAGES, run, { secretVolumes: true });
   if (!planned.ok) throw new Error(planned.error);
   return planned.plan;
 }

package/src/upgrade/upgrade.ts

@@ -177,6 +177,7 @@ export async function upgradeGcpInstallOutcome(
     },
     images.images,
     run,
+    { secretVolumes: true },
   );
   if (!planned.ok) {
     sPre.failed("Pre-flight failed");

package/src/upgrader-job.ts

@@ -519,6 +519,9 @@ export async function runUpgraderJob(deps: UpgraderJobDeps): Promise<UpgraderRun
       },
       images,
       deps.run,
+      // Holds no Secret Manager permission (upgrader-setup.ts); the
+      // converge — operator or converge job — runs the #954 volume check.
+      { secretVolumes: false },
     );
     if (!planned.ok) {
       return stopWith("failed", ["plan"], `Nothing was changed: ${planned.error}.`);