@caelo-cms/provisioning 0.10.67 → 0.10.69

package/src/stack-converge-secrets.test.ts

@@ -14,13 +14,20 @@
 
 import { describe, expect, it } from "bun:test";
 import { databaseUrlFromEnv } from "@caelo-cms/shared";
-import type { CloudRunSlug } from "./stack-contract.js";
+import type { GcloudResult } from "./gcloud.js";
 import {
+  type CloudRunSlug,
+  OPERATOR_OAUTH_CLIENT_FILE,
+  RETIRED_SECRET_FILES,
+} from "./stack-contract.js";
+import {
+  findVolumesWithMissingSecrets,
   type LiveEnvValue,
   liveContainerEnv,
   liveEnvHasInlinePassword,
   planContractEnv,
   planEnvUpdate,
+  planSecretFiles,
   serviceRollArgs,
 } from "./stack-converge.js";
 
@@ -460,3 +467,132 @@ describe("liveEnvHasInlinePassword", () => {
     expect(liveEnvHasInlinePassword(services.gateway.env)).toBe(false);
   });
 });
+
+describe("retiring a secret file removes its volume too (#954)", () => {
+  // The shape searchviu's gateway was left in: gcloud names a volume it
+  // mints for `--update-secrets` after the secret plus a random suffix.
+  const RETIRED_VOLUME = "caelo-production-mcp-oauth-client-mac-won";
+  const OPERATOR_VOLUME = "caelo-production-operator-oauth-client-puw-tuq";
+  const operatorVolume = {
+    name: OPERATOR_VOLUME,
+    secret: {
+      secretName: "caelo-production-operator-oauth-client",
+      items: [{ key: "latest", path: "client.json" }],
+    },
+  };
+  const retiredVolume = {
+    name: RETIRED_VOLUME,
+    secret: {
+      secretName: "projects/42/secrets/caelo-production-mcp-oauth-client",
+      items: [{ key: "latest", path: "client.json" }],
+    },
+  };
+  const plan = (live: Parameters<typeof planSecretFiles>[0]) =>
+    planSecretFiles(live, [OPERATOR_OAUTH_CLIENT_FILE], "production", RETIRED_SECRET_FILES.gateway);
+
+  it("a retired mount: removes the file, the mount and the volume behind it in the roll", () => {
+    const r = plan({
+      executionEnvironment: null,
+      volumes: [retiredVolume],
+      mounts: [{ name: RETIRED_VOLUME, mountPath: "/secrets/mcp-oauth-client" }],
+    });
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.flags).toContain("--remove-secrets=/secrets/mcp-oauth-client/client.json");
+    expect(r.flags).toContain(`--remove-volume=${RETIRED_VOLUME}`);
+    expect(r.flags).toContain("--remove-volume-mount=/secrets/mcp-oauth-client");
+    expect(r.changes.some((c) => c.startsWith(`volume ${RETIRED_VOLUME}: removed`))).toBe(true);
+  });
+
+  it("an orphaned retired volume (no mount, the searchviu state): removes it and says so", () => {
+    const r = plan({
+      executionEnvironment: null,
+      volumes: [retiredVolume, operatorVolume],
+      mounts: [{ name: OPERATOR_VOLUME, mountPath: "/secrets/operator-oauth-client" }],
+    });
+    expect(r).toEqual({
+      ok: true,
+      flags: [`--remove-volume=${RETIRED_VOLUME}`],
+      changes: [
+        `volume ${RETIRED_VOLUME}: removed (the Google sign-in client is mounted from operator-oauth-client now (#914); nothing mounts it)`,
+      ],
+    });
+  });
+
+  it("a converged service plans nothing", () => {
+    const r = plan({
+      executionEnvironment: null,
+      volumes: [operatorVolume],
+      mounts: [{ name: OPERATOR_VOLUME, mountPath: "/secrets/operator-oauth-client" }],
+    });
+    expect(r).toEqual({ ok: true, flags: [], changes: [] });
+  });
+
+  it("refuses a volume of the retired secret mounted somewhere else instead of pulling it", () => {
+    const r = plan({
+      executionEnvironment: null,
+      volumes: [retiredVolume, operatorVolume],
+      mounts: [
+        { name: OPERATOR_VOLUME, mountPath: "/secrets/operator-oauth-client" },
+        { name: RETIRED_VOLUME, mountPath: "/elsewhere" },
+      ],
+    });
+    expect(r.ok).toBe(false);
+    if (r.ok) return;
+    expect(r.error).toContain(`--remove-volume=${RETIRED_VOLUME}`);
+  });
+
+  describe("preflight: a volume pointing at a deleted secret", () => {
+    const ok = (stdout = ""): GcloudResult => ({ ok: true, stdout, stderr: "", exitCode: 0 });
+    const missing: GcloudResult = {
+      ok: false,
+      stdout: "",
+      stderr:
+        "ERROR: (gcloud.secrets.describe) NOT_FOUND: Secret [projects/42/secrets/x] not found",
+      exitCode: 1,
+    };
+    const where = {
+      serviceName: "caelo-production-gateway-532b35d",
+      projectId: "acme",
+      region: "europe-west1",
+    };
+    const live = {
+      executionEnvironment: null,
+      volumes: [retiredVolume, operatorVolume],
+      mounts: [{ name: OPERATOR_VOLUME, mountPath: "/secrets/operator-oauth-client" }],
+    };
+    const gcloud = (gone: string) => {
+      const calls: string[] = [];
+      const run = async (args: string[]) => {
+        calls.push(args.join(" "));
+        return args[2] === gone ? missing : ok("name");
+      };
+      return { run, calls };
+    };
+
+    it("names the volume and the exact fix", async () => {
+      const g = gcloud("caelo-production-mcp-oauth-client");
+      const problems = await findVolumesWithMissingSecrets(live, where, [], g.run);
+      expect(problems).toHaveLength(1);
+      expect(problems[0]).toContain(`volume "${RETIRED_VOLUME}" (not mounted)`);
+      expect(problems[0]).toContain("caelo-production-mcp-oauth-client, which no longer exists");
+      expect(problems[0]).toContain(
+        `gcloud run services update caelo-production-gateway-532b35d --region=europe-west1 --project=acme --remove-volume=${RETIRED_VOLUME}`,
+      );
+    });
+
+    it("skips a volume the roll removes anyway", async () => {
+      const g = gcloud("caelo-production-mcp-oauth-client");
+      expect(await findVolumesWithMissingSecrets(live, where, [RETIRED_VOLUME], g.run)).toEqual([]);
+      expect(g.calls.some((c) => c.includes("mcp-oauth-client"))).toBe(false);
+    });
+
+    it("a mounted volume's fix removes the mount too", async () => {
+      const g = gcloud("caelo-production-operator-oauth-client");
+      const [problem] = await findVolumesWithMissingSecrets(live, where, [RETIRED_VOLUME], g.run);
+      expect(problem).toContain(
+        `--remove-volume-mount=/secrets/operator-oauth-client --remove-volume=${OPERATOR_VOLUME}`,
+      );
+    });
+  });
+});

package/src/stack-converge.ts

@@ -1023,14 +1023,51 @@ export function planSecretFiles(
   const changes: string[] = [];
   // #914 — mounts an older contract made and nothing reads any more
   // (RETIRED_SECRET_FILES) go in the same revision that mounts their
-  // replacement. `--remove-secrets=<dir>/<file>` drops the file, and the
-  // mount with it when it was the volume's only file (gcloud
-  // `_PruneManagedVolumeMapping`). Repeated `--remove-secrets` /
-  // `--update-secrets` flags merge (gcloud map flags use UpdateAction).
+  // replacement. `--remove-secrets=<dir>/<file>` drops the file and its
+  // mount, but NOT the volume: gcloud's `_PruneManagedVolumeMapping` only
+  // deletes the mount, so the volume stayed behind naming a secret the
+  // upgrade deletes after the roll, and the next revision could not deploy
+  // (#954, searchviu). The plan therefore removes that volume itself, plus
+  // any volume of a retired secret that an earlier roll already orphaned.
+  // Repeated map/list flags merge (gcloud UpdateAction); removing a volume
+  // or mount that is already gone is a no-op in gcloud.
+  const removedVolumes = new Set<string>();
+  const removeVolume = (name: string, why: string) => {
+    if (removedVolumes.has(name)) return;
+    removedVolumes.add(name);
+    flags.push(`--remove-volume=${name}`);
+    changes.push(`volume ${name}: removed (${why})`);
+  };
   for (const old of retired) {
-    if (!live.mounts.some((m) => m.mountPath === old.mountDir)) continue;
-    flags.push(`--remove-secrets=${old.mountDir}/${old.fileName}`);
-    changes.push(`secret file ${old.mountDir}/${old.fileName}: removed (${old.why})`);
+    const retiredId = gcpSecretId(env, old.secret);
+    const mount = live.mounts.find((m) => m.mountPath === old.mountDir);
+    if (mount) {
+      const volume = live.volumes.find((v) => v.name === mount.name);
+      const volumeSecret = bareSecretId(volume?.secret?.secretName);
+      if (volume && volumeSecret !== retiredId) {
+        return {
+          ok: false,
+          error: `the service mounts volume "${mount.name}" (secret ${volumeSecret || "(none)"}) at ${old.mountDir}, where only the retired ${retiredId} was ever mounted. Check what it is; remove it (gcloud run services update <service> --remove-volume-mount=${old.mountDir} --remove-volume=${mount.name}) only if nothing needs it, then re-run upgrade.`,
+        };
+      }
+      flags.push(`--remove-secrets=${old.mountDir}/${old.fileName}`);
+      flags.push(`--remove-volume-mount=${old.mountDir}`);
+      changes.push(`secret file ${old.mountDir}/${old.fileName}: removed (${old.why})`);
+      if (volume) removeVolume(volume.name, old.why);
+    }
+    const mounted = new Set(
+      live.mounts.filter((m) => m.mountPath !== old.mountDir).map((m) => m.name),
+    );
+    for (const v of live.volumes) {
+      if (bareSecretId(v.secret?.secretName) !== retiredId) continue;
+      if (mounted.has(v.name)) {
+        return {
+          ok: false,
+          error: `the service mounts volume "${v.name}" of the retired secret ${retiredId} somewhere other than ${old.mountDir}. Nothing reads that secret any more and upgrade deletes it after the roll; remove the mount and the volume (gcloud run services update <service> --remove-volume-mount=<path> --remove-volume=${v.name}), then re-run upgrade.`,
+        };
+      }
+      removeVolume(v.name, `${old.why}; nothing mounts it`);
+    }
   }
   for (const file of files) {
     const secretId = gcpSecretId(env, file.secret);
@@ -1052,6 +1089,50 @@ export function planSecretFiles(
   return { ok: true, flags, changes };
 }
 
+/**
+ * #954 — the secret volumes of a service whose secret no longer exists in
+ * Secret Manager. Cloud Run refuses every new revision of such a service,
+ * so the upgrade names each one with the exact fix before it starts a roll
+ * instead of failing mid-roll. Volumes in `removing` (the roll's own
+ * `--remove-volume` flags) are skipped: the roll takes them off anyway.
+ * A secret gcloud cannot read for another reason is reported too — the
+ * roll's outcome would be unknowable.
+ */
+export async function findVolumesWithMissingSecrets(
+  live: LiveVolumes,
+  where: { readonly serviceName: string; readonly projectId: string; readonly region: string },
+  removing: readonly string[],
+  run: GcloudRunner,
+): Promise<string[]> {
+  const problems: string[] = [];
+  for (const v of live.volumes) {
+    const secretId = bareSecretId(v.secret?.secretName);
+    if (!secretId || removing.includes(v.name)) continue;
+    const r = await run([
+      "secrets",
+      "describe",
+      secretId,
+      `--project=${where.projectId}`,
+      "--format=value(name)",
+    ]);
+    if (r.ok) continue;
+    const mounts = live.mounts.filter((m) => m.name === v.name).map((m) => m.mountPath);
+    const fix = [
+      `gcloud run services update ${where.serviceName}`,
+      `--region=${where.region}`,
+      `--project=${where.projectId}`,
+      ...mounts.map((p) => `--remove-volume-mount=${p}`),
+      `--remove-volume=${v.name}`,
+    ].join(" ");
+    problems.push(
+      RESOURCE_MISSING.test(r.stderr)
+        ? `${where.serviceName} has volume "${v.name}"${mounts.length > 0 ? ` (mounted at ${mounts.join(", ")})` : " (not mounted)"} pointing at secret ${secretId}, which no longer exists, so Cloud Run would refuse the next revision. If nothing reads it, remove it: ${fix} — then re-run upgrade.`
+        : `could not check secret ${secretId} behind volume "${v.name}" of ${where.serviceName}: ${r.stderr.trim() || `gcloud exited ${r.exitCode}`}`,
+    );
+  }
+  return problems;
+}
+
 /** The request timeout of a described service, in seconds (null: Cloud Run's default, 300). */
 export function liveTimeoutSeconds(serviceJson: string): number | null {
   const svc = JSON.parse(serviceJson) as {

package/src/upgrade/app-roll.ts

@@ -37,6 +37,7 @@ import {
 import {
   type DeployedService,
   type EnvChange,
+  findVolumesWithMissingSecrets,
   type LiveEnvValue,
   type LiveVolumes,
   liveContainerEnv,
@@ -147,6 +148,15 @@ export async function planAppRoll(
   install: AppRollInstall,
   images: Readonly<Record<RollSlug, ReleaseImage>>,
   run: GcloudRunner,
+  checks: {
+    /**
+     * #954 — check that every secret volume's secret still exists. The
+     * operator / converge identity can read secret metadata; the upgrader
+     * job's narrow identity holds no Secret Manager permission at all
+     * (upgrader-setup.ts), so it passes false.
+     */
+    readonly secretVolumes: boolean;
+  },
 ): Promise<PlanResult> {
   const services: ServicePlan[] = [];
   for (const slug of ROLL_ORDER) {
@@ -223,6 +233,25 @@ export async function planAppRoll(
     }
     secretFiles[s.slug] = files;
   }
+  // #954 — a volume pointing at a deleted secret makes Cloud Run refuse
+  // every new revision: name it with its fix now, before any roll starts.
+  const orphaned: string[] = [];
+  for (const s of checks.secretVolumes ? services : []) {
+    const removing = secretFiles[s.slug].flags
+      .filter((f) => f.startsWith("--remove-volume="))
+      .map((f) => f.slice("--remove-volume=".length));
+    orphaned.push(
+      ...(await findVolumesWithMissingSecrets(
+        s.liveVolumes,
+        { serviceName: s.serviceName, projectId: install.projectId, region: install.region },
+        removing,
+        run,
+      )),
+    );
+  }
+  if (orphaned.length > 0) {
+    return { ok: false, error: `a service can't take a new revision: ${orphaned.join(" ")}` };
+  }
   const warnings: string[] = [];
   const rolls = services.map((s): RollPlan => {
     // #553 — the admin runs the Lighthouse quality audit; raise it to the

package/src/upgrade/upgrade.test.ts

@@ -906,7 +906,7 @@ const INSTALL: AppRollInstall = {
 };
 
 async function plannedRoll(run: GcloudRunner): Promise<AppRollPlan> {
-  const planned = await planAppRoll(INSTALL, IMAGES, run);
+  const planned = await planAppRoll(INSTALL, IMAGES, run, { secretVolumes: true });
   if (!planned.ok) throw new Error(planned.error);
   return planned.plan;
 }

package/src/upgrade/upgrade.ts

@@ -177,6 +177,7 @@ export async function upgradeGcpInstallOutcome(
     },
     images.images,
     run,
+    { secretVolumes: true },
   );
   if (!planned.ok) {
     sPre.failed("Pre-flight failed");

package/src/upgrader-job.ts

@@ -519,6 +519,9 @@ export async function runUpgraderJob(deps: UpgraderJobDeps): Promise<UpgraderRun
       },
       images,
       deps.run,
+      // Holds no Secret Manager permission (upgrader-setup.ts); the
+      // converge — operator or converge job — runs the #954 volume check.
+      { secretVolumes: false },
     );
     if (!planned.ok) {
       return stopWith("failed", ["plan"], `Nothing was changed: ${planned.error}.`);