@caelo-cms/provisioning 0.10.68 → 0.10.70

package/src/adopt-existing.ts

@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * Adopt cloud resources the stack declares but its Pulumi state lacks
+ * (#1079), before the installer's `pulumi up`.
+ *
+ * `cms-provision upgrade` converges an install through gcloud and never runs
+ * Pulumi (stack-converge.ts), so it creates some resources the stack also
+ * declares — the gateway's service account (runtime-secrets.ts), the MCP
+ * service account (mcp-iap.ts) — without them entering the state. The next
+ * installer run then plans to CREATE them, and `state migrate` refuses an
+ * install whose preview plans anything. So, after the preview and before
+ * the review:
+ *
+ *   1. {@link adoptionCandidates}: the planned creates of a kind in
+ *      {@link ADOPTABLE_KINDS} whose inputs name the cloud resource. Only
+ *      planned steps are looked at, so nothing the stack does not declare is
+ *      ever adopted; every other kind is left to Pulumi.
+ *   2. {@link existingCandidates}: a gcloud describe per candidate. Exists →
+ *      adopt; NOT_FOUND → a plain create; anything else stops the run (we
+ *      can't tell, and guessing either way is wrong).
+ *   3. `pulumi import` of the existing ones into the stack, under the
+ *      stack's explicit GCP provider (wizards/gcp-pulumi.ts). State only: the
+ *      cloud resources are not touched.
+ *   4. A second preview: what the review shows and `up` applies. An adopted
+ *      resource that is still planned as a create stops the run.
+ *
+ * Why only service accounts ({@link ADOPTABLE_KINDS}). A kind is adopted
+ * when the stack declares it AND the CLI also creates it through gcloud, and
+ * its name is unique in the project. Checked per kind against both stacks:
+ *
+ *   - Secret Manager secrets: the stack declares only `postgres-password` and
+ *     `secret-kek`, which the CLI never creates; the secrets the CLI makes
+ *     (stack-contract.ts CLI_GENERATED_SECRETS) the stack only references by
+ *     id. Adopting a secret without its value resource would also make `up`
+ *     add a version with a freshly generated value — a new KEK.
+ *   - Buckets, Cloud Run jobs, custom roles: the stack's buckets are made by
+ *     Pulumi alone; the CLI's own (ops, audit results, the upgrader, converge,
+ *     audit and operator-access jobs, their custom roles) are not declared.
+ *   - The admin's Cloud Run domain mapping: owned by `admin-domain enable`
+ *     (admin-domain.ts), the stack's copy stays off; Pulumi, running as the
+ *     provisioner, could not manage it.
+ *   - IAM member resources (`*IamMember`, `IAMMember`): non-authoritative.
+ *     Their create is a read-modify-write that merges the member into the
+ *     policy and succeeds when it is there already, so a planned create of
+ *     one is an adoption already.
+ *   - Cloud SQL users: the CLI's are IAM users named after service accounts,
+ *     the stack's are the built-in `admin_role` / `public_role`.
+ */
+
+import type { GcloudRunner } from "./gcloud-retry.js";
+import type { PlannedStep } from "./preview-plan.js";
+
+/** The cloud identity of a resource a planned create would make. */
+export interface AdoptionTarget {
+  /** How the operator knows it, e.g. the service account's email. */
+  readonly display: string;
+  /** gcloud args that succeed exactly when it exists. */
+  readonly describeArgs: readonly string[];
+  /** The id `pulumi import` takes for the kind. */
+  readonly importId: string;
+}
+
+/** A resource kind the installer adopts when it exists already. */
+export interface AdoptableKind {
+  /** What it is, for messages (`service account`). */
+  readonly label: string;
+  /**
+   * The resource a create with these inputs would make, or null when the
+   * inputs don't pin it down (then the create is left to Pulumi).
+   */
+  target(inputs: Readonly<Record<string, unknown>>, projectId: string): AdoptionTarget | null;
+}
+
+/** Kinds adopted before `up`, by Pulumi type token. See the file comment for the rest. */
+export const ADOPTABLE_KINDS: Readonly<Record<string, AdoptableKind>> = {
+  "gcp:serviceaccount/account:Account": {
+    label: "service account",
+    target(inputs, projectId) {
+      // GCP's account-id format; also keeps out the engine's placeholder
+      // for a value only known after apply.
+      const accountId = inputs.accountId;
+      if (typeof accountId !== "string" || !/^[a-z][a-z0-9-]{4,28}[a-z0-9]$/.test(accountId)) {
+        return null;
+      }
+      // Unset means the provider's project, which is the install's
+      // (assertStackProject). Another project is never adopted.
+      const project = inputs.project ?? projectId;
+      if (project !== projectId) return null;
+      const email = `${accountId}@${projectId}.iam.gserviceaccount.com`;
+      return {
+        display: email,
+        describeArgs: [
+          "iam",
+          "service-accounts",
+          "describe",
+          email,
+          `--project=${projectId}`,
+          "--format=value(email)",
+        ],
+        importId: `projects/${projectId}/serviceAccounts/${email}`,
+      };
+    },
+  },
+};
+
+/** A planned create the installer may adopt instead. */
+export interface AdoptionCandidate extends AdoptionTarget {
+  readonly step: PlannedStep;
+  readonly label: string;
+}
+
+/** Thrown when adoption cannot finish; nothing was applied. */
+export class AdoptionError extends Error {
+  override readonly name = "AdoptionError";
+}
+
+/** The planned creates of an adoptable kind, with what each would create. */
+export function adoptionCandidates(
+  steps: readonly PlannedStep[],
+  projectId: string,
+): AdoptionCandidate[] {
+  const out: AdoptionCandidate[] = [];
+  for (const step of steps) {
+    if (step.op !== "create") continue;
+    const kind = ADOPTABLE_KINDS[step.type];
+    if (!kind || !step.inputs) continue;
+    const target = kind.target(step.inputs, projectId);
+    if (target) out.push({ ...target, step, label: kind.label });
+  }
+  return out;
+}
+
+const NOT_FOUND = /NOT_FOUND|not found|does not exist/i;
+
+/**
+ * The candidates that exist in the project.
+ *
+ * @throws AdoptionError when a describe fails for another reason than NOT_FOUND
+ */
+export async function existingCandidates(
+  candidates: readonly AdoptionCandidate[],
+  run: GcloudRunner,
+): Promise<AdoptionCandidate[]> {
+  const existing: AdoptionCandidate[] = [];
+  for (const c of candidates) {
+    const r = await run([...c.describeArgs]);
+    if (r.ok) existing.push(c);
+    else if (!NOT_FOUND.test(r.stderr)) {
+      throw new AdoptionError(
+        `could not tell whether the ${c.label} ${c.display} exists (gcloud: ${r.stderr.trim()}). Nothing was imported or applied. Check that gcloud is signed in as the Owner (gcloud auth login) and re-run the installer.`,
+      );
+    }
+  }
+  return existing;
+}
+
+/** What adoption needs from Pulumi and gcloud; fakes in tests. */
+export interface AdoptionDeps {
+  readonly projectId: string;
+  readonly gcloud: GcloudRunner;
+  /** A fresh preview of the stack. */
+  preview(): Promise<readonly PlannedStep[]>;
+  /** `pulumi import` the candidates into the stack (state only). */
+  importResources(candidates: readonly AdoptionCandidate[]): Promise<void>;
+}
+
+/** The plan to review after adoption, and what was adopted into it. */
+export interface AdoptionResult {
+  readonly adopted: readonly AdoptionCandidate[];
+  readonly steps: readonly PlannedStep[];
+}
+
+/**
+ * The line of a failed Pulumi call that says what went wrong. The
+ * Automation API's error starts with `code: 255` and carries the CLI's
+ * stderr below; its first `error:` line is the reason.
+ */
+export function pulumiErrorReason(e: unknown): string {
+  const lines = (e instanceof Error ? e.message : String(e))
+    .split("\n")
+    .map((l) => l.trim().replace(/^stderr:\s*/, ""))
+    .filter((l) => l !== "");
+  return lines.find((l) => /^error:/i.test(l)) ?? lines[0] ?? "no output";
+}
+
+/**
+ * Adopt the planned creates of `steps` that exist in the project already,
+ * and return the plan after it. With nothing to adopt, `steps` comes back
+ * unchanged and nothing else runs.
+ *
+ * @throws AdoptionError when a probe or the import fails, or a re-plan still
+ *   creates an adopted resource; `up` must not run then
+ */
+export async function adoptExistingResources(
+  steps: readonly PlannedStep[],
+  deps: AdoptionDeps,
+): Promise<AdoptionResult> {
+  const existing = await existingCandidates(adoptionCandidates(steps, deps.projectId), deps.gcloud);
+  if (existing.length === 0) return { adopted: [], steps };
+  const names = existing.map((c) => `${c.label} ${c.display}`).join(", ");
+  try {
+    await deps.importResources(existing);
+  } catch (e) {
+    throw new AdoptionError(
+      `could not adopt ${names} (already in the project) into the stack: pulumi import says "${pulumiErrorReason(e)}". Nothing was applied. Re-run the installer: what was adopted stays adopted. If it fails the same way, report it with this message.`,
+    );
+  }
+  const after = await deps.preview();
+  const stillCreated = after.filter(
+    (s) => s.op === "create" && existing.some((c) => c.step.urn === s.urn),
+  );
+  if (stillCreated.length > 0) {
+    throw new AdoptionError(
+      `imported ${names}, but the next preview still creates ${stillCreated.map((s) => s.name).join(", ")}. Nothing was applied. Report this with the message; re-running the installer does not fix it.`,
+    );
+  }
+  return { adopted: existing, steps: after };
+}
+
+/** The adopted resources as lines for the review, or none. */
+export function formatAdopted(adopted: readonly AdoptionCandidate[]): string[] {
+  if (adopted.length === 0) return [];
+  return [
+    `Adopted (${adopted.length}) — already in the project, now in the stack instead of created again:`,
+    ...adopted.map((c) => `  ${c.label} ${c.display} (${c.step.name})`),
+  ];
+}

package/src/cli.ts

@@ -1500,6 +1500,28 @@ async function domainMappingCommand(kind: "admin" | "staging"): Promise<void> {
     // Prints the staging basic-auth password ON PURPOSE (#678): the operator
     // asked for it to be shown once; it is never written to disk or logs.
     console.log(describeStagingApply(applied.value).join("\n"));
+    // #1058 — and the admin's CAELO_STAGING_HOST, so the provisioned value
+    // agrees with install.json and the stored host (the converge job
+    // refuses an install whose records and admin disagree). After the
+    // provision call: the update starts a new admin revision.
+    const { setAdminStagingHostEnv } = await import("./admin-domain.js");
+    const env = await setAdminStagingHostEnv({
+      projectId: meta.projectId,
+      region: meta.region,
+      stagingHost: r.hostname,
+    });
+    if (env.status === "failed") {
+      console.error(
+        `${command}: ${r.hostname} is mapped and stored in the admin, but its CAELO_STAGING_HOST could not be updated: ${env.error}\n` +
+          `Run \`${retry}\` again once that is fixed (it is idempotent).`,
+      );
+      process.exit(5);
+    }
+    if (env.status === "set") {
+      console.log(
+        `The admin (${env.service}) now runs with CAELO_STAGING_HOST=${r.hostname}${env.previous ? ` (was ${env.previous})` : ""}.`,
+      );
+    }
   }
   if (r.records.length === 0) {
     console.log(

package/src/default-firewall.ts

@@ -30,7 +30,7 @@
  */
 
 import { gcloud as defaultGcloud } from "./gcloud.js";
-import type { GcloudRunner } from "./gcloud-retry.js";
+import { type GcloudRunner, SERVICE_NOT_ENABLED } from "./gcloud-retry.js";
 import {
   ABSENT_DEFAULT_FIREWALL_RULES,
   type AbsentDefaultFirewallRule,
@@ -65,8 +65,10 @@ interface LiveInstance {
  * gcloud's error when Compute Engine is not enabled on the project. Without
  * Compute Engine there is no `default` network, so there is nothing to remove.
  */
-const COMPUTE_DISABLED =
-  /SERVICE_DISABLED|accessNotConfigured|has not been used in project|API \[compute\.googleapis\.com\] not enabled|compute\.googleapis\.com.*(?:disabled|not enabled)/i;
+const COMPUTE_DISABLED = new RegExp(
+  `${SERVICE_NOT_ENABLED.source}|API \\[compute\\.googleapis\\.com\\] not enabled|compute\\.googleapis\\.com.*(?:disabled|not enabled)`,
+  "i",
+);
 
 /** gcloud's stderr when an aggregated list succeeded for only some zones. */
 const PARTIAL_LIST = /did not succeed|unreachable|did not respond|could not fetch/i;

package/src/gcloud-retry.ts

@@ -39,3 +39,40 @@ export async function runWithRetry(
   }
   return r;
 }
+
+/**
+ * gcloud's wording for a call against an API that is not enabled on the
+ * project — also for the few minutes after `services enable` returned,
+ * until the enablement has propagated ("Cloud KMS API has not been used in
+ * project N before or it is disabled … (reason: SERVICE_DISABLED)", #1060).
+ * Shared with default-firewall.ts, which reads it as "Compute Engine is off".
+ */
+export const SERVICE_NOT_ENABLED =
+  /SERVICE_DISABLED|accessNotConfigured|has not been used in project|it is disabled/i;
+
+/**
+ * Backoff while a just-enabled API propagates: ~3.5 minutes in total, the
+ * window Google documents for enablement to reach every endpoint.
+ */
+export const SERVICE_ENABLE_DELAYS_MS: readonly number[] = [
+  5_000, 10_000, 20_000, 30_000, 30_000, 30_000, 30_000, 30_000, 30_000,
+];
+
+/**
+ * Wrap `run` so a call that fails with {@link SERVICE_NOT_ENABLED} is
+ * retried with {@link SERVICE_ENABLE_DELAYS_MS} backoff. Every other result
+ * (success or a different failure) returns at once; when the backoff is
+ * exhausted the last failure is returned unchanged, so the caller can still
+ * tell the operator that the API never came up.
+ */
+export function retryWhileServiceEnabling(run: GcloudRunner, sleep: Sleep): GcloudRunner {
+  return async (args, opts) => {
+    let r = await run(args, opts);
+    for (const delay of SERVICE_ENABLE_DELAYS_MS) {
+      if (r.ok || !SERVICE_NOT_ENABLED.test(r.stderr)) break;
+      await sleep(delay);
+      r = await run(args, opts);
+    }
+    return r;
+  };
+}

package/src/ops-backend.test.ts

@@ -301,6 +301,62 @@ describe("ensureOpsBackend — new install", () => {
   });
 });
 
+describe("ensureOpsBackend — a just-enabled KMS API (#1060)", () => {
+  const DISABLED =
+    "ERROR: (gcloud.kms.keyrings.describe) PERMISSION_DENIED: Cloud Key Management Service (KMS) API has not been used in project 42 before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/cloudkms.googleapis.com/overview?project=42 then retry. (reason: SERVICE_DISABLED)";
+
+  /** KMS calls answer SERVICE_DISABLED for the first `failures` calls after enabling. */
+  function propagating(failures: number) {
+    const g = fakeGcp();
+    let left = failures;
+    const sleeps: number[] = [];
+    const run = async (args: string[]) => {
+      if (args[0] === "kms" && left > 0) {
+        left--;
+        g.calls.push(args.join(" "));
+        return fail(DISABLED);
+      }
+      return g.run(args);
+    };
+    const sleep = async (ms: number) => {
+      sleeps.push(ms);
+    };
+    return { g, run, sleep, sleeps };
+  }
+
+  it("waits for the enablement to propagate, then creates the key ring and key", async () => {
+    const p = propagating(2);
+    const r = await ensureOpsBackend(TARGET, { run: p.run, sleep: p.sleep });
+    expect(r.ok).toBe(true);
+    expect(p.g.state.ring && p.g.state.key).toBe(true);
+    expect(p.sleeps.length).toBe(2);
+    expect(p.g.calls.filter((c) => c.startsWith("kms keyrings describe")).length).toBe(3);
+  });
+
+  it("fails loudly with the next step when the API never comes up", async () => {
+    const p = propagating(Number.POSITIVE_INFINITY);
+    const r = await ensureOpsBackend(TARGET, { run: p.run, sleep: p.sleep });
+    expect(r.ok).toBe(false);
+    if (r.ok) return;
+    expect(r.error).toContain("Reading key ring caelo failed");
+    expect(r.error).toContain("still reported as disabled");
+    expect(r.error).toContain("gcloud services list --enabled --project=acme");
+    expect(r.error).toContain("re-run the same command");
+    // Waited a few minutes in total, then stopped — no endless loop.
+    const waited = p.sleeps.reduce((a, b) => a + b, 0);
+    expect(waited).toBeGreaterThanOrEqual(3 * 60_000);
+    expect(waited).toBeLessThanOrEqual(5 * 60_000);
+    expect(p.g.state.ring).toBe(false);
+  });
+
+  it("does not wait in a dry run", async () => {
+    const p = propagating(Number.POSITIVE_INFINITY);
+    const r = await ensureOpsBackend(TARGET, { run: p.run, sleep: p.sleep, dryRun: true });
+    expect(r.ok).toBe(true);
+    expect(p.sleeps).toEqual([]);
+  });
+});
+
 describe("cost table (CLAUDE.md §11.C)", () => {
   it("lists the state bucket + KMS key at under $0.10/mo without float noise in the total", () => {
     const est = estimateGcpCost({

package/src/ops-backend.ts

@@ -31,7 +31,15 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { gcloud as defaultGcloud } from "./gcloud.js";
-import { type GcloudRunner, realSleep, runWithRetry, type Sleep } from "./gcloud-retry.js";
+import {
+  type GcloudRunner,
+  realSleep,
+  retryWhileServiceEnabling,
+  runWithRetry,
+  SERVICE_ENABLE_DELAYS_MS,
+  SERVICE_NOT_ENABLED,
+  type Sleep,
+} from "./gcloud-retry.js";
 import { operatorMember } from "./provisioner-identity.js";
 import {
   gcsBackendRecord,
@@ -144,9 +152,16 @@ export async function ensureOpsBackend(
   target: OpsBackendTarget,
   deps: { run?: GcloudRunner; sleep?: Sleep; dryRun?: boolean } = {},
 ): Promise<OpsBackendResult> {
-  const run = deps.run ?? defaultGcloud;
   const sleep = deps.sleep ?? realSleep;
   const dryRun = deps.dryRun ?? false;
+  // `services enable` returns before the enablement has propagated: for a
+  // few minutes the first KMS (or storage) call still answers
+  // SERVICE_DISABLED (#1060). Every call waits that out — on a re-run whose
+  // APIs are long on it costs nothing. A dry run only reads and treats a
+  // disabled API as "not there yet", so it does not wait.
+  const run = dryRun
+    ? (deps.run ?? defaultGcloud)
+    : retryWhileServiceEnabling(deps.run ?? defaultGcloud, sleep);
   const { projectId, region } = target;
   const project = `--project=${projectId}`;
   const bucket = opsBucketName(projectId);
@@ -157,9 +172,12 @@ export async function ensureOpsBackend(
   ];
   const actions: string[] = [];
   const act = (what: string) => actions.push(dryRun ? `would ${what}` : what);
+  const waitedMinutes = Math.round(SERVICE_ENABLE_DELAYS_MS.reduce((a, b) => a + b, 0) / 60_000);
   const fail = (what: string, r: { stderr: string; exitCode: number }): OpsBackendResult => ({
     ok: false,
-    error: `${what} failed: ${reason(r.stderr, r.exitCode)}`,
+    error: SERVICE_NOT_ENABLED.test(r.stderr)
+      ? `${what} failed: the API is still reported as disabled ${waitedMinutes} minutes after it was enabled (${reason(r.stderr, r.exitCode)}). Check that ${OPS_APIS.join(" and ")} show as enabled in project ${projectId} (gcloud services list --enabled --project=${projectId}), then re-run the same command — it picks up where it stopped.`
+      : `${what} failed: ${reason(r.stderr, r.exitCode)}`,
   });
 
   // --- APIs -------------------------------------------------------------

package/src/preview-plan.ts

@@ -31,6 +31,13 @@ export interface PlannedStep {
   readonly diffs: readonly string[];
   /** Properties that force the replace (replace steps only). */
   readonly replaceReasons: readonly string[];
+  /**
+   * The inputs the program declares, for create steps only (#1079: they name
+   * the cloud resource a create would make, e.g. a service account's
+   * `accountId`, so adopt-existing.ts can tell whether it exists already).
+   * Secret values arrive masked by the engine. Never written to a plan file.
+   */
+  readonly inputs?: Readonly<Record<string, unknown>>;
 }
 
 /**
@@ -116,6 +123,7 @@ export function collectPreviewSteps(): PreviewCollector {
         urn: m.urn,
         diffs: [...(m.diffs ?? [])],
         replaceReasons: op === "replace" ? [...(m.keys ?? [])] : [],
+        ...(op === "create" && m.new?.inputs ? { inputs: { ...m.new.inputs } } : {}),
       });
     },
     steps: () => [...byUrn.values()],

package/src/remote-mcp.test.ts

@@ -284,6 +284,8 @@ describe("planSecretFiles / planRequestTimeout (upgrade)", () => {
     );
     expect(plan.ok && plan.flags).toEqual([
       "--remove-secrets=/secrets/mcp-oauth-client/client.json",
+      "--remove-volume-mount=/secrets/mcp-oauth-client",
+      "--remove-volume=mcp-oauth-client",
       "--update-secrets=/secrets/operator-oauth-client/client.json=caelo-production-operator-oauth-client:latest",
     ]);
     // Converged: nothing left to remove or mount.

package/src/stack-contract.ts

@@ -443,6 +443,12 @@ export const SERVICE_SECRET_FILES: Readonly<Record<CloudRunSlug, readonly Secret
 
 /** A secret file an older contract mounted; upgrade unmounts it in the roll. */
 export interface RetiredSecretFile {
+  /**
+   * The contract name of the secret the file was mounted from (gcp id
+   * `caelo-<env>-<secret>`). Names the volume behind the mount — and a
+   * volume left behind without one (#954) — so the roll removes it too.
+   */
+  readonly secret: string;
   readonly mountDir: string;
   readonly fileName: string;
   readonly why: string;
@@ -450,13 +456,15 @@ export interface RetiredSecretFile {
 
 /**
  * Secret files older contracts mounted that nothing reads any more, per
- * service. The roll removes each mount (`--remove-secrets=<path>`) in the
- * same revision that mounts its replacement.
+ * service. The roll removes each mount (`--remove-secrets=<path>`) and the
+ * volume of that secret (`--remove-volume=<name>`) in the same revision that
+ * mounts its replacement (stack-converge.ts planSecretFiles).
  */
 export const RETIRED_SECRET_FILES: Readonly<Record<CloudRunSlug, readonly RetiredSecretFile[]>> = {
   admin: [],
   gateway: [
     {
+      secret: "mcp-oauth-client",
       mountDir: "/secrets/mcp-oauth-client",
       fileName: "client.json",
       why: "the Google sign-in client is mounted from operator-oauth-client now (#914)",

package/src/stack-converge-secrets.test.ts

@@ -14,13 +14,20 @@
 
 import { describe, expect, it } from "bun:test";
 import { databaseUrlFromEnv } from "@caelo-cms/shared";
-import type { CloudRunSlug } from "./stack-contract.js";
+import type { GcloudResult } from "./gcloud.js";
 import {
+  type CloudRunSlug,
+  OPERATOR_OAUTH_CLIENT_FILE,
+  RETIRED_SECRET_FILES,
+} from "./stack-contract.js";
+import {
+  findVolumesWithMissingSecrets,
   type LiveEnvValue,
   liveContainerEnv,
   liveEnvHasInlinePassword,
   planContractEnv,
   planEnvUpdate,
+  planSecretFiles,
   serviceRollArgs,
 } from "./stack-converge.js";
 
@@ -460,3 +467,132 @@ describe("liveEnvHasInlinePassword", () => {
     expect(liveEnvHasInlinePassword(services.gateway.env)).toBe(false);
   });
 });
+
+describe("retiring a secret file removes its volume too (#954)", () => {
+  // The shape searchviu's gateway was left in: gcloud names a volume it
+  // mints for `--update-secrets` after the secret plus a random suffix.
+  const RETIRED_VOLUME = "caelo-production-mcp-oauth-client-mac-won";
+  const OPERATOR_VOLUME = "caelo-production-operator-oauth-client-puw-tuq";
+  const operatorVolume = {
+    name: OPERATOR_VOLUME,
+    secret: {
+      secretName: "caelo-production-operator-oauth-client",
+      items: [{ key: "latest", path: "client.json" }],
+    },
+  };
+  const retiredVolume = {
+    name: RETIRED_VOLUME,
+    secret: {
+      secretName: "projects/42/secrets/caelo-production-mcp-oauth-client",
+      items: [{ key: "latest", path: "client.json" }],
+    },
+  };
+  const plan = (live: Parameters<typeof planSecretFiles>[0]) =>
+    planSecretFiles(live, [OPERATOR_OAUTH_CLIENT_FILE], "production", RETIRED_SECRET_FILES.gateway);
+
+  it("a retired mount: removes the file, the mount and the volume behind it in the roll", () => {
+    const r = plan({
+      executionEnvironment: null,
+      volumes: [retiredVolume],
+      mounts: [{ name: RETIRED_VOLUME, mountPath: "/secrets/mcp-oauth-client" }],
+    });
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.flags).toContain("--remove-secrets=/secrets/mcp-oauth-client/client.json");
+    expect(r.flags).toContain(`--remove-volume=${RETIRED_VOLUME}`);
+    expect(r.flags).toContain("--remove-volume-mount=/secrets/mcp-oauth-client");
+    expect(r.changes.some((c) => c.startsWith(`volume ${RETIRED_VOLUME}: removed`))).toBe(true);
+  });
+
+  it("an orphaned retired volume (no mount, the searchviu state): removes it and says so", () => {
+    const r = plan({
+      executionEnvironment: null,
+      volumes: [retiredVolume, operatorVolume],
+      mounts: [{ name: OPERATOR_VOLUME, mountPath: "/secrets/operator-oauth-client" }],
+    });
+    expect(r).toEqual({
+      ok: true,
+      flags: [`--remove-volume=${RETIRED_VOLUME}`],
+      changes: [
+        `volume ${RETIRED_VOLUME}: removed (the Google sign-in client is mounted from operator-oauth-client now (#914); nothing mounts it)`,
+      ],
+    });
+  });
+
+  it("a converged service plans nothing", () => {
+    const r = plan({
+      executionEnvironment: null,
+      volumes: [operatorVolume],
+      mounts: [{ name: OPERATOR_VOLUME, mountPath: "/secrets/operator-oauth-client" }],
+    });
+    expect(r).toEqual({ ok: true, flags: [], changes: [] });
+  });
+
+  it("refuses a volume of the retired secret mounted somewhere else instead of pulling it", () => {
+    const r = plan({
+      executionEnvironment: null,
+      volumes: [retiredVolume, operatorVolume],
+      mounts: [
+        { name: OPERATOR_VOLUME, mountPath: "/secrets/operator-oauth-client" },
+        { name: RETIRED_VOLUME, mountPath: "/elsewhere" },
+      ],
+    });
+    expect(r.ok).toBe(false);
+    if (r.ok) return;
+    expect(r.error).toContain(`--remove-volume=${RETIRED_VOLUME}`);
+  });
+
+  describe("preflight: a volume pointing at a deleted secret", () => {
+    const ok = (stdout = ""): GcloudResult => ({ ok: true, stdout, stderr: "", exitCode: 0 });
+    const missing: GcloudResult = {
+      ok: false,
+      stdout: "",
+      stderr:
+        "ERROR: (gcloud.secrets.describe) NOT_FOUND: Secret [projects/42/secrets/x] not found",
+      exitCode: 1,
+    };
+    const where = {
+      serviceName: "caelo-production-gateway-532b35d",
+      projectId: "acme",
+      region: "europe-west1",
+    };
+    const live = {
+      executionEnvironment: null,
+      volumes: [retiredVolume, operatorVolume],
+      mounts: [{ name: OPERATOR_VOLUME, mountPath: "/secrets/operator-oauth-client" }],
+    };
+    const gcloud = (gone: string) => {
+      const calls: string[] = [];
+      const run = async (args: string[]) => {
+        calls.push(args.join(" "));
+        return args[2] === gone ? missing : ok("name");
+      };
+      return { run, calls };
+    };
+
+    it("names the volume and the exact fix", async () => {
+      const g = gcloud("caelo-production-mcp-oauth-client");
+      const problems = await findVolumesWithMissingSecrets(live, where, [], g.run);
+      expect(problems).toHaveLength(1);
+      expect(problems[0]).toContain(`volume "${RETIRED_VOLUME}" (not mounted)`);
+      expect(problems[0]).toContain("caelo-production-mcp-oauth-client, which no longer exists");
+      expect(problems[0]).toContain(
+        `gcloud run services update caelo-production-gateway-532b35d --region=europe-west1 --project=acme --remove-volume=${RETIRED_VOLUME}`,
+      );
+    });
+
+    it("skips a volume the roll removes anyway", async () => {
+      const g = gcloud("caelo-production-mcp-oauth-client");
+      expect(await findVolumesWithMissingSecrets(live, where, [RETIRED_VOLUME], g.run)).toEqual([]);
+      expect(g.calls.some((c) => c.includes("mcp-oauth-client"))).toBe(false);
+    });
+
+    it("a mounted volume's fix removes the mount too", async () => {
+      const g = gcloud("caelo-production-operator-oauth-client");
+      const [problem] = await findVolumesWithMissingSecrets(live, where, [RETIRED_VOLUME], g.run);
+      expect(problem).toContain(
+        `--remove-volume-mount=/secrets/operator-oauth-client --remove-volume=${OPERATOR_VOLUME}`,
+      );
+    });
+  });
+});