@caelo-cms/provisioning 0.10.68 → 0.10.70

package/src/stack-converge.ts

@@ -1023,14 +1023,51 @@ export function planSecretFiles(
   const changes: string[] = [];
   // #914 — mounts an older contract made and nothing reads any more
   // (RETIRED_SECRET_FILES) go in the same revision that mounts their
-  // replacement. `--remove-secrets=<dir>/<file>` drops the file, and the
-  // mount with it when it was the volume's only file (gcloud
-  // `_PruneManagedVolumeMapping`). Repeated `--remove-secrets` /
-  // `--update-secrets` flags merge (gcloud map flags use UpdateAction).
+  // replacement. `--remove-secrets=<dir>/<file>` drops the file and its
+  // mount, but NOT the volume: gcloud's `_PruneManagedVolumeMapping` only
+  // deletes the mount, so the volume stayed behind naming a secret the
+  // upgrade deletes after the roll, and the next revision could not deploy
+  // (#954, searchviu). The plan therefore removes that volume itself, plus
+  // any volume of a retired secret that an earlier roll already orphaned.
+  // Repeated map/list flags merge (gcloud UpdateAction); removing a volume
+  // or mount that is already gone is a no-op in gcloud.
+  const removedVolumes = new Set<string>();
+  const removeVolume = (name: string, why: string) => {
+    if (removedVolumes.has(name)) return;
+    removedVolumes.add(name);
+    flags.push(`--remove-volume=${name}`);
+    changes.push(`volume ${name}: removed (${why})`);
+  };
   for (const old of retired) {
-    if (!live.mounts.some((m) => m.mountPath === old.mountDir)) continue;
-    flags.push(`--remove-secrets=${old.mountDir}/${old.fileName}`);
-    changes.push(`secret file ${old.mountDir}/${old.fileName}: removed (${old.why})`);
+    const retiredId = gcpSecretId(env, old.secret);
+    const mount = live.mounts.find((m) => m.mountPath === old.mountDir);
+    if (mount) {
+      const volume = live.volumes.find((v) => v.name === mount.name);
+      const volumeSecret = bareSecretId(volume?.secret?.secretName);
+      if (volume && volumeSecret !== retiredId) {
+        return {
+          ok: false,
+          error: `the service mounts volume "${mount.name}" (secret ${volumeSecret || "(none)"}) at ${old.mountDir}, where only the retired ${retiredId} was ever mounted. Check what it is; remove it (gcloud run services update <service> --remove-volume-mount=${old.mountDir} --remove-volume=${mount.name}) only if nothing needs it, then re-run upgrade.`,
+        };
+      }
+      flags.push(`--remove-secrets=${old.mountDir}/${old.fileName}`);
+      flags.push(`--remove-volume-mount=${old.mountDir}`);
+      changes.push(`secret file ${old.mountDir}/${old.fileName}: removed (${old.why})`);
+      if (volume) removeVolume(volume.name, old.why);
+    }
+    const mounted = new Set(
+      live.mounts.filter((m) => m.mountPath !== old.mountDir).map((m) => m.name),
+    );
+    for (const v of live.volumes) {
+      if (bareSecretId(v.secret?.secretName) !== retiredId) continue;
+      if (mounted.has(v.name)) {
+        return {
+          ok: false,
+          error: `the service mounts volume "${v.name}" of the retired secret ${retiredId} somewhere other than ${old.mountDir}. Nothing reads that secret any more and upgrade deletes it after the roll; remove the mount and the volume (gcloud run services update <service> --remove-volume-mount=<path> --remove-volume=${v.name}), then re-run upgrade.`,
+        };
+      }
+      removeVolume(v.name, `${old.why}; nothing mounts it`);
+    }
   }
   for (const file of files) {
     const secretId = gcpSecretId(env, file.secret);
@@ -1052,6 +1089,50 @@ export function planSecretFiles(
   return { ok: true, flags, changes };
 }
 
+/**
+ * #954 — the secret volumes of a service whose secret no longer exists in
+ * Secret Manager. Cloud Run refuses every new revision of such a service,
+ * so the upgrade names each one with the exact fix before it starts a roll
+ * instead of failing mid-roll. Volumes in `removing` (the roll's own
+ * `--remove-volume` flags) are skipped: the roll takes them off anyway.
+ * A secret gcloud cannot read for another reason is reported too — the
+ * roll's outcome would be unknowable.
+ */
+export async function findVolumesWithMissingSecrets(
+  live: LiveVolumes,
+  where: { readonly serviceName: string; readonly projectId: string; readonly region: string },
+  removing: readonly string[],
+  run: GcloudRunner,
+): Promise<string[]> {
+  const problems: string[] = [];
+  for (const v of live.volumes) {
+    const secretId = bareSecretId(v.secret?.secretName);
+    if (!secretId || removing.includes(v.name)) continue;
+    const r = await run([
+      "secrets",
+      "describe",
+      secretId,
+      `--project=${where.projectId}`,
+      "--format=value(name)",
+    ]);
+    if (r.ok) continue;
+    const mounts = live.mounts.filter((m) => m.name === v.name).map((m) => m.mountPath);
+    const fix = [
+      `gcloud run services update ${where.serviceName}`,
+      `--region=${where.region}`,
+      `--project=${where.projectId}`,
+      ...mounts.map((p) => `--remove-volume-mount=${p}`),
+      `--remove-volume=${v.name}`,
+    ].join(" ");
+    problems.push(
+      RESOURCE_MISSING.test(r.stderr)
+        ? `${where.serviceName} has volume "${v.name}"${mounts.length > 0 ? ` (mounted at ${mounts.join(", ")})` : " (not mounted)"} pointing at secret ${secretId}, which no longer exists, so Cloud Run would refuse the next revision. If nothing reads it, remove it: ${fix} — then re-run upgrade.`
+        : `could not check secret ${secretId} behind volume "${v.name}" of ${where.serviceName}: ${r.stderr.trim() || `gcloud exited ${r.exitCode}`}`,
+    );
+  }
+  return problems;
+}
+
 /** The request timeout of a described service, in seconds (null: Cloud Run's default, 300). */
 export function liveTimeoutSeconds(serviceJson: string): number | null {
   const svc = JSON.parse(serviceJson) as {

package/src/state-command.ts

@@ -14,7 +14,8 @@ import { homedir } from "node:os";
 import { join } from "node:path";
 import { confirm, isCancel, log, note } from "@clack/prompts";
 import { bold, red } from "kleur/colors";
-import { checkGcloudAuth } from "./gcloud.js";
+import { adoptionCandidates, existingCandidates } from "./adopt-existing.js";
+import { checkGcloudAuth, gcloud } from "./gcloud.js";
 import {
   adoptStateBackend,
   currentMetadata,
@@ -260,6 +261,10 @@ export async function stateCommand(argv: readonly string[]): Promise<void> {
         return moved;
       },
       writePlan: (label, steps) => writePlanFile(root, label, steps),
+      existingInProject: async (steps) =>
+        (await existingCandidates(adoptionCandidates(steps, projectId), gcloud)).map(
+          (c) => `${c.label} ${c.display}`,
+        ),
     },
   );
   if (!result.ok) {

package/src/state-migrate.test.ts

@@ -154,6 +154,8 @@ function machine(initial: StateBackendRecord = { kind: "local" }) {
     acceptCost: true,
     bucketRecord: (initial.kind === "gcs" ? "gcs" : null) as "gcs" | "local" | null,
     planFiles: [] as { label: string; steps: number }[],
+    /** What existingInProject answers (#1079); an Error is thrown. */
+    existing: [] as readonly string[] | Error,
   };
   const meta = (): MigrateInstall => ({
     installId: "gcp-firebase-acme",
@@ -209,6 +211,10 @@ function machine(initial: StateBackendRecord = { kind: "local" }) {
       m.planFiles.push({ label, steps: steps.length });
       return `${ROOT}/plan-${label}.txt`;
     },
+    existingInProject: async () => {
+      if (m.existing instanceof Error) throw m.existing;
+      return m.existing;
+    },
   };
   p.stacks.set(localUrl, {
     outputs: {
@@ -380,6 +386,38 @@ describe("migrateState — failures leave the source authoritative", () => {
     expect(t.m.backups).toBe(0);
   });
 
+  it("the refusal names the planned creates the installer adopts (#1079)", async () => {
+    const t = machine();
+    const local = t.p.stacks.get(t.localUrl);
+    if (local) local.drift = 2;
+    t.m.existing = ["service account caelo-production-gateway-sa@acme.iam.gserviceaccount.com"];
+    const r = await migrateState(t.meta(), { to: "remote", dryRun: false }, t.deps);
+    expect(r.ok).toBe(false);
+    if (r.ok) return;
+    expect(r.error).toContain(
+      "1 of the planned creates exist in the project already (service account caelo-production-gateway-sa@acme.iam.gserviceaccount.com): upgrade made them without Pulumi. The installer adopts them into the stack first",
+    );
+    expect(r.error.indexOf("adopts them")).toBeLessThan(r.error.indexOf("Run upgrade first"));
+  });
+
+  it("the refusal says nothing about adopting when nothing exists, and says so when it can't check", async () => {
+    const t = machine();
+    const local = t.p.stacks.get(t.localUrl);
+    if (local) local.drift = 1;
+    const none = await migrateState(t.meta(), { to: "remote", dryRun: false }, t.deps);
+    expect(none.ok).toBe(false);
+    if (none.ok) return;
+    expect(none.error).not.toContain("adopt");
+    t.m.existing = new Error("gcloud: PERMISSION_DENIED");
+    const failed = await migrateState(t.meta(), { to: "remote", dryRun: false }, t.deps);
+    expect(failed.ok).toBe(false);
+    if (failed.ok) return;
+    expect(failed.error).toContain(
+      "(Could not check which planned creates exist in the project already: gcloud: PERMISSION_DENIED. The installer checks again.)",
+    );
+    expect(failed.error).toContain("Run upgrade first");
+  });
+
   it("refuses a stack with pending operations", async () => {
     const t = machine();
     const local = t.p.stacks.get(t.localUrl);

package/src/state-migrate.ts

@@ -117,6 +117,13 @@ export interface MigrateDeps {
    * writePlanFile); returns its path.
    */
   writePlan(label: string, steps: readonly PlannedStep[]): string;
+  /**
+   * #1079 — the planned creates whose resource exists in the project already
+   * (adopt-existing.ts: made by `upgrade` through gcloud), one
+   * `<kind> <name>` each. The installer adopts these into the stack before it
+   * applies anything. Throws when it can't tell.
+   */
+  existingInProject(steps: readonly PlannedStep[]): Promise<readonly string[]>;
 }
 
 export interface MigrateOptions {
@@ -167,6 +174,22 @@ function planReport(deps: MigrateDeps, label: string, steps: readonly PlannedSte
   return [...formatPlan(steps).map((l) => `  ${l}`), where].join("\n");
 }
 
+/**
+ * The refusal's line about planned creates the installer adopts instead
+ * (#1079), or "" when there are none. A failed check is said, not fatal: the
+ * installer checks again before it applies anything.
+ */
+async function adoptionNote(deps: MigrateDeps, steps: readonly PlannedStep[]): Promise<string> {
+  let existing: readonly string[];
+  try {
+    existing = await deps.existingInProject(steps);
+  } catch (e) {
+    return `(Could not check which planned creates exist in the project already: ${errorText(e)}. The installer checks again.)\n`;
+  }
+  if (existing.length === 0) return "";
+  return `${existing.length} of the planned creates exist in the project already (${existing.join(", ")}): upgrade made them without Pulumi. The installer adopts them into the stack first instead of creating them again, then shows what is left.\n`;
+}
+
 /**
  * Run the migration. Never throws for an expected failure: the result says
  * what failed, and at every failure point the source backend is still the
@@ -264,7 +287,7 @@ export async function migrateState(
       pre.failed(`A preview shows ${sourcePlan.length} change(s)`);
       return {
         ok: false,
-        error: `The stack is not converged (a preview shows ${sourcePlan.length} change(s)), so a copy can't be proven identical. Nothing was changed. The planned changes:\n${planReport(deps, "state-migrate", sourcePlan)}\nRun upgrade first, then re-run the installer (${selfCommand()}, pick this install): it shows this list again and asks before it replaces or deletes anything. Migrate once it reports no changes.`,
+        error: `The stack is not converged (a preview shows ${sourcePlan.length} change(s)), so a copy can't be proven identical. Nothing was changed. The planned changes:\n${planReport(deps, "state-migrate", sourcePlan)}\n${await adoptionNote(deps, sourcePlan)}Run upgrade first, then re-run the installer (${selfCommand()}, pick this install): it shows this list again and asks before it replaces or deletes anything. Migrate once it reports no changes.`,
       };
     }
     sourceDigest = await pulumi.outputsDigest(from);

package/src/upgrade/app-roll.ts

@@ -37,6 +37,7 @@ import {
 import {
   type DeployedService,
   type EnvChange,
+  findVolumesWithMissingSecrets,
   type LiveEnvValue,
   type LiveVolumes,
   liveContainerEnv,
@@ -147,6 +148,15 @@ export async function planAppRoll(
   install: AppRollInstall,
   images: Readonly<Record<RollSlug, ReleaseImage>>,
   run: GcloudRunner,
+  checks: {
+    /**
+     * #954 — check that every secret volume's secret still exists. The
+     * operator / converge identity can read secret metadata; the upgrader
+     * job's narrow identity holds no Secret Manager permission at all
+     * (upgrader-setup.ts), so it passes false.
+     */
+    readonly secretVolumes: boolean;
+  },
 ): Promise<PlanResult> {
   const services: ServicePlan[] = [];
   for (const slug of ROLL_ORDER) {
@@ -223,6 +233,25 @@ export async function planAppRoll(
     }
     secretFiles[s.slug] = files;
   }
+  // #954 — a volume pointing at a deleted secret makes Cloud Run refuse
+  // every new revision: name it with its fix now, before any roll starts.
+  const orphaned: string[] = [];
+  for (const s of checks.secretVolumes ? services : []) {
+    const removing = secretFiles[s.slug].flags
+      .filter((f) => f.startsWith("--remove-volume="))
+      .map((f) => f.slice("--remove-volume=".length));
+    orphaned.push(
+      ...(await findVolumesWithMissingSecrets(
+        s.liveVolumes,
+        { serviceName: s.serviceName, projectId: install.projectId, region: install.region },
+        removing,
+        run,
+      )),
+    );
+  }
+  if (orphaned.length > 0) {
+    return { ok: false, error: `a service can't take a new revision: ${orphaned.join(" ")}` };
+  }
   const warnings: string[] = [];
   const rolls = services.map((s): RollPlan => {
     // #553 — the admin runs the Lighthouse quality audit; raise it to the

package/src/upgrade/upgrade.test.ts

@@ -906,7 +906,7 @@ const INSTALL: AppRollInstall = {
 };
 
 async function plannedRoll(run: GcloudRunner): Promise<AppRollPlan> {
-  const planned = await planAppRoll(INSTALL, IMAGES, run);
+  const planned = await planAppRoll(INSTALL, IMAGES, run, { secretVolumes: true });
   if (!planned.ok) throw new Error(planned.error);
   return planned.plan;
 }

package/src/upgrade/upgrade.ts

@@ -177,6 +177,7 @@ export async function upgradeGcpInstallOutcome(
     },
     images.images,
     run,
+    { secretVolumes: true },
   );
   if (!planned.ok) {
     sPre.failed("Pre-flight failed");

package/src/upgrader-job.ts

@@ -519,6 +519,9 @@ export async function runUpgraderJob(deps: UpgraderJobDeps): Promise<UpgraderRun
       },
       images,
       deps.run,
+      // Holds no Secret Manager permission (upgrader-setup.ts); the
+      // converge — operator or converge job — runs the #954 volume check.
+      { secretVolumes: false },
     );
     if (!planned.ok) {
       return stopWith("failed", ["plan"], `Nothing was changed: ${planned.error}.`);

package/src/wizards/gcp-pulumi.ts

@@ -14,8 +14,14 @@
 import { mkdirSync, readFileSync, realpathSync, renameSync, writeFileSync } from "node:fs";
 import { join, relative, resolve as resolvePath } from "node:path";
 import * as pulumi from "@pulumi/pulumi/automation";
+import {
+  type AdoptionCandidate,
+  AdoptionError,
+  adoptExistingResources,
+} from "../adopt-existing.js";
 import { AUDIT_PARALLELISM_DEFAULT } from "../audit-job-setup.js";
 import { checkpointableOutputs } from "../checkpoint-secrets.js";
+import { gcloud } from "../gcloud.js";
 import { GCP_STACK_ENV } from "../gcp-names.js";
 import { type ImageDigests, stackWorkDir } from "../install-state.js";
 import { collectPreviewSteps, type PlannedStep } from "../preview-plan.js";
@@ -395,6 +401,7 @@ export function resolveSecretReplication(
 
 /** A resource as `pulumi stack export` lists it. */
 export interface StateResource {
+  urn?: string;
   type?: string;
   inputs?: Record<string, unknown>;
   outputs?: Record<string, unknown>;
@@ -534,6 +541,54 @@ export function pulumiGcpEnv(
   };
 }
 
+/**
+ * The URN of the stack's explicit GCP provider (`new gcp.Provider` in both
+ * stacks), which every stack resource uses. An adopted resource is imported
+ * under it (#1079): under Pulumi's default provider instead, the next `up`
+ * would see a provider change on it.
+ *
+ * @throws AdoptionError unless the state holds exactly one
+ */
+export function stackGcpProviderUrn(resources: ReadonlyArray<StateResource>): string {
+  const urns = resources
+    .filter((r) => r.type === "pulumi:providers:gcp" && typeof r.urn === "string")
+    .map((r) => r.urn as string);
+  if (urns.length !== 1) {
+    throw new AdoptionError(
+      `the stack's state holds ${urns.length} GCP providers, so it is unclear which one resources that already exist would be adopted under. Nothing was imported or applied. Report this; the stack needs a look before the installer can run.`,
+    );
+  }
+  return urns[0] as string;
+}
+
+/** The name {@link importIntoStack} gives the stack's provider in the import's name table. */
+const IMPORT_PROVIDER_NAME = "gcpProvider";
+
+/**
+ * `pulumi import` the candidates into the stack under its explicit provider.
+ * State only: nothing in the project changes. Not protected (the program
+ * does not protect them either) and no code generated.
+ */
+async function importIntoStack(
+  stack: pulumi.Stack,
+  providerUrn: string,
+  candidates: readonly AdoptionCandidate[],
+  onOutput: (msg: string) => void,
+): Promise<void> {
+  await stack.import({
+    resources: candidates.map((c) => ({
+      type: c.step.type,
+      name: c.step.name,
+      id: c.importId,
+      provider: IMPORT_PROVIDER_NAME,
+    })),
+    nameTable: { [IMPORT_PROVIDER_NAME]: providerUrn },
+    protect: false,
+    generateCode: false,
+    onOutput,
+  });
+}
+
 /** Thrown when the review (preview-plan.ts planGate) stops `pulumi up` after the preview. */
 export class PulumiUpDeclinedError extends Error {
   override readonly name = "PulumiUpDeclinedError";
@@ -544,16 +599,23 @@ export class PulumiUpDeclinedError extends Error {
  * Streams resource-create events to the supplied onEvent callback so
  * the wizard can render a live progress bar.
  *
- * On a stack that already holds resources, a preview runs first (#737) and
- * `review` sees its planned steps: it lists them and decides (planGate)
- * whether `up` may apply them. Nothing is applied when it says no.
+ * On a stack that already holds resources, a preview runs first (#737).
+ * Planned creates of resources that exist in the project already (made by
+ * `upgrade` through gcloud) are adopted into the stack and the stack is
+ * previewed again (adopt-existing.ts, #1079). `review` sees that plan and
+ * what was adopted: it lists them and decides (planGate) whether `up` may
+ * apply the plan. Nothing is applied when it says no.
  *
  * @throws PulumiUpDeclinedError when `review` declines the plan
+ * @throws AdoptionError when adopting fails; nothing was applied
  */
 export async function pulumiUpGcp(
   inputs: PulumiUpInputs,
   onEvent: (kind: "resource" | "error" | "log", message: string) => void,
-  review: (steps: readonly PlannedStep[]) => Promise<boolean>,
+  review: (
+    steps: readonly PlannedStep[],
+    adopted: readonly AdoptionCandidate[],
+  ) => Promise<boolean>,
 ): Promise<PulumiUpResult> {
   const env = pulumiGcpEnv(process.env, inputs);
 
@@ -608,11 +670,25 @@ export async function pulumiUpGcp(
 
   // #737 — never `up` a deployed stack blind: list what it would change
   // (replaces and deletes first) and let the review stop it. A first run
-  // on an empty stack only creates, so it skips the extra preview.
+  // on an empty stack only creates, so it skips the extra preview (and has
+  // no provider in its state to adopt anything under).
   if (stateResources.length > 0) {
-    const collector = collectPreviewSteps();
-    await stack.preview({ onEvent: collector.onEvent });
-    if (!(await review(collector.steps()))) {
+    const preview = async () => {
+      const collector = collectPreviewSteps();
+      await stack.preview({ onEvent: collector.onEvent });
+      return collector.steps();
+    };
+    // #1079 — what `upgrade` created through gcloud joins the state first.
+    const { steps, adopted } = await adoptExistingResources(await preview(), {
+      projectId: inputs.projectId,
+      gcloud,
+      preview,
+      importResources: (candidates) =>
+        importIntoStack(stack, stackGcpProviderUrn(stateResources), candidates, (msg) =>
+          onEvent("log", msg),
+        ),
+    });
+    if (!(await review(steps, adopted))) {
       throw new PulumiUpDeclinedError("pulumi up was not run: the planned changes were declined.");
     }
   }

package/src/wizards/gcp.ts

@@ -33,6 +33,7 @@ import { join } from "node:path";
 import { cancel, confirm, isCancel, log, note, select, spinner } from "@clack/prompts";
 import { bold, cyan, dim, green, red, yellow } from "kleur/colors";
 import { checkAdcImpersonation } from "../adc-impersonation.js";
+import { type AdoptionCandidate, AdoptionError, formatAdopted } from "../adopt-existing.js";
 import { ensureAuditJob, resolveAdminImage } from "../audit-job-setup.js";
 import { CERT_WAIT_TIMEOUT_MS, lbCertProbe, waitForCert } from "../cert-wait.js";
 import { redactCredentials } from "../checkpoint-secrets.js";
@@ -1486,16 +1487,21 @@ interface PulumiUpOpts {
  * The installer's review of a deployed stack's plan before `pulumi up`
  * (#737): the grouped list, the full diff in the install dir, and a y/N
  * for any replace or delete. A data-bearing one is asked about by name and
- * defaults to No; without a terminal it is refused (planGate).
+ * defaults to No; without a terminal it is refused (planGate). Resources
+ * adopted into the stack before this preview (adopt-existing.ts, #1079) are
+ * listed first; they are no longer in the plan.
  *
  * @returns whether `pulumi up` may apply the plan
  */
 async function reviewPulumiPlan(
   installId: string,
   steps: readonly PlannedStep[],
+  adopted: readonly AdoptionCandidate[],
   nonInteractive: boolean,
 ): Promise<boolean> {
   const gate = planGate(steps, { interactive: !nonInteractive && process.stdin.isTTY === true });
+  const adoptedLines = formatAdopted(adopted);
+  if (adoptedLines.length > 0) log.info(adoptedLines.join("\n"));
   if (steps.length === 0) {
     log.info(dim("Preview: no changes planned."));
     return true;
@@ -1601,11 +1607,12 @@ async function stepPulumiUp(installId: string, opts: PulumiUpOpts): Promise<void
         // logs: silent — too noisy for a live progress UI; pulumi
         // diagnostics will print on failure via the SDK's onOutput.
       },
-      (steps) => reviewPulumiPlan(installId, steps, opts.nonInteractive),
+      (steps, adopted) => reviewPulumiPlan(installId, steps, adopted, opts.nonInteractive),
     );
   } catch (e) {
-    if (!(e instanceof PulumiUpDeclinedError)) throw e;
-    declined = e.message;
+    if (e instanceof AdoptionError) declined = `pulumi up was not run: ${e.message}`;
+    else if (e instanceof PulumiUpDeclinedError) declined = e.message;
+    else throw e;
   } finally {
     releaseLock();
   }