@caelo-cms/provisioning 0.10.61 → 0.10.64

package/src/gcloud.ts

@@ -86,6 +86,71 @@ export async function activeAccount(
   return value.length > 0 ? value : null;
 }
 
+/**
+ * gcloud's stderr when the stored credentials can no longer mint a token:
+ * an expired or revoked refresh token, a reauth policy that wants the
+ * password again, or a reauth prompt it may not show because prompts are off
+ * ({@link gcloudEnv}). Every one is fixed by logging in again, not by
+ * retrying (#630).
+ */
+const GCLOUD_AUTH_ERROR =
+  /Reauthentication|reauth|invalid_grant|cannot prompt|refreshing your current auth tokens|do not currently have an active account/i;
+
+/** Whether a failed gcloud call failed because its credentials expired (#630). */
+export function isGcloudAuthError(stderr: string): boolean {
+  return GCLOUD_AUTH_ERROR.test(stderr);
+}
+
+/** The fix for expired credentials on the Owner's machine or in Cloud Shell. */
+export const GCLOUD_REAUTH_HINT =
+  "gcloud credentials expired. Run `gcloud auth login` (and `gcloud auth application-default login` where Pulumi runs), then run this command again.";
+
+/** The first non-empty line of gcloud's stderr, for a one-line error. */
+export function gcloudErrorLine(stderr: string): string {
+  return (
+    stderr
+      .split("\n")
+      .map((l) => l.replace(/^ERROR:\s*/, "").trim())
+      .find((l) => l.length > 0) ?? "no error output"
+  );
+}
+
+export type GcloudAuthCheck =
+  | { readonly ok: true; readonly account: string }
+  | { readonly ok: false; readonly error: string };
+
+/**
+ * Pre-flight for every lifecycle command that talks to GCP: an active
+ * account AND a token gcloud can mint now. {@link activeAccount} alone
+ * passes with expired credentials (`auth list` reads local config only), so
+ * the first real lookup failed and read as "service not found" (#630).
+ * The token itself is discarded.
+ */
+export async function checkGcloudAuth(
+  run: (args: string[]) => Promise<GcloudResult> = gcloud,
+): Promise<GcloudAuthCheck> {
+  const account = await activeAccount(run);
+  if (!account) {
+    return {
+      ok: false,
+      error:
+        "No gcloud account is logged in. Run `gcloud auth login` with the Owner's Google account, then run this command again.",
+    };
+  }
+  const token = await run(["auth", "print-access-token", "--quiet"]);
+  if (token.ok) return { ok: true, account };
+  if (isGcloudAuthError(token.stderr)) {
+    return {
+      ok: false,
+      error: `${GCLOUD_REAUTH_HINT} (${account}: ${gcloudErrorLine(token.stderr)})`,
+    };
+  }
+  return {
+    ok: false,
+    error: `gcloud could not get an access token for ${account}: ${gcloudErrorLine(token.stderr)}. Check \`gcloud auth list\` and the network, then run this command again.`,
+  };
+}
+
 export interface BillingAccount {
   id: string;
   displayName: string;
@@ -116,6 +181,23 @@ export async function projectExists(projectId: string): Promise<boolean> {
   return r.ok && r.stdout.trim() === projectId;
 }
 
+/**
+ * Human-readable GCP project name for an install, e.g. `Caelo searchviu-com`.
+ * Carries the domain so an operator with several installs can tell them apart
+ * in the Cloud Console project picker (every install used to be "Caelo CMS").
+ * GCP display names allow only letters, digits, spaces, hyphens, quotes and
+ * `!`, at 4–30 chars — so dots become hyphens and long domains are cut.
+ */
+export function projectDisplayName(domain: string): string {
+  const prefix = "Caelo ";
+  const slug = domain
+    .toLowerCase()
+    .replace(/[^a-z0-9-]+/g, "-")
+    .slice(0, 30 - prefix.length)
+    .replace(/^-+|-+$/g, "");
+  return `${prefix}${slug}`;
+}
+
 export async function createProject(projectId: string, displayName: string): Promise<GcloudResult> {
   return gcloud(["projects", "create", projectId, "--name", displayName]);
 }

package/src/index.test.ts

@@ -3,7 +3,11 @@
 import { describe, expect, it } from "bun:test";
 import { readFileSync } from "node:fs";
 import { join } from "node:path";
-import { CONTENT_HASHED_PATH_PATTERN, IMMUTABLE_CACHE_CONTROL } from "@caelo-cms/shared";
+import {
+  CONTENT_HASHED_PATH_PATTERN,
+  IMMUTABLE_CACHE_CONTROL,
+  SVG_DOCUMENT_CSP,
+} from "@caelo-cms/shared";
 import { generateBootstrapToken } from "./bootstrap-token.js";
 import { generateCaddyfile } from "./caddy.js";
 import { loadCdnCopyAdapter, selfHostedCdnCopy } from "./cdn-copy.js";
@@ -92,7 +96,37 @@ describe("Caddy cache policy for content-hashed assets", () => {
       );
       expect(body).toContain(`header @content_hashed Cache-Control "${IMMUTABLE_CACHE_CONTROL}"`);
       expect(body).toContain('\t@ico path *.ico\n\theader @ico Content-Type "image/x-icon"\n');
+      // #891 — the same security headers as the generated vhosts.
+      expect(body).toContain('\theader X-Content-Type-Options "nosniff"\n');
+      expect(body).toContain(
+        `\t@svg path *.svg\n\theader @svg Content-Security-Policy "${SVG_DOCUMENT_CSP}"\n`,
+      );
+    }
+  });
+
+  // #891 regression: the public vhost served uploaded SVGs without a CSP, so
+  // a script that got past sanitizeSvg ran on the site's origin (which also
+  // serves /api/*) when the SVG's URL was opened.
+  it("public vhosts (live and staging) sandbox SVGs and nosniff everything; admin vhosts are untouched", () => {
+    const out = generateCaddyfile({
+      ...spec,
+      domains: [
+        ...spec.domains,
+        { hostname: "staging.example.com", kind: "public", env: "staging" },
+      ],
+    });
+    const blocks = [
+      out.match(/\nexample\.com \{[\s\S]*?\n\}/)?.[0] ?? "",
+      out.match(/\nstaging\.example\.com \{[\s\S]*?\n\}/)?.[0] ?? "",
+    ];
+    for (const block of blocks) {
+      expect(block).toContain('  header X-Content-Type-Options "nosniff"\n');
+      expect(block).toContain(
+        `  @svg path *.svg\n  header @svg Content-Security-Policy "${SVG_DOCUMENT_CSP}"\n`,
+      );
     }
+    const adminBlock = out.match(/admin\.example\.com \{[\s\S]*?\n\}/)?.[0] ?? "";
+    expect(adminBlock).not.toContain("Content-Security-Policy");
   });
 
   it("public vhosts serve .ico favicons as image/x-icon; admin vhosts are untouched", () => {
@@ -117,6 +151,7 @@ describe("docker-compose generator", () => {
       postgresPassword: "supersecret",
       rolePasswords: { admin: "adminpw", public: "publicpw" },
       caeloSecretKek: "0".repeat(64),
+      internalSecret: "i".repeat(48),
       anthropicApiKey: "sk-ant-…",
       diskSize: "20Gi",
       images: RELEASE_IMAGES,
@@ -138,6 +173,7 @@ describe("docker-compose generator", () => {
       postgresPassword: "supersecret",
       rolePasswords: { admin: "adminpw", public: "publicpw" },
       caeloSecretKek: "0".repeat(64),
+      internalSecret: "i".repeat(48),
       diskSize: "20Gi",
       images: RELEASE_IMAGES,
       ports: { http: 80, https: 443 },
@@ -162,6 +198,7 @@ describe("docker-compose generator", () => {
       postgresPassword: 'pa"ss',
       rolePasswords: { admin: "a", public: "b" },
       caeloSecretKek: "0".repeat(64),
+      internalSecret: "i".repeat(48),
       diskSize: "1Gi",
       images: RELEASE_IMAGES,
       ports: { http: 80, https: 443 },

package/src/infra-contract.test.ts

@@ -20,8 +20,8 @@ describe("infrastructure contract steps", () => {
     );
   });
 
-  it("is at 12: public_role revoke (3), URL-map redirects role (4), no gateway cms_admin login (5), ops-bucket list (6), gateway proxy hops (7), quality-audit job (8), converge job (9), remote MCP (10), Google sign-in (11), project-level sign-in client (12)", () => {
-    expect(INFRA_CONTRACT_VERSION).toBe(12);
+  it("is at 13: public_role revoke (3), URL-map redirects role (4), no gateway cms_admin login (5), ops-bucket list (6), gateway proxy hops (7), quality-audit job (8), converge job (9), remote MCP (10), Google sign-in (11), project-level sign-in client (12), edge security headers (13)", () => {
+    expect(INFRA_CONTRACT_VERSION).toBe(13);
     expect(INFRA_CONTRACT_STEPS[3]).toContain("#633");
     expect(INFRA_CONTRACT_STEPS[4]).toContain("#587");
     expect(INFRA_CONTRACT_STEPS[5]).toContain("#635");
@@ -32,6 +32,7 @@ describe("infrastructure contract steps", () => {
     expect(INFRA_CONTRACT_STEPS[10]).toContain("#806");
     expect(INFRA_CONTRACT_STEPS[11]).toContain("#914");
     expect(INFRA_CONTRACT_STEPS[12]).toContain("project's IAP settings");
+    expect(INFRA_CONTRACT_STEPS[13]).toContain("#891");
   });
 
   it("lists the pending steps oldest first", () => {
@@ -42,6 +43,6 @@ describe("infrastructure contract steps", () => {
   });
 
   it("names a version a newer release added by number", () => {
-    expect(infraStepsText(12, 13)).toBe("infrastructure step 13 (see that release's notes)");
+    expect(infraStepsText(13, 14)).toBe("infrastructure step 14 (see that release's notes)");
   });
 });

package/src/infra-contract.ts

@@ -22,7 +22,7 @@
  * whose app-roll references infrastructure an install does not have yet fails
  * at roll time, after migrations ran.
  */
-export const INFRA_CONTRACT_VERSION = 12;
+export const INFRA_CONTRACT_VERSION = 13;
 
 // History (why each bump happened — an install below a number needs the
 // converge that introduced it):
@@ -93,6 +93,14 @@ export const INFRA_CONTRACT_VERSION = 12;
 //       (`iap.projects.getSettings` / `updateSettings`) on the project. New
 //       IAM the in-cloud upgrader cannot grant; without it the new job image
 //       fails every Google sign-in sync with 403.
+//  13 — #891: the public site's security headers at the edge. On `gcp` the
+//       static backend bucket gets the `X-Content-Type-Options: nosniff`
+//       custom response header, the live SVG objects get
+//       `Content-Disposition: attachment` and Cloud CDN is purged once
+//       (static-svg-gcp.ts); aws / azure gain response-header policies /
+//       a Front Door rule set (static-security-headers.ts). Backend-bucket,
+//       object-metadata and CDN writes the in-cloud upgrader holds no rights
+//       for, so the converge applies them.
 
 /**
  * What each contract version adds, in operator terms — the upgrader's
@@ -112,6 +120,7 @@ export const INFRA_CONTRACT_STEPS: Readonly<Record<number, string>> = {
   10: "remote MCP: the OAuth client and seal-key secrets, their IAM, the gateway's secret file and timeout, IAP programmatic-client sync and the load-balancer rules (#806)",
   11: "Google sign-in: the renamed sign-in client secret, its IAM and mounts, the operator-access job's read access to it, and the admin's URL and organisation domain (#914)",
   12: "Google sign-in at project level: the operator-access job's permission to set the sign-in client in the project's IAP settings (#914)",
+  13: "the public site's security headers at the edge: nosniff on every file, uploaded SVGs no longer run as documents on the site's origin, and a one-time CDN purge (#891)",
 };
 
 /**

package/src/install-job.ts

@@ -69,11 +69,14 @@ export const OPS_IMAGE_REPOSITORY =
  * tail it. Granted only to installs that use the job. Not an escalation: the
  * provisioner already holds `resourcemanager.projectIamAdmin`.
  */
-export const INSTALL_JOB_ROLES: readonly string[] = [
+export const INSTALL_JOB_ROLES = [
   "roles/iam.roleAdmin",
   "roles/cloudscheduler.admin",
   "roles/logging.logWriter",
-];
+] as const;
+
+/** One of {@link INSTALL_JOB_ROLES}. */
+export type InstallJobRole = (typeof INSTALL_JOB_ROLES)[number];
 
 /** Resources the job runs with: Pulumi + the Node language host want memory. */
 const JOB_CPU = "2";

package/src/install-state.ts

@@ -230,6 +230,19 @@ export function installRoot(installId: string): string {
   return join(homedir(), `${ROOT_PREFIX}${installId}`);
 }
 
+/**
+ * Where Pulumi runs for one install (#736): `<installRoot>/stack/<provider>/`.
+ * It holds the install's own stack settings file (`Pulumi.production.yaml`)
+ * and a project file pointing at the package's stack program
+ * (wizards/gcp-pulumi.ts `prepareStackWorkDir`). The settings file used to
+ * live in the package's stack dir, shared by every install of the provider
+ * on the machine: two CLI runs for different installs at once could preview
+ * or `up` one install's state with the other's config.
+ */
+export function stackWorkDir(installRoot: string, provider: "gcp" | "gcp-firebase"): string {
+  return join(installRoot, "stack", provider);
+}
+
 /**
  * Stable install id from the (provider, projectId-or-domain) pair so a re-run
  * with the same inputs always lands on the same `~/.caelo-<id>/` directory.

package/src/lifecycle.ts

@@ -16,11 +16,11 @@
  *   - emits human-readable progress + a final summary
  */
 
-import { join, resolve as resolvePath } from "node:path";
+import { join } from "node:path";
 import { provisioningUpgradeCommand } from "@caelo-cms/shared";
 import { cancel, confirm, isCancel, log, note, select, spinner } from "@clack/prompts";
 import { bold, cyan, dim, green, red, yellow } from "kleur/colors";
-import { activeAccount, gcloud } from "./gcloud.js";
+import { activeAccount, checkGcloudAuth, gcloud } from "./gcloud.js";
 import { type GcloudRunner, realSleep } from "./gcloud-retry.js";
 import { GCP_STACK_ENV } from "./gcp-names.js";
 import { INFRA_CONTRACT_VERSION } from "./infra-contract.js";
@@ -39,6 +39,7 @@ import {
   recordInfraContractVersion,
   scrubInstallProgress,
   selectInstall,
+  stackWorkDir,
   writeMetadata,
 } from "./install-state.js";
 import { runMigrationsViaCloudRunJob } from "./migration-runner.js";
@@ -73,11 +74,12 @@ import {
   type RollSlug,
 } from "./upgrade/app-roll.js";
 import { restoreCommand, takeSqlBackup } from "./upgrade/backup.js";
-import { findServiceUrl, resolveGcpResourceName } from "./upgrade/gcp-lookup.js";
+import { findServiceUrl, lookupFailure, resolveGcpResourceName } from "./upgrade/gcp-lookup.js";
 import { createIapHealthProbe, type HealthProbe } from "./upgrade/health.js";
 import { recordRolledDigests } from "./upgrade/record-rolled.js";
 import {
   canonicalReleaseTag,
+  ensureCosignAvailable,
   fetchLatestReleaseTag,
   RELEASE_REGISTRY,
   releaseImageRef,
@@ -148,14 +150,22 @@ async function chooseInstalls(opts: {
 
 function exitNoInstall(message: string): never {
   log.error(red(message));
-  if (listInstalls().length === 0) {
-    log.warn(
-      `Run ${bold(selfCommand())} first to provision an install, OR copy ${dim("~/.caelo-<install-id>/")} from the provisioning machine.`,
-    );
-  }
+  log.warn(noInstallHint());
   process.exit(1);
 }
 
+/**
+ * What to do when this machine has no copy of the install. A Google Cloud
+ * install keeps its records in its own project, so the fix is `attach`, not
+ * the installer: the installer would provision a new site (#816).
+ */
+export function noInstallHint(): string {
+  return [
+    `A Google Cloud install keeps its records in its own project: run ${bold(selfCommand("attach", "--project", "<gcp-project-id>"))} once on this machine (or pass ${bold("--project <gcp-project-id>")} to upgrade, which attaches it first).`,
+    `A self-hosted install is managed on its own server. To create a new install instead, run ${bold(selfCommand())}.`,
+  ].join("\n");
+}
+
 /** The one install a single-install command (rotate-secret, truncate, destroy) acts on. */
 async function requireInstall(verb: string): Promise<{ installId: string; meta: InstallMetadata }> {
   const [meta] = await chooseInstalls({ verb, allowAll: false });
@@ -212,6 +222,27 @@ function installRegion(meta: InstallMetadata, requested?: string): string {
   }
 }
 
+/**
+ * Pre-flight for a lifecycle command on a GCP install, before any lock,
+ * lookup or change: gcloud can mint a token now (#630: expired credentials
+ * otherwise surface as "service not found" half-way through) and, for an
+ * upgrade that verifies signatures, cosign is on PATH (#816).
+ *
+ * @returns false when the command must stop (the reason is printed)
+ */
+async function gcpPreflight(opts: { readonly cosign: boolean }): Promise<boolean> {
+  const auth = await checkGcloudAuth();
+  if (!auth.ok) {
+    log.error(red(`Nothing was changed: ${auth.error}`));
+    return false;
+  }
+  if (opts.cosign && !(await ensureCosignAvailable())) {
+    log.error(red("Nothing was changed: the release images cannot be verified without cosign."));
+    return false;
+  }
+  return true;
+}
+
 // =========================================================================
 // status — health check + monthly cost
 // =========================================================================
@@ -264,27 +295,34 @@ async function getLatestReleaseTag(): Promise<string | null> {
 
 async function gcpStatus(meta: InstallMetadata): Promise<void> {
   if (!meta.projectId) return;
+  if (!(await gcpPreflight({ cosign: false }))) return;
   const s = spinner();
   s.start("Resolving deployed resources + checking health...");
 
   const region = installRegion(meta);
-  const adminName = await resolveGcpResourceName(
+  const admin = await resolveGcpResourceName(
     gcloud,
     "run-service",
     "caelo-production-admin",
     meta.projectId,
     region,
   );
-  const sqlName = await resolveGcpResourceName(
+  const sql = await resolveGcpResourceName(
     gcloud,
     "sql-instance",
     "caelo-production-pg",
     meta.projectId,
   );
-  if (!adminName || !sqlName) {
-    s.stop(red("Could not resolve deployed resource names — is the install live?"));
+  if (admin.status !== "found") {
+    s.stop(red(lookupFailure(admin, "caelo-production-admin* Cloud Run service", "operator")));
+    return;
+  }
+  if (sql.status !== "found") {
+    s.stop(red(lookupFailure(sql, "caelo-production-pg* Cloud SQL instance", "operator")));
     return;
   }
+  const adminName = admin.name;
+  const sqlName = sql.name;
 
   const adminUri = await gcloud([
     "run",
@@ -366,9 +404,43 @@ interface UpgradeOpts {
    * `--install` or the picker (chooseInstalls).
    */
   readonly installId?: string;
+  /**
+   * #816 — the GCP project the install lives in. The upgrade attaches it
+   * from the project's ops bucket first (state-command.ts attachProject),
+   * so it works on a machine (or a fresh Cloud Shell) with no local copy.
+   */
+  readonly project?: string;
 }
 
 export async function upgradeCommand(opts: UpgradeOpts = {}): Promise<void> {
+  if (opts.project) {
+    if (!(await gcpPreflight({ cosign: false }))) {
+      process.exitCode = 1;
+      return;
+    }
+    const { attachProject } = await import("./state-command.js");
+    const attached = attachProject(opts.project);
+    let wanted: string | undefined;
+    try {
+      wanted = installFlag(process.argv);
+    } catch (e) {
+      log.error(red(e instanceof Error ? e.message : String(e)));
+      process.exit(2);
+    }
+    const selected = selectInstall(listInstalls(), wanted ?? attached);
+    if (!selected.ok) return exitNoInstall(selected.message);
+    if (selected.meta.installId !== attached) {
+      log.error(
+        red(
+          `--install ${wanted} is not the install in project ${opts.project} (${attached}). Drop one of the two flags.`,
+        ),
+      );
+      process.exit(2);
+    }
+    const meta = currentMetadata(selected.meta);
+    if (!(await upgradeInstall(meta.installId, meta, opts))) process.exitCode = 1;
+    return;
+  }
   if (opts.installId) {
     const selected = selectInstall(listInstalls(), opts.installId);
     if (!selected.ok) return exitNoInstall(selected.message);
@@ -493,6 +565,7 @@ async function upgradeInstall(
   if (!meta.projectId) return false;
   const provider = meta.provider;
   const projectId = meta.projectId;
+  if (!(await gcpPreflight({ cosign: opts.skipVerify !== true }))) return false;
 
   // #815 — the install's state lock, the one the in-app upgrader job,
   // `pulumi up` and `state migrate` take: a Cloud Shell upgrade must not
@@ -569,6 +642,7 @@ async function upgradeLockedInstall(
       infraContractVersion: INFRA_CONTRACT_VERSION,
       skipVerify: opts.skipVerify === true,
       skipHealthCheck: opts.skipHealthCheck === true,
+      credentials: "operator",
     },
     {
       run,
@@ -578,7 +652,7 @@ async function upgradeLockedInstall(
       verifyImages: verifyCosignAll,
       migrate: (imageRef) =>
         runMigrationsViaCloudRunJob({ projectId, region, imageOverride: imageRef }, { run }),
-      healthProbe: (plan) => gcpHealthProbe(run, http, plan),
+      healthProbe: (plan, context) => gcpHealthProbe(run, http, plan, context),
       recordDigests: async (digests) => {
         recorded = await recordRolledRelease(
           installId,
@@ -621,6 +695,7 @@ async function gcpHealthProbe(
   run: GcloudRunner,
   http: HttpFetch,
   plan: AppRollPlan,
+  context: { readonly accessJustChanged: boolean },
 ): Promise<{ ok: true; probe: HealthProbe } | { ok: false; error: string }> {
   const { install } = plan;
   const urls: Partial<Record<RollSlug, string>> = {};
@@ -644,7 +719,7 @@ async function gcpHealthProbe(
         adminServiceUrl: urls.admin as string,
         gatewayServiceUrl: urls.gateway as string,
       },
-      { run, http, sleep: realSleep },
+      { run, http, sleep: realSleep, accessJustChanged: context.accessJustChanged },
     ),
   };
 }
@@ -676,7 +751,12 @@ export function recordRolledRelease(
         readPassphrase: () => readSecret(installId, "pulumi-passphrase"),
       });
       const { writeImageDigestsToStack } = await import("./wizards/gcp-pulumi.js");
-      return writeImageDigestsToStack({ backend, provider, digests: d });
+      return writeImageDigestsToStack({
+        installRoot: installRoot(installId),
+        backend,
+        provider,
+        digests: d,
+      });
     },
     stackRecovery: (d) => stackConfigRecovery(installId, local ?? {}, provider, d),
     installJsonLocation: bucket
@@ -702,7 +782,7 @@ function stackConfigRecovery(
   digests: ImageDigests,
 ): string {
   const env = backendShellEnv(meta, installRoot(installId));
-  const cwd = `--cwd ${resolvePath(import.meta.dir, "../stacks", provider)} --stack ${GCP_STACK_ENV}`;
+  const cwd = `--cwd ${stackWorkDir(installRoot(installId), provider)} --stack ${GCP_STACK_ENV}`;
   const lines = Object.entries(digests).map(
     ([svc, d]) => `  ${env} pulumi config set ${cwd} caelo-${provider}:image-digest-${svc} ${d}`,
   );
@@ -730,12 +810,13 @@ async function backupOf(meta: InstallMetadata): Promise<boolean> {
     return false;
   }
   if (!meta.projectId) return false;
+  if (!(await gcpPreflight({ cosign: false }))) return false;
 
   const s = spinner();
   s.start("Resolving Cloud SQL instance + triggering on-demand backup...");
   const r = await takeSqlBackup(
     gcloud,
-    { projectId: meta.projectId, env: GCP_STACK_ENV },
+    { projectId: meta.projectId, env: GCP_STACK_ENV, credentials: "operator" },
     "backup",
   );
   if (!r.ok) {
@@ -776,21 +857,28 @@ export async function rotateSecretCommand(
   }
   if (!meta.projectId) return;
   const region = installRegion(meta);
+  if (!(await gcpPreflight({ cosign: false }))) {
+    process.exitCode = 1;
+    return;
+  }
 
   if (name === SEAL_KEY_SECRET) {
-    const gatewayService = await resolveGcpResourceName(
+    const gateway = await resolveGcpResourceName(
       gcloud,
       "run-service",
       "caelo-production-gateway",
       meta.projectId,
       region,
     );
-    if (!gatewayService) {
+    if (gateway.status !== "found") {
       log.error(
-        red("Could not find the caelo-production-gateway* Cloud Run service — nothing rotated"),
+        red(
+          `${lookupFailure(gateway, "caelo-production-gateway* Cloud Run service", "operator")} — nothing rotated`,
+        ),
       );
       return;
     }
+    const gatewayService = gateway.name;
     const target = { projectId: meta.projectId, region, env: GCP_STACK_ENV, gatewayService };
     const report = opts.finish
       ? await finishSealKeyRotation(target, {}, { now: opts.now === true })
@@ -807,23 +895,41 @@ export async function rotateSecretCommand(
   s.start(`Rotating ${secret}...`);
   const services: Partial<Record<"admin" | "gateway", string>> = {};
   for (const slug of ["admin", "gateway"] as const) {
-    const serviceName = await resolveGcpResourceName(
+    const service = await resolveGcpResourceName(
       gcloud,
       "run-service",
       `caelo-production-${slug}`,
       meta.projectId,
       region,
     );
-    if (!serviceName) {
-      s.stop(red(`Could not find caelo-production-${slug}* Cloud Run service — nothing rotated`));
+    if (service.status !== "found") {
+      s.stop(
+        red(
+          `${lookupFailure(service, `caelo-production-${slug}* Cloud Run service`, "operator")} — nothing rotated`,
+        ),
+      );
+      return;
+    }
+    services[slug] = service.name;
+  }
+  let sqlInstance: string | null = null;
+  if (rolesOfSecret(secret).length > 0) {
+    const sql = await resolveGcpResourceName(
+      gcloud,
+      "sql-instance",
+      "caelo-production-pg",
+      meta.projectId,
+    );
+    if (sql.status !== "found") {
+      s.stop(
+        red(
+          `${lookupFailure(sql, "caelo-production-pg* Cloud SQL instance", "operator")} — nothing rotated`,
+        ),
+      );
       return;
     }
-    services[slug] = serviceName;
+    sqlInstance = sql.name;
   }
-  const sqlInstance =
-    rolesOfSecret(secret).length > 0
-      ? await resolveGcpResourceName(gcloud, "sql-instance", "caelo-production-pg", meta.projectId)
-      : null;
   const report = await rotateRuntimeSecret(
     {
       projectId: meta.projectId,