@caelo-cms/provisioning 0.10.55 → 0.10.57

package/src/upgrade/converge.ts

@@ -7,17 +7,20 @@
  * CLI-generated runtime secrets, the stack's IAM/CDN/firewall invariants,
  * MCP access through IAP, public_role's Cloud SQL role memberships (#633),
  * the gateway's and public_role's database logins, the operator-access sync
- * job, the quality-audit job (#875), and dropping bindings nothing reads any
- * more.
+ * job, the quality-audit job (#875), the remote-MCP load-balancer rules and
+ * identity env (#806), and dropping bindings nothing reads any more.
  *
- * It needs the rights of whoever provisioned the install, so only
- * `cms-provision upgrade` (the operator's own identity) runs it — never the
- * in-cloud upgrader job, which only app-rolls and refuses while
- * {@link needsConverge} says the install is behind.
+ * It needs the rights of whoever provisioned the install: `cms-provision
+ * upgrade` runs it as the operator's own identity, and the converge job
+ * (#819, converge-job.ts) runs the same upgrade as the install's
+ * `caelo-provisioner` account. Never the in-cloud upgrader job, which only
+ * app-rolls and refuses while {@link needsConverge} says the install is
+ * behind.
  *
  * The keyless-provisioner step (#696, provisioner-identity.ts) is converge
  * too; it needs the operator's local install dir, so the CLI injects it as
- * {@link ConvergeDeps.provisionerIdentity}.
+ * {@link ConvergeDeps.provisionerIdentity} (the converge job, which IS the
+ * provisioner, has nothing to move and passes none).
  *
  * Some steps must sit between app-roll steps (a role migration 0248 creates
  * gets its password after migrations, before the gateway rolls onto it), so
@@ -26,6 +29,7 @@
  */
 
 import { ensureAuditJob, resolveAdminImage } from "../audit-job-setup.js";
+import { ensureConvergeJob } from "../converge-job-setup.js";
 import { findSqlInstance, planPublicRoleSwitch } from "../gateway-credentials.js";
 import type { GcloudRunner, Sleep } from "../gcloud-retry.js";
 import { upgraderServiceAccountEmail } from "../gcp-names.js";
@@ -33,6 +37,7 @@ import type { InstallMetadata } from "../install-state.js";
 import { ensureMcpIapAccess, type IapResource } from "../mcp-iap.js";
 import { ensureOperatorAccessSync, resolveOperatorAccessTarget } from "../operator-access.js";
 import { revokePublicRoleMemberships } from "../public-role-isolation.js";
+import { ensureRemoteMcpEdge, resolveRemoteMcpIdentity } from "../remote-mcp.js";
 import {
   ensureGatewayServiceAccount,
   ensureGeneratedSecrets,
@@ -44,8 +49,11 @@ import { selfCommand } from "../self-command.js";
 import {
   AUDIT_JOB_ENV_VAR,
   AUDIT_RESULTS_BUCKET_ENV_VAR,
+  type CloudRunEnvVar,
+  CONVERGE_JOB_ENV_VAR,
   MCP_ENV_VAR,
   OPERATOR_ACCESS_JOB_ENV_VAR,
+  remoteMcpIdentityEnv,
   UPGRADER_JOB_ENV_VAR,
 } from "../stack-contract.js";
 import {
@@ -308,6 +316,65 @@ export function startConverge(
     leaveUntouched.push(AUDIT_JOB_ENV_VAR, AUDIT_RESULTS_BUCKET_ENV_VAR);
   }
 
+  async function remoteMcpEdge(): Promise<void> {
+    // #806 — on `gcp` the load balancer routes the remote-MCP discovery
+    // paths to the gateway and Cloud Armor lets agent traffic through on
+    // /api/mcp/* (remote-mcp-edge.ts). Only remote MCP depends on it, so a
+    // failure warns.
+    if (install.provider !== "gcp") return;
+    const s = r.step("Ensuring the load balancer's remote-MCP routes and Cloud Armor rule...");
+    const edge = await ensureRemoteMcpEdge(
+      {
+        provider: install.provider,
+        projectId: install.projectId,
+        env: install.env,
+        domain: install.domain,
+      },
+      { ...runDeps, ...(deps.http ? { http: deps.http } : {}) },
+    );
+    if (edge.ok) {
+      for (const line of edge.done) r.info(`  ${line}`);
+      s.done("Remote-MCP load-balancer rules ready");
+      return;
+    }
+    s.warned(`Remote-MCP load-balancer rules not set up: ${edge.error}`);
+    r.warn(
+      "The upgrade continues; remote MCP (Settings → MCP → Remote) cannot be reached through the site's domain until this succeeds. Re-run upgrade once the cause is fixed.",
+    );
+    incomplete = true;
+  }
+
+  async function remoteMcpIdentity(): Promise<Record<RollSlug, CloudRunEnvVar[]> | null> {
+    // #806 — CAELO_ADMIN_URL / CAELO_MCP_PUBLIC_URL / CAELO_IAP_AUDIENCE
+    // name identities the env contract cannot (stack-contract.ts
+    // RemoteMcpIdentity). Resolved here, read-only, and set by the rolls
+    // (AppRollHooks.afterMigrations `extraEnv`), so they ride the new
+    // revisions. A failure leaves the vars as the services have them and
+    // warns: only remote MCP depends on them.
+    const s = r.step("Resolving the remote-MCP URLs and IAP audience...");
+    const resolved = await resolveRemoteMcpIdentity(
+      {
+        provider: install.provider,
+        projectId: install.projectId,
+        region: install.region,
+        env: install.env,
+        domain: install.domain,
+        services,
+      },
+      runDeps,
+    );
+    if (resolved.ok) {
+      s.done(`Remote MCP can be set up under Settings → MCP (${resolved.identity.mcpPublicUrl})`);
+      return remoteMcpIdentityEnv(resolved.identity);
+    }
+    s.warned(`Remote-MCP env not resolved: ${resolved.error}`);
+    r.warn(
+      "The upgrade continues; Settings → MCP reports remote MCP as not prepared on this install until this succeeds. Re-run upgrade once the cause is fixed.",
+    );
+    incomplete = true;
+    return null;
+  }
+
   /** The admin's IAP resource: the service on gcp-firebase, the LB backend on gcp. */
   async function adminIapResource(): Promise<IapResource | null> {
     if (install.provider === "gcp-firebase") {
@@ -394,6 +461,54 @@ export function startConverge(
     leaveUntouched.push(UPGRADER_JOB_ENV_VAR);
   }
 
+  async function convergeJob(): Promise<void> {
+    // #819 — the job "Install update" starts for the next release that
+    // changes the cloud setup, on this release's ops image. After
+    // migrations: its database login gets the role migration 0267 creates.
+    // A failure only costs in-Caelo converges (the terminal path stays), so
+    // it warns.
+    const s = r.step("Ensuring the converge job (cloud-setup updates from Settings → Updates)...");
+    const resolved = deps.opsImageRef
+      ? await resolveUpgraderTarget(
+          {
+            projectId: install.projectId,
+            region: install.region,
+            env: install.env,
+            opsImageRef: deps.opsImageRef,
+          },
+          { run },
+        )
+      : { ok: false as const, error: "the release's caelo-ops image could not be resolved" };
+    const cj = resolved.ok
+      ? await ensureConvergeJob(resolved.target, runDeps)
+      : { ok: false as const, done: [], error: resolved.error };
+    for (const step of cj.done) r.info(`  ${step}`);
+    if (cj.ok) {
+      s.done("Converge job ready (updates that change the cloud setup install from the admin)");
+      return;
+    }
+    if (cj.reason === "no-provisioner") {
+      // #883 review — optional for contract 9: an install without a
+      // provisioner (installed before it existed) has nothing the job could
+      // run as, and creating one is a privileged identity no update should
+      // mint. It stays fully converged; its admin keeps no
+      // CAELO_CONVERGE_JOB and so routes cloud-setup releases to the
+      // terminal, saying so on Settings → Updates.
+      s.done("No converge job: this install has no caelo-provisioner to run it as");
+      r.info(
+        "Updates that change the cloud setup keep installing from Cloud Shell or the computer you installed from; Settings → Updates says so when one comes.",
+      );
+      leaveUntouched.push(CONVERGE_JOB_ENV_VAR);
+      return;
+    }
+    s.warned(`Converge job not set up: ${cj.error}`);
+    r.warn(
+      "The upgrade continues; releases that change the cloud setup keep installing from a terminal until this succeeds.",
+    );
+    incomplete = true;
+    leaveUntouched.push(CONVERGE_JOB_ENV_VAR);
+  }
+
   async function operatorAccess(): Promise<void> {
     // The job that keeps IAP in step with the user list (operator-access.ts).
     // After migrations: its database user gets the role migration 0239
@@ -447,6 +562,7 @@ export function startConverge(
       await mcpAccess();
       await urlMapRedirectsAccess();
       await auditJob();
+      await remoteMcpEdge();
       return true;
     },
 
@@ -454,8 +570,14 @@ export function startConverge(
       async afterMigrations() {
         if (!(await databaseLogins())) return { ok: false };
         await upgraderJob();
+        await convergeJob();
         await operatorAccess();
-        return { ok: true, leaveUntouched: [...leaveUntouched] };
+        const extraEnv = await remoteMcpIdentity();
+        return {
+          ok: true,
+          leaveUntouched: [...leaveUntouched],
+          ...(extraEnv ? { extraEnv } : {}),
+        };
       },
       async beforeRoll(slug) {
         if (slug !== "gateway" || !publicRole) return { ok: true };

package/src/upgrade/gcp-lookup.test.ts

@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * findServingImage against a fake gcloud: the digest a revision was deployed
+ * with is the release's index digest, Cloud Run's `status.imageDigest` the
+ * platform manifest inside it (#860). Both must come back, so a recorded
+ * index digest matches what serves.
+ */
+
+import { describe, expect, it } from "bun:test";
+import type { GcloudResult } from "../gcloud.js";
+import type { GcloudRunner } from "../gcloud-retry.js";
+import { findServingImage, servingImageDigests } from "./gcp-lookup.js";
+
+const INDEX = `sha256:${"a".repeat(64)}`;
+const PLATFORM = `sha256:${"b".repeat(64)}`;
+const REPO = "europe-west1-docker.pkg.dev/caelo-website/caelo-cms-images/admin";
+
+const ok = (stdout: string): GcloudResult => ({ ok: true, stdout, stderr: "", exitCode: 0 });
+
+function gcloud(revision: GcloudResult): GcloudRunner {
+  return async (args) => {
+    if (args.includes("services")) return ok("caelo-production-admin-x-00042-abc\n");
+    return revision;
+  };
+}
+
+const revision = (image: string, imageDigest?: string) =>
+  ok(JSON.stringify({ spec: { containers: [{ image }] }, status: { imageDigest } }));
+
+describe("findServingImage", () => {
+  it("returns the deployed index digest and the platform digest Cloud Run resolved", async () => {
+    const image = await findServingImage(
+      gcloud(revision(`${REPO}@${INDEX}`, `${REPO}@${PLATFORM}`)),
+      "acme",
+      "europe-west1",
+      "caelo-production-admin-x",
+    );
+    expect(image).toEqual({ deployedDigest: INDEX, resolvedDigest: PLATFORM });
+    expect(servingImageDigests(image ?? { deployedDigest: null, resolvedDigest: null })).toEqual([
+      INDEX,
+      PLATFORM,
+    ]);
+  });
+
+  it("a revision deployed by tag is known by its resolved digest only", async () => {
+    const image = await findServingImage(
+      gcloud(revision(`${REPO}:0.10.52`, `${REPO}@${PLATFORM}`)),
+      "acme",
+      "europe-west1",
+      "svc",
+    );
+    expect(image).toEqual({ deployedDigest: null, resolvedDigest: PLATFORM });
+  });
+
+  it("one digest when both name the same image", () => {
+    expect(servingImageDigests({ deployedDigest: INDEX, resolvedDigest: INDEX })).toEqual([INDEX]);
+  });
+
+  it("null when no digest can be read or the output is not JSON", async () => {
+    expect(
+      await findServingImage(gcloud(revision(`${REPO}:latest`)), "acme", "europe-west1", "svc"),
+    ).toBeNull();
+    expect(
+      await findServingImage(gcloud(ok("not json")), "acme", "europe-west1", "svc"),
+    ).toBeNull();
+    // JSON of an unexpected shape reads as unknown instead of throwing.
+    for (const odd of ["null", "[]", '{"spec":{"containers":{"image":1}}}', '{"status":"x"}']) {
+      expect(await findServingImage(gcloud(ok(odd)), "acme", "europe-west1", "svc")).toBeNull();
+    }
+    expect(
+      await findServingImage(
+        gcloud({ ok: false, stdout: "", stderr: "denied", exitCode: 1 }),
+        "acme",
+        "europe-west1",
+        "svc",
+      ),
+    ).toBeNull();
+  });
+});

package/src/upgrade/gcp-lookup.ts

@@ -7,6 +7,7 @@
  * by a different identity in the in-cloud upgrader job (epic #696).
  */
 
+import { z } from "zod";
 import type { GcloudRunner } from "../gcloud-retry.js";
 
 /**
@@ -102,16 +103,57 @@ export async function findCurrentRevision(
 }
 
 /**
- * The image digest the revision serving traffic runs (not the service
- * template's: after a rollback the template names the failed image while
- * traffic stays on the previous revision).
+ * The image the revision serving traffic runs, by every digest it is known
+ * by (not the service template's: after a rollback the template names the
+ * failed image while traffic stays on the previous revision).
+ *
+ * Why two digests (#860): Caelo's release images are OCI image *indexes*
+ * (buildx adds a provenance/SBOM attestation manifest even to a
+ * single-platform build), and the signed release manifest, install.json
+ * and every deploy pin the index digest. Cloud Run's `status.imageDigest`
+ * names what it resolved and pulled, the linux/amd64 *platform manifest*
+ * inside that index, so comparing it with a recorded index digest never
+ * matches. The digest the revision was deployed with
+ * (`spec.containers[0].image`, `…@sha256:<index>`) is the form everything
+ * else records; it is content-addressed, so the revision necessarily runs
+ * that index's image.
  */
-export async function findServingImageDigest(
+export interface ServingImage {
+  /** The digest the revision was deployed with; null when deployed by tag. */
+  readonly deployedDigest: string | null;
+  /** The digest Cloud Run resolved and runs (`status.imageDigest`); null when not reported. */
+  readonly resolvedDigest: string | null;
+}
+
+/** Every digest `image` is known by, deployed form first. */
+export function servingImageDigests(image: ServingImage): string[] {
+  const out: string[] = [];
+  for (const d of [image.deployedDigest, image.resolvedDigest]) {
+    if (d && !out.includes(d)) out.push(d);
+  }
+  return out;
+}
+
+const PINNED_DIGEST = /@(sha256:[0-9a-f]{64})$/;
+
+/** The fields of `gcloud run revisions describe --format=json` the lookup reads. */
+const RevisionJsonSchema = z.object({
+  spec: z
+    .object({ containers: z.array(z.object({ image: z.string().optional() })).optional() })
+    .optional(),
+  status: z.object({ imageDigest: z.string().optional() }).optional(),
+});
+
+/**
+ * The serving revision's {@link ServingImage}, or null when the revision or
+ * any digest of it cannot be read.
+ */
+export async function findServingImage(
   run: GcloudRunner,
   projectId: string,
   region: string,
   serviceName: string,
-): Promise<string | null> {
+): Promise<ServingImage | null> {
   const revision = await findCurrentRevision(run, projectId, region, serviceName);
   if (!revision) return null;
   const r = await run([
@@ -123,11 +165,26 @@ export async function findServingImageDigest(
     region,
     "--project",
     projectId,
-    "--format=value(status.imageDigest)",
+    "--format=json",
   ]);
   if (!r.ok) return null;
-  const m = /@(sha256:[0-9a-f]{64})\s*$/.exec(r.stdout.trim());
-  return m?.[1] ?? null;
+  let json: unknown;
+  try {
+    json = JSON.parse(r.stdout);
+  } catch {
+    return null;
+  }
+  // gcloud output is a boundary (CLAUDE.md §4): an unexpected shape (`null`,
+  // an array, a non-string image) reads as "unknown" instead of throwing.
+  const parsed = RevisionJsonSchema.safeParse(json);
+  if (!parsed.success) return null;
+  const digestOf = (ref: string | undefined): string | null =>
+    ref === undefined ? null : (PINNED_DIGEST.exec(ref.trim())?.[1] ?? null);
+  const image: ServingImage = {
+    deployedDigest: digestOf(parsed.data.spec?.containers?.[0]?.image),
+    resolvedDigest: digestOf(parsed.data.status?.imageDigest),
+  };
+  return servingImageDigests(image).length > 0 ? image : null;
 }
 
 /** The service's own `run.app` URL. */

package/src/upgrade/record-rolled.test.ts

@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * recordRolledDigests (#860): a CLI upgrade whose records could not be
+ * written used to exit 0 with a warning, leaving install.json naming the
+ * previous release. It now reports failure (the command exits non-zero)
+ * with the exact fix, and never skips one record because another failed.
+ */
+
+import { describe, expect, it } from "bun:test";
+import type { ImageDigests } from "../install-state.js";
+import { type RecordRolledDeps, recordRolledDigests } from "./record-rolled.js";
+
+const DIGESTS: ImageDigests = {
+  admin: `sha256:${"a".repeat(64)}`,
+  gateway: `sha256:${"b".repeat(64)}`,
+};
+
+function deps(overrides: Partial<RecordRolledDeps> = {}) {
+  const lines: { level: string; message: string }[] = [];
+  const calls: string[] = [];
+  const d: RecordRolledDeps = {
+    recordInstall: (_digests, version) => {
+      calls.push(`install ${version}`);
+    },
+    pinStack: async () => {
+      calls.push("stack");
+      return { removedOverrides: [] };
+    },
+    stackRecovery: () => "  pulumi config set …",
+    installJsonLocation: "gs://acme-caelo-ops/install/install.json",
+    log: {
+      info: (message) => lines.push({ level: "info", message }),
+      warn: (message) => lines.push({ level: "warn", message }),
+      error: (message) => lines.push({ level: "error", message }),
+    },
+    ...overrides,
+  };
+  return { d, lines, calls };
+}
+
+describe("recordRolledDigests", () => {
+  it("records install.json and the stack config", async () => {
+    const { d, calls, lines } = deps();
+    expect(await recordRolledDigests(DIGESTS, "0.10.53", d)).toBe(true);
+    expect(calls).toEqual(["install 0.10.53", "stack"]);
+    expect(lines.filter((l) => l.level === "error")).toEqual([]);
+  });
+
+  it("regression #860: a failed install.json write fails the upgrade, with the exact fix", async () => {
+    const { d, calls, lines } = deps({
+      recordInstall: () => {
+        throw new Error("Could not write gs://acme-caelo-ops/install/install.json: 403");
+      },
+    });
+    expect(await recordRolledDigests(DIGESTS, "0.10.53", d)).toBe(false);
+    const error = lines.find((l) => l.level === "error")?.message ?? "";
+    expect(error).toContain("gs://acme-caelo-ops/install/install.json");
+    expect(error).toContain(DIGESTS.admin);
+    expect(error).toContain('"releaseVersion": "0.10.53"');
+    // The stack config is still pinned.
+    expect(calls).toEqual(["stack"]);
+  });
+
+  it("regression #865: with no known version, the manual fix removes the stale releaseVersion", async () => {
+    const { d, lines } = deps({
+      recordInstall: () => {
+        throw new Error("403");
+      },
+    });
+    expect(await recordRolledDigests(DIGESTS, null, d)).toBe(false);
+    const error = lines.find((l) => l.level === "error")?.message ?? "";
+    expect(error).toContain(DIGESTS.admin);
+    expect(error).not.toContain('"releaseVersion":');
+    expect(error).toContain('delete the "releaseVersion" field');
+  });
+
+  it("with a known version, the manual fix sets it and says nothing about deleting it", async () => {
+    const { d, lines } = deps({
+      recordInstall: () => {
+        throw new Error("403");
+      },
+    });
+    await recordRolledDigests(DIGESTS, "0.10.53", d);
+    const error = lines.find((l) => l.level === "error")?.message ?? "";
+    expect(error).not.toContain("delete the");
+  });
+
+  it("a failed stack-config pin fails the upgrade too, with the commands to run", async () => {
+    const { d, lines } = deps({
+      pinStack: async () => {
+        throw new Error("passphrase missing");
+      },
+    });
+    expect(await recordRolledDigests(DIGESTS, "0.10.53", d)).toBe(false);
+    expect(lines.find((l) => l.level === "error")?.message).toContain("pulumi config set");
+  });
+
+  it("an unknown release version is recorded as none, with a warning, not a failure", async () => {
+    const { d, calls, lines } = deps();
+    expect(await recordRolledDigests(DIGESTS, null, d)).toBe(true);
+    expect(calls).toEqual(["install null", "stack"]);
+    expect(lines.some((l) => l.level === "warn")).toBe(true);
+  });
+});

package/src/upgrade/record-rolled.ts

@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * After a CLI upgrade rolled: record the digests that now serve, so nothing
+ * re-deploys an older release — install.json (what a wizard re-run deploys
+ * and what the in-cloud upgrader judges the next update by) and the Pulumi
+ * stack config (what a plain `pulumi up` deploys).
+ *
+ * The roll already succeeded, so a failed write never undoes it, and one
+ * failing never skips the other. But it is a failed upgrade, not a warning
+ * (#860, CLAUDE.md §2 no fallbacks): a run that exits 0 while install.json
+ * still names the previous release leaves every later update judging
+ * against a stale record. So each failure prints the exact manual fix and
+ * the result makes the command exit non-zero.
+ */
+
+import type { ImageDigests } from "../install-state.js";
+
+/** Where the records go; the real ones are wired in lifecycle.ts. */
+export interface RecordRolledDeps {
+  /** Write install.json (recordImageDigests). Throws when it cannot. */
+  readonly recordInstall: (digests: ImageDigests, releaseVersion: string | null) => void;
+  /** Pin the digests in the Pulumi stack config. Throws when it cannot. */
+  readonly pinStack: (
+    digests: ImageDigests,
+  ) => Promise<{ readonly removedOverrides: readonly string[] }>;
+  /** The commands that pin `digests` in the stack by hand. */
+  readonly stackRecovery: (digests: ImageDigests) => string;
+  /** Where install.json lives, for the manual fix (`gs://…/install.json` or a path). */
+  readonly installJsonLocation: string;
+  readonly log: {
+    readonly info: (message: string) => void;
+    readonly warn: (message: string) => void;
+    readonly error: (message: string) => void;
+  };
+}
+
+/**
+ * Record the rolled `digests` (and `releaseVersion`, when known).
+ *
+ * @returns whether every record was written; false means the upgrade must
+ *   exit non-zero — the fix for each failure has been printed.
+ */
+export async function recordRolledDigests(
+  digests: ImageDigests,
+  releaseVersion: string | null,
+  deps: RecordRolledDeps,
+): Promise<boolean> {
+  const { log } = deps;
+  const reason = (e: unknown) => (e instanceof Error ? e.message.split("\n")[0] : String(e));
+  if (!releaseVersion) {
+    log.warn(
+      "Could not tell which Caelo release these images are (no version tag points at them), so install.json records none; in-product updates stay unavailable until an upgrade to a released version.",
+    );
+  }
+  let ok = true;
+  try {
+    deps.recordInstall(digests, releaseVersion);
+  } catch (e) {
+    ok = false;
+    const fields = {
+      imageDigests: digests,
+      ...(releaseVersion ? { releaseVersion } : {}),
+    };
+    // With no known version the fix must REMOVE the recorded one: left in
+    // place, the previous release's version would sit next to the new
+    // digests, and the upgrader (which matches records by digest only)
+    // would judge the next update's downgrade / minFromVersion against it.
+    const versionFix = releaseVersion
+      ? ""
+      : '\nAlso delete the "releaseVersion" field from that file: it names the previous release, not the one that serves now.';
+    log.error(
+      `The new release serves, but its digests could not be recorded in ${deps.installJsonLocation} (${reason(e)}). Set these fields there before anything else, or a re-run of the installer deploys another release and in-product updates refuse to run:\n${JSON.stringify(fields, null, 2)}${versionFix}`,
+    );
+  }
+  try {
+    const { removedOverrides } = await deps.pinStack(digests);
+    log.info("Pinned the rolled image digests in the Pulumi stack config.");
+    if (removedOverrides.length > 0) {
+      log.warn(
+        `Removed ${removedOverrides.join(", ")} from the stack config: those image overrides win over the digest pins, so a later \`pulumi up\` would have rolled back to them.`,
+      );
+    }
+  } catch (e) {
+    ok = false;
+    log.error(
+      `Could not pin the rolled digests in the Pulumi stack config (${reason(e)}). Before any manual \`pulumi up\`, run:\n${deps.stackRecovery(digests)}`,
+    );
+  }
+  return ok;
+}