@caelo-cms/provisioning 0.10.64 → 0.10.66

package/src/install-state.ts

@@ -78,6 +78,16 @@ export interface InstallMetadata {
    * installs that predate it.
    */
   imageDigests?: ImageDigests;
+  /**
+   * #1025 — the linux/amd64 platform manifest inside each index of
+   * `imageDigests`: the form Cloud Run records for a serving revision
+   * (upgrade/image-forms.ts). Recorded next to the index digests from the
+   * signed release manifest or the content-checked index, so the upgrader
+   * matches the serving images without a registry lookup. Belongs to
+   * `imageDigests`: dropped whenever those change without it. Absent on
+   * installs recorded before it; the upgrader then reads the index.
+   */
+  platformDigests?: ImageDigests;
   /**
    * #696 — the Caelo release (`0.11.0`) those digests belong to, recorded
    * next to them by `upgrade` and the upgrader job. The upgrader job reads
@@ -182,11 +192,17 @@ export function resumedMetadata(
   };
 }
 
-/** Record the image digests an install now runs (see {@link InstallMetadata.imageDigests}). */
+/**
+ * Record the image digests an install now runs (see
+ * {@link InstallMetadata.imageDigests}).
+ *
+ * @param platformDigests - their linux/amd64 forms (#1025), when known
+ */
 export function recordImageDigests(
   installId: string,
   digests: ImageDigests,
   releaseVersion?: string | null,
+  platformDigests?: ImageDigests | null,
 ): void {
   const meta = readMetadata(installId);
   if (!meta) throw new Error(`No install.json for install '${installId}'.`);
@@ -196,11 +212,14 @@ export function recordImageDigests(
   // refuses on none and judges downgrades by it).
   const same =
     meta.imageDigests?.admin === digests.admin && meta.imageDigests?.gateway === digests.gateway;
-  const { releaseVersion: previous, ...rest } = meta;
+  const { releaseVersion: previous, platformDigests: previousPlatform, ...rest } = meta;
   const version = releaseVersion ?? (same ? previous : undefined);
+  // The platform forms belong to their indexes just as the version does.
+  const platform = platformDigests ?? (same ? previousPlatform : undefined);
   writeMetadata(installId, {
     ...rest,
     imageDigests: { admin: digests.admin, gateway: digests.gateway },
+    ...(platform ? { platformDigests: { admin: platform.admin, gateway: platform.gateway } } : {}),
     ...(version ? { releaseVersion: version } : {}),
   });
 }

package/src/lifecycle.ts

@@ -48,6 +48,7 @@ import {
   provisionerServiceAccountEmail,
 } from "./provisioner-identity.js";
 import { assertRegionUnchanged } from "./regions.js";
+import type { ReleaseManifest } from "./release-manifest.js";
 import {
   type HttpFetch,
   ROTATABLE_SECRETS,
@@ -76,6 +77,7 @@ import {
 import { restoreCommand, takeSqlBackup } from "./upgrade/backup.js";
 import { findServiceUrl, lookupFailure, resolveGcpResourceName } from "./upgrade/gcp-lookup.js";
 import { createIapHealthProbe, type HealthProbe } from "./upgrade/health.js";
+import { createReleaseImageForms, platformDigestsOf } from "./upgrade/image-forms.js";
 import { recordRolledDigests } from "./upgrade/record-rolled.js";
 import {
   canonicalReleaseTag,
@@ -729,20 +731,35 @@ async function gcpHealthProbe(
  * install's install.json and Pulumi stack. The converge job (#819,
  * converge-job-gcp.ts) records through the same function as the terminal.
  *
+ * @param manifest - the verified manifest of the rolled release, when the
+ *   caller holds it: its signed platform digests save the index lookup
  * @returns whether every record was written; false fails the upgrade.
  */
-export function recordRolledRelease(
+export async function recordRolledRelease(
   installId: string,
   provider: "gcp" | "gcp-firebase",
   digests: ImageDigests,
   releaseVersion: string | null,
+  manifest?: ReleaseManifest | null,
 ): Promise<boolean> {
   // The local copy always names where the state lives (install-state.ts
   // stateStoreFor), and reading it cannot fail on bucket access.
   const local = readLocalMetadata(installId);
   const bucket = local?.stateBackend?.kind === "gcs" ? local.stateBackend.bucket : null;
+  // #1025 — record the linux/amd64 forms Cloud Run reports next to the
+  // indexes. Not knowing them is no failed upgrade: the upgrader job then
+  // reads the index from the release registry itself.
+  const platform = await platformDigestsOf(digests, createReleaseImageForms(), manifest);
+  if (!platform.ok) {
+    log.warn(
+      yellow(
+        `install.json records the release by its image indexes only; their linux/amd64 forms could not be read (${platform.message}).`,
+      ),
+    );
+  }
   return recordRolledDigests(digests, releaseVersion, {
-    recordInstall: (d, v) => recordImageDigests(installId, d, v),
+    recordInstall: (d, v) =>
+      recordImageDigests(installId, d, v, platform.ok ? platform.platformDigests : null),
     pinStack: async (d) => {
       const meta = readMetadata(installId);
       if (!meta) throw new Error(`no install.json for '${installId}'`);

package/src/release-manifest.test.ts

@@ -230,6 +230,16 @@ describe("ReleaseManifestSchema", () => {
     const ghsa = { ...MANIFEST, security: true, advisories: ["GHSA-2c2m-4x8q-wx9f"] };
     expect(ReleaseManifestSchema.safeParse(ghsa).success).toBe(true);
   });
+
+  it("#1025: platformDigests is optional, and must name all three images when present", () => {
+    // The canonical fixture has none: manifests signed before the field stay valid.
+    expect("platformDigests" in MANIFEST).toBe(false);
+    const platformDigests = { admin: d("1"), gateway: d("2"), ops: d("3") };
+    const parsed = ReleaseManifestSchema.safeParse({ ...MANIFEST, platformDigests });
+    expect(parsed.success && parsed.data.platformDigests).toEqual(platformDigests);
+    const partial = { ...MANIFEST, platformDigests: { admin: d("1"), gateway: d("2") } };
+    expect(ReleaseManifestSchema.safeParse(partial).success).toBe(false);
+  });
 });
 
 describe("compareImageDigests", () => {
@@ -241,8 +251,23 @@ describe("compareImageDigests", () => {
 
   it("names each mismatching or unresolved image, ignores absent ones", () => {
     expect(compareImageDigests(MANIFEST, { admin: d("f"), gateway: null })).toEqual([
-      { image: "admin", expected: d("a"), actual: d("f") },
-      { image: "gateway", expected: d("b"), actual: null },
+      { image: "admin", form: "index", expected: d("a"), actual: d("f") },
+      { image: "gateway", form: "index", expected: d("b"), actual: null },
+    ]);
+  });
+
+  it("#1025: compares the linux/amd64 platform digests the manifest signs", () => {
+    const withPlatform = {
+      ...MANIFEST,
+      platformDigests: { admin: d("1"), gateway: d("2"), ops: d("3") },
+    };
+    expect(compareImageDigests(withPlatform, {}, { admin: d("1"), ops: d("3") })).toEqual([]);
+    expect(compareImageDigests(withPlatform, {}, { gateway: d("9") })).toEqual([
+      { image: "gateway", form: "platform", expected: d("2"), actual: d("9") },
+    ]);
+    // A manifest without the field cannot vouch for a platform digest.
+    expect(compareImageDigests(MANIFEST, {}, { admin: d("1") })).toEqual([
+      { image: "admin", form: "platform", expected: null, actual: d("1") },
     ]);
   });
 });

package/src/release-manifest.ts

@@ -221,27 +221,40 @@ export async function fetchVerifiedReleaseManifest(
 /** One image whose resolved digest differs from the manifest's. */
 export interface DigestMismatch {
   readonly image: ReleaseImage;
-  readonly expected: string;
+  /** Which digest differs: the image index, or its linux/amd64 manifest (#1025). */
+  readonly form: "index" | "platform";
+  /** The manifest's digest; null when the manifest has no platform digests. */
+  readonly expected: string | null;
   readonly actual: string | null;
 }
 
 /**
  * Compare the digests a registry resolved for the release's tags with the
- * signed manifest. Only images present in `resolved` are compared, except
- * that an explicitly `null` entry (tag did not resolve) is a mismatch.
+ * signed manifest. Only images present in `resolved` (or `resolvedPlatform`)
+ * are compared, except that an explicitly `null` entry (tag did not
+ * resolve) is a mismatch.
  *
+ * @param resolvedPlatform - each index's linux/amd64 manifest, compared
+ *   with the manifest's `platformDigests` (#1025)
  * @returns every mismatch; empty means the registry serves the signed images.
  */
 export function compareImageDigests(
   manifest: ReleaseManifest,
   resolved: Partial<Record<ReleaseImage, string | null>>,
+  resolvedPlatform: Partial<Record<ReleaseImage, string | null>> = {},
 ): DigestMismatch[] {
   const out: DigestMismatch[] = [];
   for (const image of RELEASE_IMAGES) {
-    if (!(image in resolved)) continue;
-    const actual = resolved[image] ?? null;
-    if (actual !== manifest.digests[image]) {
-      out.push({ image, expected: manifest.digests[image], actual });
+    if (image in resolved) {
+      const actual = resolved[image] ?? null;
+      if (actual !== manifest.digests[image]) {
+        out.push({ image, form: "index", expected: manifest.digests[image], actual });
+      }
+    }
+    if (image in resolvedPlatform) {
+      const actual = resolvedPlatform[image] ?? null;
+      const expected = manifest.platformDigests?.[image] ?? null;
+      if (actual !== expected) out.push({ image, form: "platform", expected, actual });
     }
   }
   return out;

package/src/upgrade/__tests__/fixtures/admin-0.10.64-index.json

@@ -0,0 +1,28 @@
+{
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.oci.image.index.v1+json",
+  "manifests": [
+    {
+      "mediaType": "application/vnd.oci.image.manifest.v1+json",
+      "digest": "sha256:2e75b714662ec6b17d32666b546378e2850920950076c4e911deddc05596e656",
+      "size": 4673,
+      "platform": {
+        "architecture": "amd64",
+        "os": "linux"
+      }
+    },
+    {
+      "mediaType": "application/vnd.oci.image.manifest.v1+json",
+      "digest": "sha256:06f9defdb92e8252e9faab8fcb8fb340d3949d5cdec09e06b42905ade7ba30c0",
+      "size": 1113,
+      "annotations": {
+        "vnd.docker.reference.digest": "sha256:2e75b714662ec6b17d32666b546378e2850920950076c4e911deddc05596e656",
+        "vnd.docker.reference.type": "attestation-manifest"
+      },
+      "platform": {
+        "architecture": "unknown",
+        "os": "unknown"
+      }
+    }
+  ]
+}

package/src/upgrade/__tests__/fixtures/gateway-0.10.64-index.json

@@ -0,0 +1,28 @@
+{
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.oci.image.index.v1+json",
+  "manifests": [
+    {
+      "mediaType": "application/vnd.oci.image.manifest.v1+json",
+      "digest": "sha256:d24f2675d43e7891f217938c74475e9c31a64ffda0c96cf4fbb0a3f73e31367e",
+      "size": 3527,
+      "platform": {
+        "architecture": "amd64",
+        "os": "linux"
+      }
+    },
+    {
+      "mediaType": "application/vnd.oci.image.manifest.v1+json",
+      "digest": "sha256:a5a3180c2f8e971ad8650026b1e9734f2d7289d74c12179e194b8d48380e2738",
+      "size": 1113,
+      "annotations": {
+        "vnd.docker.reference.digest": "sha256:d24f2675d43e7891f217938c74475e9c31a64ffda0c96cf4fbb0a3f73e31367e",
+        "vnd.docker.reference.type": "attestation-manifest"
+      },
+      "platform": {
+        "architecture": "unknown",
+        "os": "unknown"
+      }
+    }
+  ]
+}

package/src/upgrade/__tests__/live-indexes.ts

@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * The v0.10.64 admin + gateway image indexes exactly as the release mirror
+ * serves them (fixtures/, byte for byte), and a fake `fetch` that serves
+ * them by digest — the shapes of #1025: each index lists its linux/amd64
+ * manifest plus one buildx attestation manifest, and Cloud Run records the
+ * linux/amd64 digest for a serving revision.
+ */
+
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+const fixture = (name: string): Uint8Array<ArrayBuffer> =>
+  new Uint8Array(readFileSync(join(import.meta.dir, "fixtures", name)));
+
+/** v0.10.64 admin: the index the signed manifest names, and what it lists. */
+export const ADMIN_INDEX =
+  "sha256:bcda1725a5390c910c56cfedb6f55bc4f38cebb0231e9e9b4297bb5c08aec3e0";
+export const ADMIN_AMD64 =
+  "sha256:2e75b714662ec6b17d32666b546378e2850920950076c4e911deddc05596e656";
+export const ADMIN_ATTESTATION =
+  "sha256:06f9defdb92e8252e9faab8fcb8fb340d3949d5cdec09e06b42905ade7ba30c0";
+/** v0.10.64 gateway. */
+export const GATEWAY_INDEX =
+  "sha256:fbb0be7a106f75c25e44bca1898fd56af27d11fa599659afe55e4f6ffaeca4a3";
+export const GATEWAY_AMD64 =
+  "sha256:d24f2675d43e7891f217938c74475e9c31a64ffda0c96cf4fbb0a3f73e31367e";
+
+/** The index bytes, by digest. */
+export const LIVE_INDEXES: Readonly<Record<string, Uint8Array<ArrayBuffer>>> = {
+  [ADMIN_INDEX]: fixture("admin-0.10.64-index.json"),
+  [GATEWAY_INDEX]: fixture("gateway-0.10.64-index.json"),
+};
+
+/** How the fake registry answers. */
+export interface FakeRegistryOptions {
+  /** Answer every request with this HTTP status instead. */
+  readonly status?: number;
+  /** Throw (network failure) instead of answering. */
+  readonly unreachable?: boolean;
+  /** Serve these bytes for every digest (a tampering registry). */
+  readonly tampered?: Uint8Array<ArrayBuffer>;
+  /** Index bytes to serve besides {@link LIVE_INDEXES}. */
+  readonly extra?: Readonly<Record<string, Uint8Array<ArrayBuffer>>>;
+}
+
+/** A `fetch` serving the mirror's `/v2/<repo>/manifests/<digest>`; records each URL in `calls`. */
+export function fakeRegistryFetch(calls: string[], opts: FakeRegistryOptions = {}): typeof fetch {
+  const impl = async (input: string | URL | Request): Promise<Response> => {
+    const url = String(input instanceof Request ? input.url : input);
+    calls.push(url);
+    if (opts.unreachable) throw new Error("getaddrinfo ENOTFOUND");
+    if (opts.status) return new Response("", { status: opts.status });
+    const digest = url.split("/manifests/")[1] ?? "";
+    const bytes = opts.tampered ?? opts.extra?.[digest] ?? LIVE_INDEXES[digest];
+    if (!bytes) return new Response("", { status: 404 });
+    return new Response(bytes, { status: 200 });
+  };
+  return impl as typeof fetch;
+}

package/src/upgrade/gcp-lookup.test.ts

@@ -1,10 +1,11 @@
 // SPDX-License-Identifier: MPL-2.0
 
 /**
- * findServingImage against a fake gcloud: the digest a revision was deployed
- * with is the release's index digest, Cloud Run's `status.imageDigest` the
- * platform manifest inside it (#860). Both must come back, so a recorded
- * index digest matches what serves.
+ * findServingImage against a fake gcloud. A revision's spec and
+ * `status.imageDigest` may name the release's index digest or the
+ * platform manifest inside it (#860); on live installs BOTH name the
+ * platform manifest (#1025), and only the service template keeps the index
+ * (findTemplateImageDigest). Every digest found must come back.
  */
 
 import { describe, expect, it } from "bun:test";
@@ -12,6 +13,7 @@ import type { GcloudResult } from "../gcloud.js";
 import type { GcloudRunner } from "../gcloud-retry.js";
 import {
   findServingImage,
+  findTemplateImageDigest,
   lookupFailure,
   resolveGcpResourceName,
   servingImageDigests,
@@ -108,15 +110,15 @@ const revision = (image: string, imageDigest?: string) =>
   ok(JSON.stringify({ spec: { containers: [{ image }] }, status: { imageDigest } }));
 
 describe("findServingImage", () => {
-  it("returns the deployed index digest and the platform digest Cloud Run resolved", async () => {
+  it("returns the spec's index digest and the platform digest Cloud Run resolved", async () => {
     const image = await findServingImage(
       gcloud(revision(`${REPO}@${INDEX}`, `${REPO}@${PLATFORM}`)),
       "acme",
       "europe-west1",
       "caelo-production-admin-x",
     );
-    expect(image).toEqual({ deployedDigest: INDEX, resolvedDigest: PLATFORM });
-    expect(servingImageDigests(image ?? { deployedDigest: null, resolvedDigest: null })).toEqual([
+    expect(image).toEqual({ specDigest: INDEX, resolvedDigest: PLATFORM });
+    expect(servingImageDigests(image ?? { specDigest: null, resolvedDigest: null })).toEqual([
       INDEX,
       PLATFORM,
     ]);
@@ -129,11 +131,26 @@ describe("findServingImage", () => {
       "europe-west1",
       "svc",
     );
-    expect(image).toEqual({ deployedDigest: null, resolvedDigest: PLATFORM });
+    expect(image).toEqual({ specDigest: null, resolvedDigest: PLATFORM });
+  });
+
+  it("regression #1025: the live shape, spec and status both name the platform manifest", async () => {
+    // caelo-production-admin-482044c-00015-2vc on searchviu-website, deployed
+    // as the v0.10.64 index admin@sha256:bcda… (the service template).
+    const live = `${REPO}@sha256:2e75b714662ec6b17d32666b546378e2850920950076c4e911deddc05596e656`;
+    const image = await findServingImage(
+      gcloud(revision(live, live)),
+      "acme",
+      "europe-west1",
+      "svc",
+    );
+    expect(servingImageDigests(image ?? { specDigest: null, resolvedDigest: null })).toEqual([
+      "sha256:2e75b714662ec6b17d32666b546378e2850920950076c4e911deddc05596e656",
+    ]);
   });
 
   it("one digest when both name the same image", () => {
-    expect(servingImageDigests({ deployedDigest: INDEX, resolvedDigest: INDEX })).toEqual([INDEX]);
+    expect(servingImageDigests({ specDigest: INDEX, resolvedDigest: INDEX })).toEqual([INDEX]);
   });
 
   it("null when no digest can be read or the output is not JSON", async () => {
@@ -157,3 +174,30 @@ describe("findServingImage", () => {
     ).toBeNull();
   });
 });
+
+describe("findTemplateImageDigest (#1025)", () => {
+  const service = (image: unknown) =>
+    ok(JSON.stringify({ spec: { template: { spec: { containers: [{ image }] } } } }));
+  const runner =
+    (r: GcloudResult): GcloudRunner =>
+    async () =>
+      r;
+
+  it("reads the index digest the template pins", async () => {
+    expect(
+      await findTemplateImageDigest(runner(service(`${REPO}@${INDEX}`)), "acme", "r", "svc"),
+    ).toBe(INDEX);
+  });
+
+  it("null for a tag, an odd shape, bad JSON or a failed describe", async () => {
+    for (const r of [
+      service(`${REPO}:latest`),
+      service(7),
+      ok("null"),
+      ok("not json"),
+      fail("denied"),
+    ]) {
+      expect(await findTemplateImageDigest(runner(r), "acme", "r", "svc")).toBeNull();
+    }
+  });
+});

package/src/upgrade/gcp-lookup.ts

@@ -157,28 +157,30 @@ export async function findCurrentRevision(
  * by (not the service template's: after a rollback the template names the
  * failed image while traffic stays on the previous revision).
  *
- * Why two digests (#860): Caelo's release images are OCI image *indexes*
- * (buildx adds a provenance/SBOM attestation manifest even to a
- * single-platform build), and the signed release manifest, install.json
- * and every deploy pin the index digest. Cloud Run's `status.imageDigest`
- * names what it resolved and pulled, the linux/amd64 *platform manifest*
- * inside that index, so comparing it with a recorded index digest never
- * matches. The digest the revision was deployed with
- * (`spec.containers[0].image`, `…@sha256:<index>`) is the form everything
- * else records; it is content-addressed, so the revision necessarily runs
- * that index's image.
+ * Which form these digests take (#860, corrected by #1025): Caelo's release
+ * images are OCI image *indexes* (buildx adds a provenance/SBOM attestation
+ * manifest even to a single-platform build), and the signed release
+ * manifest, install.json and every deploy pin the index digest. Cloud Run
+ * resolves that index to its linux/amd64 *platform manifest* and records
+ * the platform digest — in `status.imageDigest` AND, contrary to what #860
+ * assumed, in the revision's `spec.containers[0].image` too: a revision
+ * deployed as `admin@sha256:bcda…` (the v0.10.64 index) reads back as
+ * `admin@sha256:2e75…` in both fields (live installs, 2026-10-23). Only the
+ * service template keeps the index. So these digests are matched against a
+ * release's index AND the platform manifests inside it
+ * (upgrade/image-forms.ts), never against the index alone.
  */
 export interface ServingImage {
-  /** The digest the revision was deployed with; null when deployed by tag. */
-  readonly deployedDigest: string | null;
+  /** The digest the revision's `spec.containers[0].image` names; null when it names a tag. */
+  readonly specDigest: string | null;
   /** The digest Cloud Run resolved and runs (`status.imageDigest`); null when not reported. */
   readonly resolvedDigest: string | null;
 }
 
-/** Every digest `image` is known by, deployed form first. */
+/** Every digest `image` is known by, spec form first. */
 export function servingImageDigests(image: ServingImage): string[] {
   const out: string[] = [];
-  for (const d of [image.deployedDigest, image.resolvedDigest]) {
+  for (const d of [image.specDigest, image.resolvedDigest]) {
     if (d && !out.includes(d)) out.push(d);
   }
   return out;
@@ -231,12 +233,55 @@ export async function findServingImage(
   const digestOf = (ref: string | undefined): string | null =>
     ref === undefined ? null : (PINNED_DIGEST.exec(ref.trim())?.[1] ?? null);
   const image: ServingImage = {
-    deployedDigest: digestOf(parsed.data.spec?.containers?.[0]?.image),
+    specDigest: digestOf(parsed.data.spec?.containers?.[0]?.image),
     resolvedDigest: digestOf(parsed.data.status?.imageDigest),
   };
   return servingImageDigests(image).length > 0 ? image : null;
 }
 
+/** The fields of `gcloud run services describe --format=json` {@link findTemplateImageDigest} reads. */
+const ServiceJsonSchema = z.object({
+  spec: z
+    .object({
+      template: z
+        .object({
+          spec: z
+            .object({
+              containers: z.array(z.object({ image: z.string().optional() })).optional(),
+            })
+            .optional(),
+        })
+        .optional(),
+    })
+    .optional(),
+});
+
+/**
+ * The digest the service's template pins (the index a deploy named, #1025),
+ * or null when it cannot be read or names a tag. NOT what serves — after a
+ * rollback the template names the failed image — so only a hint for which
+ * release to check (upgrader-records.ts).
+ */
+export async function findTemplateImageDigest(
+  run: GcloudRunner,
+  projectId: string,
+  region: string,
+  serviceName: string,
+): Promise<string | null> {
+  const stdout = await describeServiceJson(run, projectId, region, serviceName);
+  if (stdout === null) return null;
+  let json: unknown;
+  try {
+    json = JSON.parse(stdout);
+  } catch {
+    return null;
+  }
+  const parsed = ServiceJsonSchema.safeParse(json);
+  if (!parsed.success) return null;
+  const ref = parsed.data.spec?.template?.spec?.containers?.[0]?.image;
+  return ref === undefined ? null : (PINNED_DIGEST.exec(ref.trim())?.[1] ?? null);
+}
+
 /** The service's own `run.app` URL. */
 export async function findServiceUrl(
   run: GcloudRunner,