@caelo-cms/provisioning 0.11.9 → 0.11.11

package/src/upgrade/health.test.ts

@@ -6,7 +6,7 @@ import type { HttpFetch } from "../runtime-secrets.js";
 import {
   createIapHealthProbe,
   evaluateAdminDeepHealth,
-  evaluateGatewayDeepHealth,
+  evaluatePublicServiceDeepHealth,
   type HealthTarget,
   healthOrigins,
   IAM_PROPAGATION_WINDOW_MS,
@@ -72,27 +72,29 @@ describe("evaluateAdminDeepHealth", () => {
   });
 });
 
-describe("evaluateGatewayDeepHealth", () => {
-  it("accepts a healthy gateway on the admin's version", () => {
-    expect(
-      evaluateGatewayDeepHealth(200, JSON.stringify({ ok: true, version: "0.11.0" }), "0.11.0"),
-    ).toEqual({ ok: true, detail: "gateway 0.11.0" });
+describe("evaluatePublicServiceDeepHealth", () => {
+  it("accepts a healthy gateway; its version is not public any more (#1326)", () => {
+    expect(evaluatePublicServiceDeepHealth("gateway", 200, JSON.stringify({ ok: true }))).toEqual({
+      ok: true,
+      detail: "gateway healthy",
+    });
   });
 
-  it("refuses a gateway on another version than the admin", () => {
-    const r = evaluateGatewayDeepHealth(
-      200,
-      JSON.stringify({ ok: true, version: "0.10.0" }),
-      "0.11.0",
-    );
-    expect(r.ok).toBe(false);
+  it("ignores a version an older gateway still reports", () => {
+    expect(
+      evaluatePublicServiceDeepHealth(
+        "gateway",
+        200,
+        JSON.stringify({ ok: true, version: "0.10.0" }),
+      ).ok,
+    ).toBe(true);
   });
 
   it("refuses a gateway that can't reach its database", () => {
-    const r = evaluateGatewayDeepHealth(
+    const r = evaluatePublicServiceDeepHealth(
+      "gateway",
       503,
-      JSON.stringify({ ok: false, version: "0.11.0", error: { kind: "DbUnreachable" } }),
-      "0.11.0",
+      JSON.stringify({ ok: false, error: { kind: "DbUnreachable" } }),
     );
     expect(r).toEqual({ ok: false, error: "gateway reports unhealthy (HTTP 503, DbUnreachable)" });
   });
@@ -103,7 +105,8 @@ const FIREBASE: HealthTarget = {
   projectId: "acme",
   domain: "acme.com",
   adminServiceUrl: "https://caelo-production-admin-aaa-xyz.a.run.app",
-  gatewayServiceUrl: "https://caelo-production-gateway-bbb-xyz.a.run.app",
+  gatewayServiceUrl: "https://caelo-production-data-gateway-ddd-xyz.a.run.app",
+  mcpAuthProxyServiceUrl: "https://caelo-production-gateway-bbb-xyz.a.run.app",
 };
 
 describe("healthOrigins", () => {
@@ -111,13 +114,15 @@ describe("healthOrigins", () => {
     expect(healthOrigins({ ...FIREBASE, provider: "gcp" })).toEqual({
       admin: "https://admin.acme.com",
       gateway: "https://acme.com",
+      mcpAuthProxy: "https://acme.com",
     });
   });
 
   it("uses the run.app URLs on gcp-firebase", () => {
     expect(healthOrigins(FIREBASE)).toEqual({
       admin: "https://caelo-production-admin-aaa-xyz.a.run.app",
-      gateway: "https://caelo-production-gateway-bbb-xyz.a.run.app",
+      gateway: "https://caelo-production-data-gateway-ddd-xyz.a.run.app",
+      mcpAuthProxy: "https://caelo-production-gateway-bbb-xyz.a.run.app",
     });
   });
 });
@@ -130,8 +135,12 @@ describe("createIapHealthProbe", () => {
     exitCode: 0,
   });
 
-  /** A fake network: signJwt + the two health endpoints. */
-  function network(answers: { admin: () => [number, string]; gateway: () => [number, string] }) {
+  /** A fake network: signJwt + the three health endpoints (the proxy answers healthy unless given). */
+  function network(answers: {
+    admin: () => [number, string];
+    gateway: () => [number, string];
+    proxy?: () => [number, string];
+  }) {
     const requests: { url: string; headers: Record<string, string>; body?: string }[] = [];
     const http: HttpFetch = async (url, init) => {
       requests.push({ url, headers: init.headers, ...(init.body ? { body: init.body } : {}) });
@@ -139,7 +148,9 @@ describe("createIapHealthProbe", () => {
         ? [200, JSON.stringify({ signedJwt: "signed.jwt" })]
         : url.includes("admin")
           ? answers.admin()
-          : answers.gateway();
+          : url.endsWith("/api/mcp/health")
+            ? (answers.proxy ?? (() => [200, JSON.stringify({ ok: true, version: "0.11.0" })]))()
+            : answers.gateway();
       return { ok: status < 400, status, text: async () => text };
     };
     return { http, requests };
@@ -148,7 +159,7 @@ describe("createIapHealthProbe", () => {
   it("signs an IAP credential for the admin origin and sends it to the admin only", async () => {
     const net = network({
       admin: () => [200, adminBody()],
-      gateway: () => [200, JSON.stringify({ ok: true, version: "0.11.0" })],
+      gateway: () => [200, JSON.stringify({ ok: true })],
     });
     const probe = createIapHealthProbe(FIREBASE, {
       run: gcloudOk,
@@ -160,7 +171,7 @@ describe("createIapHealthProbe", () => {
     expect(await probe.deep({ version: "0.11.0" })).toEqual({
       ok: true,
       detail:
-        "admin 0.11.0, migrations at 0262_tool_approval_branch.sql / 0040_x.sql; gateway 0.11.0",
+        "admin 0.11.0, migrations at 0262_tool_approval_branch.sql / 0040_x.sql; gateway healthy; MCP-Auth-Proxy healthy",
     });
     const sign = net.requests[0];
     expect(sign?.url).toBe(
@@ -180,9 +191,36 @@ describe("createIapHealthProbe", () => {
     expect(admin?.headers.Authorization).toBe("Bearer signed.jwt");
     const gateway = net.requests.find((r) => r.url.endsWith("/api/_caelo/health?deep=1"));
     expect(gateway?.url).toBe(
-      "https://caelo-production-gateway-bbb-xyz.a.run.app/api/_caelo/health?deep=1",
+      "https://caelo-production-data-gateway-ddd-xyz.a.run.app/api/_caelo/health?deep=1",
     );
     expect(gateway?.headers.Authorization).toBeUndefined();
+    // #1328 — the proxy, on the service that was the gateway, by version.
+    const proxy = net.requests.find((r) => r.url.endsWith("/api/mcp/health"));
+    expect(proxy?.url).toBe("https://caelo-production-gateway-bbb-xyz.a.run.app/api/mcp/health");
+    expect(proxy?.headers.Authorization).toBeUndefined();
+  });
+
+  it("#1328 — fails the deep check while the MCP-Auth-Proxy is unhealthy or does not answer", async () => {
+    const deepWith = (proxy: () => [number, string]) =>
+      createIapHealthProbe(FIREBASE, {
+        run: gcloudOk,
+        http: network({
+          admin: () => [200, adminBody()],
+          gateway: () => [200, JSON.stringify({ ok: true, version: "0.11.0" })],
+          proxy,
+        }).http,
+        sleep: async () => {},
+        accessJustChanged: false,
+        attempts: 1,
+      }).deep({ version: "0.11.0" });
+    expect(await deepWith(() => [503, JSON.stringify({ ok: false })])).toEqual({
+      ok: false,
+      error: "MCP-Auth-Proxy reports unhealthy (HTTP 503, no reason)",
+    });
+    expect(await deepWith(() => [404, "<html>Not Found</html>"])).toEqual({
+      ok: false,
+      error: "MCP-Auth-Proxy answered HTTP 404 without JSON",
+    });
   });
 
   it("retries while the new revisions warm up, then gives the last error", async () => {
@@ -194,7 +232,7 @@ describe("createIapHealthProbe", () => {
           ? [503, adminBody({ ok: false, migrations: migrations(1) })]
           : [200, adminBody()];
       },
-      gateway: () => [200, JSON.stringify({ ok: true, version: "0.11.0" })],
+      gateway: () => [200, JSON.stringify({ ok: true })],
     });
     const probe = createIapHealthProbe(FIREBASE, {
       run: gcloudOk,

package/src/upgrade/health.ts

@@ -11,9 +11,12 @@
  *   - admin   `GET /_caelo/health?deep=1` → `{ ok, version, migrations }`:
  *             both databases reachable as the expected roles, and every
  *             migration the image ships recorded as applied.
- *   - gateway `GET /api/_caelo/health?deep=1` → `{ ok, version }`: its
- *             database logins work. Under `/api/` because that is the only
+ *   - gateway `GET /api/_caelo/health?deep=1` → `{ ok }` (no version since
+ *             #1326): its database login works. Under `/api/` because that is the only
  *             path the gcp load balancer routes to the gateway.
+ *   - MCP-Auth-Proxy `GET /api/mcp/health` → `{ ok }` (#1328, no version
+ *             either): it has no database, so it answers once it runs; under
+ *             `/api/mcp/`, the prefix the gcp load balancer routes to it.
  *
  * Both answer with no secrets and no raw database errors. The admin sits
  * behind IAP, so the probe signs an IAP credential as the install's MCP
@@ -134,29 +137,41 @@ export function evaluateAdminDeepHealth(
   };
 }
 
-/** Judge the gateway's deep health answer against the version the admin runs. */
-export function evaluateGatewayDeepHealth(
+/**
+ * Judge a public service's deep health answer: the gateway's, or the
+ * MCP-Auth-Proxy's (#1328). Since #1326 the public services report no
+ * release version (a public version string names the flaws to try), so
+ * this proves only that the service answers healthy — for the gateway,
+ * that its database login works. Which release it runs the roll itself
+ * establishes: `update --image` with the pinned digest succeeded and
+ * traffic moved to that revision (app-roll.ts), or the roll already failed.
+ *
+ * @param service how the errors name it (`gateway`, `MCP-Auth-Proxy`)
+ */
+export function evaluatePublicServiceDeepHealth(
+  service: string,
   status: number,
   text: string,
-  expectedVersion: string,
 ): HealthOutcome {
-  const body = parseJson<{ ok?: boolean; version?: string; error?: { kind?: string } }>(text);
-  if (!body) return { ok: false, error: `gateway answered HTTP ${status} without JSON` };
+  const body = parseJson<{ ok?: boolean; error?: { kind?: string } }>(text);
+  if (!body) return { ok: false, error: `${service} answered HTTP ${status} without JSON` };
   if (status !== 200 || body.ok !== true) {
     return {
       ok: false,
-      error: `gateway reports unhealthy (HTTP ${status}, ${body.error?.kind ?? "no reason"})`,
+      error: `${service} reports unhealthy (HTTP ${status}, ${body.error?.kind ?? "no reason"})`,
     };
   }
-  if (body.version !== expectedVersion) {
-    return {
-      ok: false,
-      error: `gateway runs ${body.version ?? "an unknown version"}, the admin runs ${expectedVersion}`,
-    };
-  }
-  return { ok: true, detail: `gateway ${body.version}` };
+  return { ok: true, detail: `${service} healthy` };
 }
 
+/**
+ * The MCP-Auth-Proxy's health path (apps/mcp-auth-proxy server.ts). Under
+ * `/api/mcp/` because on gcp that is the prefix the load balancer routes to
+ * the proxy; it answers before any other routing, also while remote MCP is
+ * off.
+ */
+export const MCP_AUTH_PROXY_HEALTH_PATH = "/api/mcp/health";
+
 /** Where the probe reaches the install. */
 export interface HealthTarget {
   readonly provider: "gcp" | "gcp-firebase";
@@ -166,20 +181,32 @@ export interface HealthTarget {
   readonly adminServiceUrl: string;
   /** The gateway service's `run.app` URL (public on gcp-firebase). */
   readonly gatewayServiceUrl: string;
+  /** The MCP-Auth-Proxy service's `run.app` URL (public on gcp-firebase). */
+  readonly mcpAuthProxyServiceUrl: string;
 }
 
 /**
  * The origins the probe calls. On gcp the admin's IAP lives on the load
- * balancer (`admin.<domain>`) and the gateway's ingress is load-balancer-only
- * (`<domain>/api/*`); on gcp-firebase both `run.app` URLs are reachable — the
- * admin's through Cloud Run's own IAP, the gateway's publicly.
+ * balancer (`admin.<domain>`), and the gateway's and the proxy's ingress is
+ * load-balancer-only (`<domain>/api/*`, `<domain>/api/mcp/*`); on
+ * gcp-firebase every `run.app` URL is reachable — the admin's through Cloud
+ * Run's own IAP, the others publicly.
  */
-export function healthOrigins(target: HealthTarget): { admin: string; gateway: string } {
+export function healthOrigins(target: HealthTarget): {
+  admin: string;
+  gateway: string;
+  mcpAuthProxy: string;
+} {
   return target.provider === "gcp"
-    ? { admin: `https://admin.${target.domain}`, gateway: `https://${target.domain}` }
+    ? {
+        admin: `https://admin.${target.domain}`,
+        gateway: `https://${target.domain}`,
+        mcpAuthProxy: `https://${target.domain}`,
+      }
     : {
         admin: new URL(target.adminServiceUrl).origin,
         gateway: new URL(target.gatewayServiceUrl).origin,
+        mcpAuthProxy: new URL(target.mcpAuthProxyServiceUrl).origin,
       };
 }
 
@@ -437,7 +464,12 @@ export function createIapHealthProbe(target: HealthTarget, deps: HealthProbeDeps
     if (!a.ok) return a;
     const gateway = await get(`${origins.gateway}/api/_caelo/health?deep=1`, {});
     if (!gateway.reached) return { ok: false, error: `gateway unreachable: ${gateway.error}` };
-    const g = evaluateGatewayDeepHealth(gateway.status, gateway.text, a.version);
-    return g.ok ? { ok: true, detail: `${a.detail}; ${g.detail}` } : g;
+    const g = evaluatePublicServiceDeepHealth("gateway", gateway.status, gateway.text);
+    if (!g.ok) return g;
+    // #1328 — the proxy holds no database login: healthy once it answers.
+    const proxy = await get(`${origins.mcpAuthProxy}${MCP_AUTH_PROXY_HEALTH_PATH}`, {});
+    if (!proxy.reached) return { ok: false, error: `MCP-Auth-Proxy unreachable: ${proxy.error}` };
+    const p = evaluatePublicServiceDeepHealth("MCP-Auth-Proxy", proxy.status, proxy.text);
+    return p.ok ? { ok: true, detail: `${a.detail}; ${g.detail}; ${p.detail}` } : p;
   }
 }

package/src/upgrade/image-forms.test.ts

@@ -184,6 +184,40 @@ describe("platformDigestsOf", () => {
     expect(calls).toEqual([]);
   });
 
+  it("#1328: carries the MCP-Auth-Proxy's signed form, and skips a proxy digest install.json lacks", async () => {
+    const calls: string[] = [];
+    const forms = createReleaseImageForms(fakeRegistryFetch(calls));
+    const m = {
+      version: "0.12.0",
+      digests: { admin: D("a"), gateway: D("b"), ops: D("c"), "mcp-auth-proxy": D("d") },
+      platformDigests: { admin: D("1"), gateway: D("2"), ops: D("3"), "mcp-auth-proxy": D("4") },
+    };
+    expect(
+      await platformDigestsOf(
+        { admin: D("a"), gateway: D("b"), "mcp-auth-proxy": D("d") },
+        forms,
+        m,
+      ),
+    ).toEqual({
+      ok: true,
+      platformDigests: { admin: D("1"), gateway: D("2"), "mcp-auth-proxy": D("4") },
+    });
+    // Recorded before the proxy existed: no proxy form, and nothing looked up for it.
+    expect(await platformDigestsOf({ admin: D("a"), gateway: D("b") }, forms, m)).toEqual({
+      ok: true,
+      platformDigests: { admin: D("1"), gateway: D("2") },
+    });
+    expect(calls).toEqual([]);
+  });
+
+  it("#1328: a release from before the proxy ships no proxy image form", async () => {
+    const forms = createReleaseImageForms(fakeRegistryFetch([]));
+    const m = { version: "0.11.0", digests: { admin: D("a"), gateway: D("b"), ops: D("c") } };
+    const r = await forms.of(m, "mcp-auth-proxy");
+    expect(r).toMatchObject({ ok: false, transient: false });
+    expect(r.ok ? "" : r.message).toContain("ships no mcp-auth-proxy image");
+  });
+
   it("says why when an index cannot be read", async () => {
     const forms = createReleaseImageForms(fakeRegistryFetch([], { status: 503 }));
     const r = await platformDigestsOf({ admin: ADMIN_INDEX, gateway: GATEWAY_INDEX }, forms);

package/src/upgrade/image-forms.ts

@@ -30,7 +30,7 @@
 import { z } from "zod";
 import type { ImageDigests } from "../install-state.js";
 import type { ReleaseManifest } from "../release-manifest.js";
-import type { RollSlug } from "./app-roll.js";
+import { ROLL_ORDER, type RollSlug } from "./app-roll.js";
 import { fetchRegistryManifest, IMAGE_DIGEST, RELEASE_REGISTRY } from "./release-images.js";
 
 /** The digests one release image is known by. */
@@ -178,12 +178,20 @@ export function createReleaseImageForms(fetchImpl: typeof fetch = fetch): Releas
   return {
     ofIndex,
     of: async (m, slug) => {
+      const index = m.digests[slug];
+      if (!index) {
+        // #1328 — a release from before the MCP-Auth-Proxy signs no image of it.
+        return {
+          ok: false,
+          transient: false,
+          message: `release ${m.version} ships no ${slug} image`,
+        };
+      }
       const signed = m.platformDigests?.[slug];
       if (signed) {
-        const index = m.digests[slug];
         return { ok: true, digests: [index, ...(signed === index ? [] : [signed])], amd64: signed };
       }
-      return ofIndex(slug, m.digests[slug]);
+      return ofIndex(slug, index);
     },
   };
 }
@@ -205,22 +213,29 @@ export async function platformDigestsOf(
   | { readonly ok: false; readonly message: string }
 > {
   const out: Partial<Record<RollSlug, string>> = {};
-  for (const slug of ["admin", "gateway"] as const) {
+  for (const slug of ROLL_ORDER) {
+    const index = digests[slug];
+    // #1328 — digests recorded before the MCP-Auth-Proxy existed carry none.
+    if (!index) continue;
     const found =
-      manifest && manifest.digests[slug] === digests[slug]
+      manifest && manifest.digests[slug] === index
         ? await forms.of(manifest, slug)
-        : await forms.ofIndex(slug, digests[slug]);
+        : await forms.ofIndex(slug, index);
     if (!found.ok) return { ok: false, message: found.message };
     if (!found.amd64) {
       return {
         ok: false,
-        message: `the ${slug} image index ${digests[slug]} lists no single linux/amd64 manifest`,
+        message: `the ${slug} image index ${index} lists no single linux/amd64 manifest`,
       };
     }
     out[slug] = found.amd64;
   }
   const { admin, gateway } = out;
+  const proxy = out["mcp-auth-proxy"];
   return admin && gateway
-    ? { ok: true, platformDigests: { admin, gateway } }
+    ? {
+        ok: true,
+        platformDigests: { admin, gateway, ...(proxy ? { "mcp-auth-proxy": proxy } : {}) },
+      }
     : { ok: false, message: "the platform digests could not be read" };
 }

package/src/upgrade/release-images.test.ts

@@ -70,7 +70,11 @@ describe("resolveRegistryDigest", () => {
     const missing = fakeFetch(() => new Response(null, { status: 404, statusText: "Not Found" }));
     expect(
       await resolveRegistryDigest("ghcr.io", "caelo-cms/admin", "9.9.9", missing.impl),
-    ).toEqual({ ok: false, reason: "HTTP 404 Not Found" });
+    ).toEqual({ ok: false, reason: "HTTP 404 Not Found", notFound: true });
+    // Only a 404 is the verdict "no such tag" (#1328); an outage is not.
+    const down = fakeFetch(() => new Response(null, { status: 503 }));
+    const outage = await resolveRegistryDigest("ghcr.io", "caelo-cms/admin", "9.9.9", down.impl);
+    expect(outage.ok === false && "notFound" in outage).toBe(false);
     const bad = fakeFetch(
       () => new Response(null, { status: 200, headers: { "docker-content-digest": "md5:x" } }),
     );

package/src/upgrade/release-images.ts

@@ -17,11 +17,15 @@ import { IMAGE_DIGEST } from "@caelo-cms/shared";
 import { confirm, isCancel, log, spinner } from "@clack/prompts";
 import { green, red } from "kleur/colors";
 
-/** The two Caelo services shipped as release images. */
-export type ReleaseService = "admin" | "gateway";
+/** The Caelo services shipped as release images (#1328 added the MCP-Auth-Proxy). */
+export type ReleaseService = "admin" | "gateway" | "mcp-auth-proxy";
 
-/** The services every install runs, in roll order (admin first, then gateway). */
-export const RELEASE_SERVICES: readonly ReleaseService[] = ["admin", "gateway"];
+/**
+ * The services every install runs, in roll order: the admin first, then the
+ * gateway, then the MCP-Auth-Proxy (its own service since #1328, so a hole in
+ * a visitor handler never shares a process with the operators' tokens).
+ */
+export const RELEASE_SERVICES: readonly ReleaseService[] = ["admin", "gateway", "mcp-auth-proxy"];
 
 /** GHCR — the canonical registry (release-images.yml), used by self-hosted Compose. */
 export const GHCR_REGISTRY = "ghcr.io";
@@ -116,7 +120,13 @@ export function releaseTag(opts: {
   return "latest";
 }
 
-export type DigestResult = { ok: true; digest: string } | { ok: false; reason: string };
+/**
+ * `notFound`: the registry answered that the tag does not exist (HTTP 404)
+ * — a verdict, unlike an outage, a refused token or a malformed answer.
+ */
+export type DigestResult =
+  | { ok: true; digest: string }
+  | { ok: false; reason: string; notFound?: true };
 
 /** Manifest media types: single-arch manifests and multi-arch indexes, Docker + OCI. */
 const MANIFEST_ACCEPT = [
@@ -155,7 +165,13 @@ export async function resolveRegistryDigest(
         headers: { Accept: MANIFEST_ACCEPT, Authorization: `Bearer ${token.token}` },
       });
     }
-    if (!res.ok) return { ok: false, reason: `HTTP ${res.status} ${res.statusText}` };
+    if (!res.ok) {
+      return {
+        ok: false,
+        reason: `HTTP ${res.status} ${res.statusText}`,
+        ...(res.status === 404 ? { notFound: true as const } : {}),
+      };
+    }
     const header = res.headers.get("docker-content-digest");
     const digest = header ? canonicalDigest(header) : null;
     if (!digest) {