@caelo-cms/provisioning 0.11.9 → 0.11.11

package/src/compose.ts

@@ -3,9 +3,9 @@
 /**
  * P14 — production docker-compose.yml generator.
  *
- * Emits the production stack: Postgres + Caddy + the two Caelo services
- * (admin, gateway). Idempotent: same input →
- * byte-identical output. The CLI's `init` / `up` / `upgrade` sub-commands
+ * Emits the production stack: Postgres + Caddy + the Caelo services
+ * (admin, gateway and — #1328 — the MCP-Auth-Proxy). Idempotent: same
+ * input → byte-identical output. The CLI's `init` / `up` / `upgrade` sub-commands
  * write this to `.caelo/docker-compose.yml`.
  *
  * #696 — the admin and gateway run the PREBUILT, cosign-verified release
@@ -27,23 +27,39 @@
  * every fresh `docker compose up` fail.
  */
 
-import { CADDY_REDIRECTS_FILE } from "@caelo-cms/shared";
+import { CADDY_REDIRECTS_FILE, SELF_HOSTED_CA_FILE, verifyFullQuery } from "@caelo-cms/shared";
 import { caddyRedirectReloadCommand } from "./caddy-redirects.js";
 import { DEFAULT_MAX_CONNECTIONS, stackPoolBudget } from "./pool-budget-stack.js";
+import {
+  SELF_HOSTED_DB_HOST,
+  SELF_HOSTED_TLS_DIR,
+  SELF_HOSTED_TLS_FILES,
+} from "./self-hosted-db-tls.js";
 import {
   ADMIN_MEMORY_DEFAULT,
+  GATEWAY_ENVIRONMENT_ENV,
   GATEWAY_TRUSTED_PROXY_HOPS,
+  MCP_AUTH_PROXY_TRUSTED_PROXY_HOPS,
   memoryQuantityMiB,
+  parseSchedulerIntervalMinutes,
   publicSiteUrl,
+  REMOTE_MCP_MODE,
+  REMOTE_MCP_MODE_ENV,
   TRUSTED_PROXY_HOPS_ENV,
 } from "./stack-contract.js";
 import { ghcrRepository, isImageDigest, type ReleaseService } from "./upgrade/release-images.js";
 
-/** The admin + gateway release images, each `ghcr.io/caelo-cms/<svc>@sha256:…`. */
+/** The release images, each `ghcr.io/caelo-cms/<svc>@sha256:…`. */
 export interface ReleaseImages {
   readonly mode: "release";
   readonly admin: string;
   readonly gateway: string;
+  /**
+   * #1328 — the MCP-Auth-Proxy. Absent only for a release recorded before
+   * the proxy shipped (a rollback target in release.json): its Compose file
+   * then has no proxy service, exactly as that release ran.
+   */
+  readonly "mcp-auth-proxy"?: string;
 }
 
 /**
@@ -70,6 +86,8 @@ export const COMPOSE_POOL_BUDGET = stackPoolBudget("self-hosted", {
   maxConnections: DEFAULT_MAX_CONNECTIONS["self-hosted"],
   adminMaxInstances: 1,
   gatewayMaxInstances: 1,
+  // #1329 — one staging gateway container (P3b-2) next to the live one.
+  stagingGatewayMaxInstances: 1,
 });
 
 /** The budget's env as Compose `environment:` lines for `service`. */
@@ -102,11 +120,20 @@ export const MEDIA_ROOT = "/data/media";
 /** The gateway's HTTP port inside the Compose network. */
 export const COMPOSE_GATEWAY_PORT = 8090;
 
+/** #1328 — the MCP-Auth-Proxy's HTTP port inside the Compose network. */
+export const COMPOSE_MCP_AUTH_PROXY_PORT = 8095;
+
+/** #1328 — the Compose service name of the MCP-Auth-Proxy. */
+export const COMPOSE_MCP_AUTH_PROXY_SERVICE = "caelo-mcp-auth-proxy";
+
 /**
- * The Compose services that run Caelo code — what `restore` stops while it
- * rewrites the databases, and what the health checks probe. Workers run
- * inside the admin (CLAUDE.md §11.B Tier 5); there is no orchestrator or
- * runner service.
+ * The Compose services that touch the databases — what `backup` and
+ * `restore` stop while they read or rewrite them. Workers run inside the
+ * admin (CLAUDE.md §11.B Tier 5); there is no orchestrator or runner
+ * service. The MCP-Auth-Proxy (#1328) is deliberately not here: it holds no
+ * database login, so it keeps running, and a release from before it has no
+ * such service for `up` to start. Its health is checked wherever the
+ * install starts (self-hosted-start.ts) or rolls (self-hosted-upgrade.ts).
  */
 export const COMPOSE_APP_SERVICES = ["caelo-admin", "caelo-gateway"] as const;
 
@@ -117,6 +144,7 @@ export const COMPOSE_APP_SERVICES = ["caelo-admin", "caelo-gateway"] as const;
  */
 export const COMPOSE_ADMIN_UPSTREAM = `caelo-admin:${COMPOSE_ADMIN_PORT}`;
 export const COMPOSE_GATEWAY_UPSTREAM = `caelo-gateway:${COMPOSE_GATEWAY_PORT}`;
+export const COMPOSE_MCP_AUTH_PROXY_UPSTREAM = `${COMPOSE_MCP_AUTH_PROXY_SERVICE}:${COMPOSE_MCP_AUTH_PROXY_PORT}`;
 
 /**
  * The Caddy image, pinned to the release the generated Caddyfile is
@@ -192,6 +220,8 @@ export function ghcrImageDigest(service: ReleaseService, ref: string): string |
 export interface RolePasswords {
   readonly admin: string;
   readonly public: string;
+  /** #1329 — public_staging_role's, the staging gateway's login on cms_public_staging. */
+  readonly publicStaging: string;
 }
 
 export interface ComposeSpec {
@@ -213,6 +243,15 @@ export interface ComposeSpec {
    * the gateway never gets it.
    */
   readonly internalSecret: string;
+  /**
+   * #1338 — SHA-256 (hex) of the database server certificate the Postgres
+   * service mounts (`.caelo/postgres-tls/server.crt`). It is part of the
+   * service's environment so a renewed certificate changes the service's
+   * definition and `docker compose up -d` recreates Postgres: a bind-mounted
+   * file whose content changes does not, and Postgres would keep serving
+   * the old certificate until verify-full clients refuse it.
+   */
+  readonly dbTlsServerCertSha256: string;
   readonly anthropicApiKey?: string;
   readonly resendApiKey?: string;
   readonly diskSize: string;
@@ -223,8 +262,14 @@ export interface ComposeSpec {
    * busy host never OOM-kills the admin at the limit.
    */
   readonly adminMemory?: string;
-  /** Which admin + gateway images run (see {@link ComposeImages}). */
+  /** Which Caelo images run (see {@link ComposeImages}). */
   readonly images: ComposeImages;
+  /**
+   * #1140 — the `schedulerIntervalMinutes` knob: how often the scheduler
+   * container calls the admin's tick (1–15, default 5; stack-contract.ts
+   * SCHEDULER_INTERVAL_MINUTES).
+   */
+  readonly schedulerIntervalMinutes?: number;
   /**
    * The host ports Caddy listens on (`80` / `443` for a public install —
    * ACME's HTTP and TLS-ALPN challenges need exactly those).
@@ -274,6 +319,11 @@ export function generateDockerCompose(spec: ComposeSpec): string {
   if (spec.internalSecret.length < 32) {
     throw new Error("internalSecret is shorter than 32 characters (.caelo/internal-secret.json)");
   }
+  if (!/^[0-9a-f]{64}$/.test(spec.dbTlsServerCertSha256)) {
+    throw new Error(
+      "dbTlsServerCertSha256 is not a SHA-256 hex digest of .caelo/postgres-tls/server.crt",
+    );
+  }
   return `# SPDX-License-Identifier: MPL-2.0
 # Generated by cms-provision — do not edit; re-run \`bunx cms-provision up\`.
 # Domain: ${spec.domain}
@@ -281,10 +331,16 @@ export function generateDockerCompose(spec: ComposeSpec): string {
 name: ${spec.project}
 
 services:
-  postgres:
+  ${SELF_HOSTED_DB_HOST}:
     image: postgres:16-alpine
-    # The pool budget (COMPOSE_POOL_BUDGET) is computed from this number.
-    command: ["postgres", "-c", "max_connections=${DEFAULT_MAX_CONNECTIONS["self-hosted"]}"]
+    # #1338 — TLS with the install's own certificate for the host name
+    # \`${SELF_HOSTED_DB_HOST}\` (self-hosted-db-tls.ts); every client connects with
+    # verify-full. Postgres refuses a key file it does not own, and a bind
+    # mount keeps the host's owner, so the entrypoint copies the pair to a
+    # container-local path owned by postgres first.
+    entrypoint: ${POSTGRES_TLS_ENTRYPOINT}
+    # The pool budget (COMPOSE_POOL_BUDGET) is computed from max_connections.
+    command: ["postgres", "-c", "max_connections=${DEFAULT_MAX_CONNECTIONS["self-hosted"]}", "-c", "ssl=on", "-c", "ssl_cert_file=${POSTGRES_TLS_RUNTIME_DIR}/server.crt", "-c", "ssl_key_file=${POSTGRES_TLS_RUNTIME_DIR}/server.key", "-c", "ssl_min_protocol_version=TLSv1.2"]
     restart: unless-stopped
     environment:
       POSTGRES_PASSWORD: "${escapeYaml(spec.postgresPassword)}"
@@ -292,9 +348,14 @@ services:
       POSTGRES_DB: caelo
       ADMIN_ROLE_PASSWORD: "${escapeYaml(spec.rolePasswords.admin)}"
       PUBLIC_ROLE_PASSWORD: "${escapeYaml(spec.rolePasswords.public)}"
+      PUBLIC_STAGING_ROLE_PASSWORD: "${escapeYaml(spec.rolePasswords.publicStaging)}"
+      # Recreates Postgres when its certificate is renewed (see the spec).
+      CAELO_DB_TLS_SERVER_CERT_SHA256: "${spec.dbTlsServerCertSha256}"
 ${postgresAgent(spec.agent)}    volumes:
       - caelo-pg:/var/lib/postgresql/data
       - ./bootstrap.sh:/docker-entrypoint-initdb.d/00_bootstrap.sh:ro
+      - ./${SELF_HOSTED_TLS_DIR}/${SELF_HOSTED_TLS_FILES.serverCert}:${POSTGRES_TLS_MOUNT_DIR}/server.crt:ro
+      - ./${SELF_HOSTED_TLS_DIR}/${SELF_HOSTED_TLS_FILES.serverKey}:${POSTGRES_TLS_MOUNT_DIR}/server.key:ro
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U caelo"]
       interval: 10s
@@ -317,7 +378,8 @@ ${postgresAgent(spec.agent)}    volumes:
 ${adminService(spec, adminMemoryMiB, env)}
 
 ${gatewayService(spec)}
-
+${mcpAuthProxyService(spec.images)}
+${schedulerService(spec)}
 volumes:
   caelo-pg:
   caelo-caddy-data:
@@ -369,10 +431,11 @@ function serviceRuntime(
       ...volumes(["../..:/app", ...mounts]),
     ].join("\n");
   }
-  const digest = ghcrImageDigest(service, images[service]);
+  const ref = images[service] ?? "";
+  const digest = ghcrImageDigest(service, ref);
   if (!digest) {
     throw new Error(
-      `${service} image ${JSON.stringify(images[service])} is not ${ghcrRepository(service)}@sha256:<digest>`,
+      `${service} image ${JSON.stringify(ref)} is not ${ghcrRepository(service)}@sha256:<digest>`,
     );
   }
   // The image's own CMD starts the service; no source is mounted.
@@ -398,6 +461,7 @@ ${serviceRuntime(
     `./${BOOTSTRAP_DIR}:/app/.caelo`,
     `./output:${SITE_OUTPUT_ROOT}`,
     `./${MEDIA_DIR}:${MEDIA_ROOT}`,
+    DATABASE_CA_MOUNT,
   ],
 )}
     restart: unless-stopped
@@ -407,6 +471,8 @@ ${serviceRuntime(
     environment:
       ADMIN_DATABASE_URL: "${dbUrl("admin_role", spec.rolePasswords.admin, "cms_admin")}"
       PUBLIC_ADMIN_DATABASE_URL: "${dbUrl("admin_role", spec.rolePasswords.admin, "cms_public")}"
+      # #1329 — the staging database's plugin schemas and projection.
+      PUBLIC_STAGING_ADMIN_DATABASE_URL: "${dbUrl("admin_role", spec.rolePasswords.admin, "cms_public_staging")}"
       CAELO_SECRET_KEK: "${escapeYaml(spec.caeloSecretKek)}"
       CAELO_INTERNAL_SECRET: "${escapeYaml(spec.internalSecret)}"
       CAELO_SITE_URL: "${escapeYaml(selfHostedSiteUrl(spec.domain, spec.ports.https))}"
@@ -422,7 +488,7 @@ ${env("RESEND_API_KEY", spec.resendApiKey)}`;
 
 function gatewayService(spec: ComposeSpec): string {
   return `  caelo-gateway:
-${serviceRuntime(spec.images, "gateway", "bun run apps/api-gateway/src/server.ts")}
+${serviceRuntime(spec.images, "gateway", "bun run apps/api-gateway/src/server.ts", [DATABASE_CA_MOUNT])}
     restart: unless-stopped
     depends_on:
       postgres: { condition: service_healthy }
@@ -430,23 +496,126 @@ ${serviceRuntime(spec.images, "gateway", "bun run apps/api-gateway/src/server.ts
       # public_role on cms_public is its only database login (#613, #635,
       # CLAUDE.md §2: the public side has no path to the authoring database).
       PUBLIC_DATABASE_URL: "${dbUrl("public_role", spec.rolePasswords.public, "cms_public")}"
+      # #1329 — the live site's gateway: it refuses anything of the staging data.
+      ${GATEWAY_ENVIRONMENT_ENV}: "live"
       GATEWAY_PORT: "${COMPOSE_GATEWAY_PORT}"
       # Caddy is the one proxy in front (client-ip.ts in the gateway).
       ${TRUSTED_PROXY_HOPS_ENV}: "${GATEWAY_TRUSTED_PROXY_HOPS["self-hosted"]}"
 ${poolEnvLines("gateway")}
-      # #806 — no remote MCP here: it rides Google IAP, which this install
-      # does not have. CAELO_ADMIN_URL / CAELO_MCP_PUBLIC_URL /
-      # CAELO_OPERATOR_OAUTH_CLIENT_FILE stay unset, so the gateway's /api/mcp/*
-      # answers 503 remote_mcp_not_configured and Settings → MCP says
-      # "not available on this install".
       NODE_ENV: production`;
 }
 
-/** A compose-network database URL, password URL-encoded and YAML-escaped. */
+/**
+ * #1328 — the MCP-Auth-Proxy: remote MCP's sign-in broker, its own
+ * container so it never shares a process with visitor handlers. Remote MCP
+ * needs the admin behind Google IAP, which a Compose install does not have,
+ * so it runs with `CAELO_REMOTE_MCP=unavailable` and answers every request
+ * but its health check with one fixed 404 (E14 a of #763: the service
+ * exists on every provider, its attack surface one branch). Caddy serves it
+ * on its own host (`mcp.<domain>`, caddy.ts), never on the site's origin.
+ *
+ * It gets no database URL, no secret and no mount: there is nothing it
+ * could read. The container is locked down on top — read-only root
+ * filesystem, every capability dropped, no privilege escalation — because
+ * it is the Caelo service that holds operators' Google tokens wherever
+ * remote MCP runs. Empty for a release recorded before the proxy shipped
+ * ({@link ReleaseImages}).
+ */
+function mcpAuthProxyService(images: ComposeImages): string {
+  if (images.mode === "release" && images["mcp-auth-proxy"] === undefined) return "";
+  return `
+  ${COMPOSE_MCP_AUTH_PROXY_SERVICE}:
+${serviceRuntime(images, "mcp-auth-proxy", "bun run apps/mcp-auth-proxy/src/server.ts")}
+    restart: unless-stopped
+    read_only: true
+    tmpfs: ["/tmp"]
+    cap_drop: ["ALL"]
+    security_opt: ["no-new-privileges:true"]
+    environment:
+      CAELO_PROVIDER: "self-hosted"
+      ${REMOTE_MCP_MODE_ENV}: "${REMOTE_MCP_MODE["self-hosted"]}"
+      MCP_AUTH_PROXY_PORT: "${COMPOSE_MCP_AUTH_PROXY_PORT}"
+      # Caddy is the one proxy in front (@caelo-cms/shared client-address.ts).
+      ${TRUSTED_PROXY_HOPS_ENV}: "${MCP_AUTH_PROXY_TRUSTED_PROXY_HOPS["self-hosted"]}"
+      NODE_ENV: production
+`;
+}
+
+/** #1140 — the Compose service that calls the admin's scheduler tick. */
+export const COMPOSE_SCHEDULER_SERVICE = "caelo-scheduler";
+
+/** Where the admin image (and the dev checkout) ships the scheduler clock. */
+export const SCHEDULER_CLOCK_SCRIPT = "packages/admin-core/src/scheduler/clock.ts";
+
+/** The admin's tick endpoint on the Compose network. */
+export const COMPOSE_SCHEDULER_TICK_URL = `http://${COMPOSE_ADMIN_UPSTREAM}/api/internal/scheduler/tick`;
+
+/**
+ * #1140 — the scheduler: the self-hosted counterpart of the Cloud Scheduler
+ * job on Google Cloud. Every `schedulerIntervalMinutes` it POSTs the admin's
+ * `/api/internal/scheduler/tick` with an internal token (scope
+ * `scheduler.tick`); the admin runs whatever its due-table says is due. The
+ * clock only calls, it does no work itself, so the admin's background work
+ * runs inside a request on every provider.
+ *
+ * It runs the admin image with another entry point (dependency-free, node
+ * built-ins only) and holds only the internal secret it signs with: no
+ * database URL, no KEK, no mount, read-only root filesystem, every
+ * capability dropped. A rollback to a release from before #1140 renders it
+ * too; that admin image has no clock script, so the container exits and
+ * Docker restarts it with back-off — that release runs its work in-process.
+ */
+function schedulerService(spec: ComposeSpec): string {
+  const minutes = parseSchedulerIntervalMinutes(spec.schedulerIntervalMinutes);
+  const runtime =
+    spec.images.mode === "dev"
+      ? serviceRuntime(spec.images, "admin", `bun run ${SCHEDULER_CLOCK_SCRIPT}`)
+      : [
+          serviceRuntime(spec.images, "admin", ""),
+          `    command: ["bun", "--bun", "/app/${SCHEDULER_CLOCK_SCRIPT}"]`,
+        ].join("\n");
+  return `
+  ${COMPOSE_SCHEDULER_SERVICE}:
+${runtime}
+    restart: unless-stopped
+    read_only: true
+    tmpfs: ["/tmp"]
+    cap_drop: ["ALL"]
+    security_opt: ["no-new-privileges:true"]
+    depends_on: [caelo-admin]
+    environment:
+      CAELO_INTERNAL_SECRET: "${escapeYaml(spec.internalSecret)}"
+      CAELO_SCHEDULER_TICK_URL: "${COMPOSE_SCHEDULER_TICK_URL}"
+      CAELO_SCHEDULER_INTERVAL_MINUTES: "${minutes}"
+      NODE_ENV: production
+`;
+}
+
+/**
+ * A compose-network database URL, password URL-encoded and YAML-escaped.
+ * #1338 — verify-full against the install's own CA (self-hosted-db-tls.ts),
+ * mounted at shared `SELF_HOSTED_CA_FILE` ({@link DATABASE_CA_MOUNT}).
+ */
 function dbUrl(role: string, password: string, database: string): string {
-  return escapeYaml(`postgres://${role}:${encodeURIComponent(password)}@postgres:5432/${database}`);
+  return escapeYaml(
+    `postgres://${role}:${encodeURIComponent(password)}@${SELF_HOSTED_DB_HOST}:5432/${database}?${verifyFullQuery(SELF_HOSTED_CA_FILE)}`,
+  );
 }
 
+/** Where the server certificate + key are bind-mounted into Postgres (host-owned). */
+const POSTGRES_TLS_MOUNT_DIR = "/caelo-tls";
+/** Where the entrypoint copies them, owned by postgres (Postgres refuses others' keys). */
+const POSTGRES_TLS_RUNTIME_DIR = "/var/lib/postgresql/caelo-tls";
+/** The Postgres entrypoint: copy the TLS pair into place, then the image's own entrypoint. */
+const POSTGRES_TLS_ENTRYPOINT = JSON.stringify([
+  "/bin/sh",
+  "-c",
+  `install -d -o postgres -g postgres -m 0700 ${POSTGRES_TLS_RUNTIME_DIR} && install -o postgres -g postgres -m 0644 ${POSTGRES_TLS_MOUNT_DIR}/server.crt ${POSTGRES_TLS_RUNTIME_DIR}/server.crt && install -o postgres -g postgres -m 0600 ${POSTGRES_TLS_MOUNT_DIR}/server.key ${POSTGRES_TLS_RUNTIME_DIR}/server.key && exec docker-entrypoint.sh "$$@"`,
+  "--",
+]);
+/** The install's database CA, read-only, where every verify-full URL names it. */
+const DATABASE_CA_MOUNT = `./${SELF_HOSTED_TLS_DIR}/${SELF_HOSTED_TLS_FILES.caCert}:${SELF_HOSTED_CA_FILE}:ro`;
+
 function escapeYaml(s: string): string {
   // Conservative escape for double-quoted YAML strings.
   return s.replace(/\\/g, "\\\\").replace(/"/g, '\\"');

package/src/converge-checkpoint.ts

@@ -44,7 +44,14 @@ export const ConvergeCheckpointSchema = InstallJobStatusSchema.extend({
       backupId: z.string().min(1),
     })
     .optional(),
-  priorRevisions: z.object({ admin: z.string().min(1), gateway: z.string().min(1) }).optional(),
+  priorRevisions: z
+    .object({
+      admin: z.string().min(1),
+      gateway: z.string().min(1),
+      // #1328 — absent in a checkpoint an older release wrote.
+      "mcp-auth-proxy": z.string().min(1).optional(),
+    })
+    .optional(),
 });
 /**
  * The ops-bucket prefix of the converge job's own files — outside the

package/src/converge-db-trust.test.ts

@@ -0,0 +1,237 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * converge-db-trust.ts (#1373) against a fake gcloud and a fake Cloud SQL
+ * Admin API: which host the converge job dials before the database TLS
+ * step, and whose certificate it accepts there.
+ */
+
+import { describe, expect, it } from "bun:test";
+import {
+  convergeDatabaseTarget,
+  normalizedSha1,
+  serverIdentityProblem,
+} from "./converge-db-trust.js";
+import type { GcloudResult } from "./gcloud.js";
+import type { HttpFetch } from "./runtime-secrets.js";
+
+const DNS = "abc123.p-label.europe-west1.sql-psa.goog";
+const IP = "10.20.0.3";
+const PLACEMENT = { projectId: "p", region: "europe-west1" };
+const OWN_CA = "-----BEGIN CERTIFICATE-----\nper-instance\n-----END CERTIFICATE-----";
+const SHARED_BUNDLE = "-----BEGIN CERTIFICATE-----\nshared\n-----END CERTIFICATE-----";
+const ACTIVE = "a212b6cf51b3736b34a5d4a3194b200db6b3532a";
+const OLD = "0123456789abcdef0123456789abcdef01234567";
+
+const ok = (stdout = ""): GcloudResult => ({ ok: true, stdout, stderr: "", exitCode: 0 });
+const fail = (stderr: string): GcloudResult => ({ ok: false, stdout: "", stderr, exitCode: 1 });
+
+/** The admin service as `gcloud run services describe --format=json` prints it. */
+function adminService(host: string): string {
+  return JSON.stringify({
+    spec: {
+      template: {
+        metadata: {
+          annotations: {
+            "run.googleapis.com/network-interfaces": JSON.stringify([
+              { network: "net", subnetwork: "sub" },
+            ]),
+          },
+        },
+        spec: {
+          containers: [
+            {
+              image: "img@sha256:x",
+              env: [
+                {
+                  name: "ADMIN_DATABASE_URL",
+                  value: `postgresql://admin_role@${host}:5432/cms_admin?sslmode=require`,
+                },
+                {
+                  name: "ADMIN_DATABASE_PASSWORD",
+                  valueFrom: { secretKeyRef: { name: "pw", key: "latest" } },
+                },
+              ],
+            },
+          ],
+        },
+      },
+    },
+  });
+}
+
+function gcloud(opts: { host: string; instance?: object; describeFails?: boolean }) {
+  const calls: string[] = [];
+  const run = async (args: string[]) => {
+    const line = args.join(" ");
+    calls.push(line);
+    if (line.startsWith("run services list")) return ok("caelo-production-admin-abc1234\n");
+    if (line.startsWith("run services describe")) return ok(adminService(opts.host));
+    if (line.startsWith("sql instances list")) return ok("caelo-production-pg-1\n");
+    if (line.startsWith("sql instances describe")) {
+      return opts.describeFails ? fail("PERMISSION_DENIED") : ok(JSON.stringify(opts.instance));
+    }
+    if (line.startsWith("auth print-access-token")) return ok("ya29.token\n");
+    return fail(`unexpected: ${line}`);
+  };
+  return { run, calls };
+}
+
+function api(body: object, status = 200): HttpFetch {
+  return async () => ({ ok: status < 300, status, text: async () => JSON.stringify(body) });
+}
+
+const perInstance = {
+  serverCaCert: { cert: OWN_CA },
+  settings: { ipConfiguration: { serverCaMode: "GOOGLE_MANAGED_INTERNAL_CA" } },
+};
+const shared = {
+  serverCaCert: { cert: OWN_CA },
+  settings: { ipConfiguration: { serverCaMode: "GOOGLE_MANAGED_CAS_CA" } },
+};
+
+describe("convergeDatabaseTarget", () => {
+  it("an install on the DNS name connects with verify-full and asks Cloud SQL nothing", async () => {
+    const g = gcloud({ host: DNS });
+    expect(await convergeDatabaseTarget(PLACEMENT, { run: g.run })).toEqual({
+      kind: "dns",
+      host: DNS,
+    });
+    expect(g.calls.some((c) => c.startsWith("sql "))).toBe(false);
+  });
+
+  it("on the IP with the per-instance CA it trusts exactly that CA", async () => {
+    const g = gcloud({ host: IP, instance: perInstance });
+    const target = await convergeDatabaseTarget(PLACEMENT, {
+      run: g.run,
+      http: async () => {
+        throw new Error("the per-instance CA needs no certificate list");
+      },
+    });
+    expect(target).toEqual({ kind: "ip", host: IP, caPem: OWN_CA, pinnedSha1: null });
+  });
+
+  it("on the IP with the shared CA active it trusts the shared bundle and pins the instance's certificates", async () => {
+    const g = gcloud({ host: IP, instance: shared });
+    const target = await convergeDatabaseTarget(PLACEMENT, {
+      run: g.run,
+      http: api({ activeVersion: ACTIVE, serverCerts: [{ sha1Fingerprint: ACTIVE }] }),
+      readSharedCaBundle: () => SHARED_BUNDLE,
+    });
+    expect(target).toEqual({ kind: "ip", host: IP, caPem: SHARED_BUNDLE, pinnedSha1: [ACTIVE] });
+  });
+
+  it("moved to the shared CA but not rotated in: pinned to the one certificate the API names active", async () => {
+    const g = gcloud({ host: IP, instance: shared });
+    const target = await convergeDatabaseTarget(PLACEMENT, {
+      run: g.run,
+      http: api({ activeVersion: OLD.toUpperCase(), serverCerts: [] }),
+      readSharedCaBundle: () => SHARED_BUNDLE,
+    });
+    expect(target).toEqual({
+      kind: "ip",
+      host: IP,
+      caPem: `${OWN_CA}\n${SHARED_BUNDLE}`,
+      pinnedSha1: [OLD],
+    });
+  });
+
+  it("shared CA: a listing that does not positively name the active certificate is refused (fail closed)", async () => {
+    const answers: { body: unknown; why: RegExp }[] = [
+      { body: {}, why: /does not name the active certificate/ },
+      { body: { serverCerts: [{ sha1Fingerprint: ACTIVE }] }, why: /does not name/ },
+      { body: { activeVersion: "", serverCerts: [] }, why: /does not name/ },
+      { body: { activeVersion: "old-per-instance", serverCerts: [] }, why: /does not name/ },
+      {
+        body: { activeVersion: ACTIVE, serverCerts: [{ sha1Fingerprint: "nope" }] },
+        why: /does not name/,
+      },
+      { body: { activeVersion: ACTIVE, serverCerts: [{}] }, why: /does not name/ },
+      { body: { activeVersion: ACTIVE, serverCerts: "x" }, why: /does not name/ },
+    ];
+    for (const a of answers) {
+      await expect(
+        convergeDatabaseTarget(PLACEMENT, {
+          run: gcloud({ host: IP, instance: shared }).run,
+          http: api(a.body as object),
+          readSharedCaBundle: () => SHARED_BUNDLE,
+        }),
+      ).rejects.toThrow(a.why);
+    }
+    const notJson: HttpFetch = async () => ({ ok: true, status: 200, text: async () => "<html>" });
+    await expect(
+      convergeDatabaseTarget(PLACEMENT, {
+        run: gcloud({ host: IP, instance: shared }).run,
+        http: notJson,
+        readSharedCaBundle: () => SHARED_BUNDLE,
+      }),
+    ).rejects.toThrow(/answered no JSON/);
+  });
+
+  it("refuses a CA mode it cannot reason about", async () => {
+    await expect(
+      convergeDatabaseTarget(PLACEMENT, {
+        run: gcloud({
+          host: IP,
+          instance: {
+            serverCaCert: { cert: OWN_CA },
+            settings: { ipConfiguration: { serverCaMode: "CUSTOMER_MANAGED_CAS_CA" } },
+          },
+        }).run,
+      }),
+    ).rejects.toThrow(/CUSTOMER_MANAGED_CAS_CA/);
+  });
+
+  it("an instance that names no CA mode is on the per-instance default", async () => {
+    const g = gcloud({ host: IP, instance: { serverCaCert: { cert: OWN_CA } } });
+    expect(await convergeDatabaseTarget(PLACEMENT, { run: g.run })).toEqual({
+      kind: "ip",
+      host: IP,
+      caPem: OWN_CA,
+      pinnedSha1: null,
+    });
+  });
+
+  it("refuses to connect when it cannot tell whom to trust", async () => {
+    await expect(
+      convergeDatabaseTarget(PLACEMENT, { run: gcloud({ host: IP, describeFails: true }).run }),
+    ).rejects.toThrow("PERMISSION_DENIED");
+    await expect(
+      convergeDatabaseTarget(PLACEMENT, {
+        run: gcloud({
+          host: IP,
+          instance: {
+            settings: { ipConfiguration: { serverCaMode: "GOOGLE_MANAGED_INTERNAL_CA" } },
+          },
+        }).run,
+      }),
+    ).rejects.toThrow("names no server CA certificate");
+    await expect(
+      convergeDatabaseTarget(PLACEMENT, {
+        run: gcloud({ host: IP, instance: shared }).run,
+        http: api({ error: "denied" }, 403),
+      }),
+    ).rejects.toThrow("HTTP 403");
+  });
+});
+
+describe("serverIdentityProblem", () => {
+  const pinned = { kind: "ip", host: IP, caPem: SHARED_BUNDLE, pinnedSha1: [ACTIVE] } as const;
+
+  it("accepts a pinned certificate in node:tls's colon notation", () => {
+    const colons = ACTIVE.toUpperCase().replace(/(..)(?!$)/g, "$1:");
+    expect(serverIdentityProblem(pinned, colons)).toBeNull();
+  });
+
+  it("refuses a certificate the instance does not list", () => {
+    expect(serverIdentityProblem(pinned, "00:11")).toContain("does not list");
+  });
+
+  it("without a pin the verified chain to the instance's own CA decides", () => {
+    expect(serverIdentityProblem({ ...pinned, pinnedSha1: null }, "00:11")).toBeNull();
+  });
+
+  it("normalizes fingerprints", () => {
+    expect(normalizedSha1("AB:cd:01")).toBe("abcd01");
+  });
+});