@caelo-cms/provisioning 0.11.10 → 0.11.11

package/src/cli.ts

@@ -86,11 +86,12 @@ import {
   resolveVerifiedRelease,
 } from "./self-hosted-release.js";
 import {
-  applyPublicRoleLimits,
+  applyPreMigrationRoles,
   convergeSelfHostedRoles,
   decideUp,
   selfHostedBootstrapScript,
 } from "./self-hosted-roles.js";
+import { parseSchedulerIntervalMinutes } from "./stack-contract.js";
 
 interface CaeloConfig {
   domain: string;
@@ -153,6 +154,12 @@ interface CaeloConfig {
    * carry it).
    */
   composeProject?: string;
+  /**
+   * #1140 — the `schedulerIntervalMinutes` knob (`init
+   * --scheduler-interval-minutes`, 1–15): how often the scheduler container
+   * calls the admin's tick. Absent: the default (5).
+   */
+  schedulerIntervalMinutes?: number;
 }
 
 const REPO_ROOT = resolve(import.meta.dir, "../../..");
@@ -255,6 +262,15 @@ function loadConfig(): CaeloConfig | null {
     cfg.composeProject ??= "caelo";
     saveConfig(cfg);
   }
+  // #1329 — the staging gateway's login came with the staging database. An
+  // install from before gets its own password now; the next `up` or
+  // upgrade creates the role with it (self-hosted-roles.ts) before anything
+  // uses it, so the roles count as not converged until then.
+  if (cfg.rolePasswords && !cfg.rolePasswords.publicStaging) {
+    cfg.rolePasswords = { ...cfg.rolePasswords, publicStaging: randomSecret(32) };
+    cfg.rolesConverged = false;
+    saveConfig(cfg);
+  }
   return cfg;
 }
 
@@ -269,9 +285,9 @@ function arg(name: string): string | undefined {
   return process.argv[idx + 1];
 }
 
-/** Fresh passwords for the two application roles. */
+/** Fresh passwords for the application roles. */
 function newRolePasswords(): RolePasswords {
-  return { admin: randomSecret(32), public: randomSecret(32) };
+  return { admin: randomSecret(32), public: randomSecret(32), publicStaging: randomSecret(32) };
 }
 
 function emitConfig(cfg: CaeloConfig, extraDomains: CaddyDomainSpec[] = []): void {
@@ -369,6 +385,9 @@ function renderCompose(cfg: CaeloConfig, images: ComposeImages): string {
     images,
     ports: cfg.ports,
     project: cfg.composeProject,
+    ...(cfg.schedulerIntervalMinutes !== undefined
+      ? { schedulerIntervalMinutes: cfg.schedulerIntervalMinutes }
+      : {}),
     ...(activeAgent(cfg) ? { agent: activeAgent(cfg) } : {}),
   });
 }
@@ -405,8 +424,20 @@ function agentContext(cfg: CaeloConfig): AgentCliContext {
         ...(activeAgent(cfg) ? { agentPassword: activeAgent(cfg)?.dbPassword } : {}),
       });
     },
-    applyPublicRoleLimits: () =>
-      applyPublicRoleLimits({ probe: postgresIsUp, psql: superuserPsql }),
+    applyPreMigrationRoles: () => {
+      if (!cfg.rolePasswords) {
+        return Promise.resolve({
+          ok: false as const,
+          reachable: true,
+          error: "config.json has no role passwords — run `cms-provision up` first",
+        });
+      }
+      return applyPreMigrationRoles({
+        probe: postgresIsUp,
+        psql: superuserPsql,
+        stagingPassword: cfg.rolePasswords.publicStaging,
+      });
+    },
     renderCompose: (images) => renderCompose(cfg, images),
     refreshCaddy: () => refreshCaddyfile(cfg),
     randomSecret: () => randomSecret(32),
@@ -562,7 +593,7 @@ async function init(): Promise<void> {
   }
   if (!domain || !ownerEmail) {
     console.error(
-      "Usage: cms-provision init [--provider self-hosted] --domain example.com [--staging-domain staging.example.com] --owner-email me@example.com [--version x.y.z] [--skip-verify] [--dev] [--tls acme|internal] [--http-port 80] [--https-port 443]\n" +
+      "Usage: cms-provision init [--provider self-hosted] --domain example.com [--staging-domain staging.example.com] --owner-email me@example.com [--version x.y.z] [--skip-verify] [--dev] [--tls acme|internal] [--http-port 80] [--https-port 443] [--scheduler-interval-minutes 5]\n" +
         "Cloud providers (gcp | gcp-firebase | aws | azure) run the wizard: cms-provision --provider <name>",
     );
     process.exit(2);
@@ -577,6 +608,17 @@ async function init(): Promise<void> {
     console.error("--http-port and --https-port must differ");
     process.exit(2);
   }
+  const schedulerArg = arg("scheduler-interval-minutes");
+  let schedulerIntervalMinutes: number | undefined;
+  try {
+    schedulerIntervalMinutes =
+      schedulerArg === undefined
+        ? undefined
+        : parseSchedulerIntervalMinutes(schedulerArg, "--scheduler-interval-minutes");
+  } catch (e) {
+    console.error((e as Error).message);
+    process.exit(2);
+  }
   const stagingArg = arg("staging-domain");
   const stagingProblem = stagingArg
     ? stagingHostProblem(stagingArg, {
@@ -645,6 +687,7 @@ async function init(): Promise<void> {
     tls,
     ports,
     composeProject: composeProjectName(domain),
+    ...(schedulerIntervalMinutes !== undefined ? { schedulerIntervalMinutes } : {}),
   };
   saveConfig(cfg);
   // P15.1 — the shared HMAC secret for /api/internal/* endpoints, read by
@@ -1057,7 +1100,7 @@ async function upgradeSelfHostedCommand(opts: {
       runner: bunHostRunner,
       renderCompose: (images) => renderCompose(cfg, images),
       convergeRoles: agentContext(cfg).convergeRoles,
-      applyPublicRoleLimits: agentContext(cfg).applyPublicRoleLimits,
+      applyPreMigrationRoles: agentContext(cfg).applyPreMigrationRoles,
       refreshCaddy: () => refreshCaddyfile(cfg),
       log: {
         step: (m) => console.log(`\n==> ${m}`),
@@ -1625,13 +1668,25 @@ async function waitForDomainMappingCert(target: {
  * §11.C — interactive wizard. Loaded lazily so the bare-CLI startup
  * cost stays small (clack + kleur are pulled in only when needed).
  * Reads `--provider`, `--domain`, `--owner-email`, `--project-id`, `--region`,
- * `--non-interactive` and the run mode (`--local` / `--install-job`, default:
+ * `--scheduler-interval-minutes` (#1140), `--non-interactive` and the run mode (`--local` / `--install-job`, default:
  * the install job in Cloud Shell, local elsewhere — run-mode.ts) from argv.
  */
 async function wizardCommand(): Promise<void> {
   const { runWizard } = await import("./wizard.js");
   const { chooseRunMode } = await import("./run-mode.js");
+  const schedulerArg = arg("scheduler-interval-minutes");
+  let schedulerIntervalMinutes: number | undefined;
+  try {
+    schedulerIntervalMinutes =
+      schedulerArg === undefined
+        ? undefined
+        : parseSchedulerIntervalMinutes(schedulerArg, "--scheduler-interval-minutes");
+  } catch (e) {
+    console.error((e as Error).message);
+    process.exit(2);
+  }
   await runWizard({
+    ...(schedulerIntervalMinutes !== undefined ? { schedulerIntervalMinutes } : {}),
     runMode: chooseRunMode(process.argv, process.env),
     nonInteractive: process.argv.includes("--non-interactive"),
     provider: arg("provider") as Provider | undefined,

package/src/cloudsql-tls.test.ts

@@ -94,6 +94,36 @@ describe("ensureCloudSqlVerifyFull", () => {
     ]);
   });
 
+  it("reads the DNS name after the move when the shared CA's certificate is active at once", async () => {
+    // searchviu-website, v0.11.10: the patch made the shared CA's certificate
+    // the active one, so no rotation re-read the instance, and the
+    // description from before the patch named no DNS host.
+    let described = 0;
+    const g = fake((a) => {
+      const line = a.join(" ");
+      if (line.startsWith("sql instances describe")) {
+        described += 1;
+        return ok(
+          instance(
+            described === 1
+              ? { caMode: "GOOGLE_MANAGED_INTERNAL_CA", sslMode: "ENCRYPTED_ONLY", dns: false }
+              : { caMode: "GOOGLE_MANAGED_CAS_CA", sslMode: "ENCRYPTED_ONLY", dns: true },
+          ),
+        );
+      }
+      if (line.startsWith("auth print-access-token")) return ok("token");
+      if (line.includes(" describe ")) return notFound();
+      return ok();
+    });
+    const r = await ensureCloudSqlVerifyFull(TARGET, {
+      run: g.run,
+      http: certs({ activeVersion: "f1", serverCerts: [{ sha1Fingerprint: "f1" }] }),
+      sleep: async () => {},
+    });
+    expect(r).toMatchObject({ ok: true, host: DNS });
+    expect(g.calls.filter((c) => c.startsWith("sql ssl"))).toEqual([]);
+  });
+
   it("changes nothing on an instance already in shape", async () => {
     const g = fake((a) => {
       const line = a.join(" ");

package/src/cloudsql-tls.ts

@@ -72,8 +72,10 @@ export type CloudSqlTlsResult =
   | { readonly ok: true; readonly host: string; readonly done: string[] }
   | { readonly ok: false; readonly error: string; readonly done: string[] };
 
-/** The parts of `gcloud sql instances describe --format=json` this reads. */
-interface InstanceDescription {
+/** The parts of `gcloud sql instances describe --format=json` this module and converge-db-trust.ts read. */
+export interface InstanceDescription {
+  /** The CA of the instance's current server certificate (PEM in `cert`). */
+  readonly serverCaCert?: { readonly cert?: string };
   readonly settings?: {
     readonly ipConfiguration?: {
       readonly serverCaMode?: string;
@@ -88,7 +90,8 @@ interface InstanceDescription {
 const NOT_FOUND = /NOT_FOUND|not found|does not exist|404/i;
 const SQL_ADMIN = "https://sqladmin.googleapis.com/v1";
 
-async function describeInstance(
+/** `gcloud sql instances describe` of the install's instance, parsed. */
+export async function describeInstance(
   t: CloudSqlTlsTarget,
   run: GcloudRunner,
 ): Promise<{ ok: true; d: InstanceDescription } | { ok: false; error: string }> {
@@ -170,6 +173,10 @@ export async function ensureCloudSqlVerifyFull(
     const r = await run(["sql", "instances", "patch", t.sqlInstance, project, ...patch, "--quiet"]);
     if (!r.ok) return fail(`patch ${t.sqlInstance} (${patch.join(" ")}): ${r.stderr.trim()}`);
     done.push(`Cloud SQL ${t.sqlInstance}: ${patch.join(" ")}`);
+    // The instance reports its DNS name only once it uses the shared CA, so
+    // the description from before the patch cannot name the host.
+    described = await describeInstance(t, run);
+    if (!described.ok) return fail(described.error);
   }
 
   // The active server certificate must come from the shared CA. Before the

package/src/compose.ts

@@ -37,9 +37,11 @@ import {
 } from "./self-hosted-db-tls.js";
 import {
   ADMIN_MEMORY_DEFAULT,
+  GATEWAY_ENVIRONMENT_ENV,
   GATEWAY_TRUSTED_PROXY_HOPS,
   MCP_AUTH_PROXY_TRUSTED_PROXY_HOPS,
   memoryQuantityMiB,
+  parseSchedulerIntervalMinutes,
   publicSiteUrl,
   REMOTE_MCP_MODE,
   REMOTE_MCP_MODE_ENV,
@@ -84,6 +86,8 @@ export const COMPOSE_POOL_BUDGET = stackPoolBudget("self-hosted", {
   maxConnections: DEFAULT_MAX_CONNECTIONS["self-hosted"],
   adminMaxInstances: 1,
   gatewayMaxInstances: 1,
+  // #1329 — one staging gateway container (P3b-2) next to the live one.
+  stagingGatewayMaxInstances: 1,
 });
 
 /** The budget's env as Compose `environment:` lines for `service`. */
@@ -216,6 +220,8 @@ export function ghcrImageDigest(service: ReleaseService, ref: string): string |
 export interface RolePasswords {
   readonly admin: string;
   readonly public: string;
+  /** #1329 — public_staging_role's, the staging gateway's login on cms_public_staging. */
+  readonly publicStaging: string;
 }
 
 export interface ComposeSpec {
@@ -258,6 +264,12 @@ export interface ComposeSpec {
   readonly adminMemory?: string;
   /** Which Caelo images run (see {@link ComposeImages}). */
   readonly images: ComposeImages;
+  /**
+   * #1140 — the `schedulerIntervalMinutes` knob: how often the scheduler
+   * container calls the admin's tick (1–15, default 5; stack-contract.ts
+   * SCHEDULER_INTERVAL_MINUTES).
+   */
+  readonly schedulerIntervalMinutes?: number;
   /**
    * The host ports Caddy listens on (`80` / `443` for a public install —
    * ACME's HTTP and TLS-ALPN challenges need exactly those).
@@ -336,6 +348,7 @@ services:
       POSTGRES_DB: caelo
       ADMIN_ROLE_PASSWORD: "${escapeYaml(spec.rolePasswords.admin)}"
       PUBLIC_ROLE_PASSWORD: "${escapeYaml(spec.rolePasswords.public)}"
+      PUBLIC_STAGING_ROLE_PASSWORD: "${escapeYaml(spec.rolePasswords.publicStaging)}"
       # Recreates Postgres when its certificate is renewed (see the spec).
       CAELO_DB_TLS_SERVER_CERT_SHA256: "${spec.dbTlsServerCertSha256}"
 ${postgresAgent(spec.agent)}    volumes:
@@ -366,6 +379,7 @@ ${adminService(spec, adminMemoryMiB, env)}
 
 ${gatewayService(spec)}
 ${mcpAuthProxyService(spec.images)}
+${schedulerService(spec)}
 volumes:
   caelo-pg:
   caelo-caddy-data:
@@ -457,6 +471,8 @@ ${serviceRuntime(
     environment:
       ADMIN_DATABASE_URL: "${dbUrl("admin_role", spec.rolePasswords.admin, "cms_admin")}"
       PUBLIC_ADMIN_DATABASE_URL: "${dbUrl("admin_role", spec.rolePasswords.admin, "cms_public")}"
+      # #1329 — the staging database's plugin schemas and projection.
+      PUBLIC_STAGING_ADMIN_DATABASE_URL: "${dbUrl("admin_role", spec.rolePasswords.admin, "cms_public_staging")}"
       CAELO_SECRET_KEK: "${escapeYaml(spec.caeloSecretKek)}"
       CAELO_INTERNAL_SECRET: "${escapeYaml(spec.internalSecret)}"
       CAELO_SITE_URL: "${escapeYaml(selfHostedSiteUrl(spec.domain, spec.ports.https))}"
@@ -480,6 +496,8 @@ ${serviceRuntime(spec.images, "gateway", "bun run apps/api-gateway/src/server.ts
       # public_role on cms_public is its only database login (#613, #635,
       # CLAUDE.md §2: the public side has no path to the authoring database).
       PUBLIC_DATABASE_URL: "${dbUrl("public_role", spec.rolePasswords.public, "cms_public")}"
+      # #1329 — the live site's gateway: it refuses anything of the staging data.
+      ${GATEWAY_ENVIRONMENT_ENV}: "live"
       GATEWAY_PORT: "${COMPOSE_GATEWAY_PORT}"
       # Caddy is the one proxy in front (client-ip.ts in the gateway).
       ${TRUSTED_PROXY_HOPS_ENV}: "${GATEWAY_TRUSTED_PROXY_HOPS["self-hosted"]}"
@@ -523,6 +541,56 @@ ${serviceRuntime(images, "mcp-auth-proxy", "bun run apps/mcp-auth-proxy/src/serv
 `;
 }
 
+/** #1140 — the Compose service that calls the admin's scheduler tick. */
+export const COMPOSE_SCHEDULER_SERVICE = "caelo-scheduler";
+
+/** Where the admin image (and the dev checkout) ships the scheduler clock. */
+export const SCHEDULER_CLOCK_SCRIPT = "packages/admin-core/src/scheduler/clock.ts";
+
+/** The admin's tick endpoint on the Compose network. */
+export const COMPOSE_SCHEDULER_TICK_URL = `http://${COMPOSE_ADMIN_UPSTREAM}/api/internal/scheduler/tick`;
+
+/**
+ * #1140 — the scheduler: the self-hosted counterpart of the Cloud Scheduler
+ * job on Google Cloud. Every `schedulerIntervalMinutes` it POSTs the admin's
+ * `/api/internal/scheduler/tick` with an internal token (scope
+ * `scheduler.tick`); the admin runs whatever its due-table says is due. The
+ * clock only calls, it does no work itself, so the admin's background work
+ * runs inside a request on every provider.
+ *
+ * It runs the admin image with another entry point (dependency-free, node
+ * built-ins only) and holds only the internal secret it signs with: no
+ * database URL, no KEK, no mount, read-only root filesystem, every
+ * capability dropped. A rollback to a release from before #1140 renders it
+ * too; that admin image has no clock script, so the container exits and
+ * Docker restarts it with back-off — that release runs its work in-process.
+ */
+function schedulerService(spec: ComposeSpec): string {
+  const minutes = parseSchedulerIntervalMinutes(spec.schedulerIntervalMinutes);
+  const runtime =
+    spec.images.mode === "dev"
+      ? serviceRuntime(spec.images, "admin", `bun run ${SCHEDULER_CLOCK_SCRIPT}`)
+      : [
+          serviceRuntime(spec.images, "admin", ""),
+          `    command: ["bun", "--bun", "/app/${SCHEDULER_CLOCK_SCRIPT}"]`,
+        ].join("\n");
+  return `
+  ${COMPOSE_SCHEDULER_SERVICE}:
+${runtime}
+    restart: unless-stopped
+    read_only: true
+    tmpfs: ["/tmp"]
+    cap_drop: ["ALL"]
+    security_opt: ["no-new-privileges:true"]
+    depends_on: [caelo-admin]
+    environment:
+      CAELO_INTERNAL_SECRET: "${escapeYaml(spec.internalSecret)}"
+      CAELO_SCHEDULER_TICK_URL: "${COMPOSE_SCHEDULER_TICK_URL}"
+      CAELO_SCHEDULER_INTERVAL_MINUTES: "${minutes}"
+      NODE_ENV: production
+`;
+}
+
 /**
  * A compose-network database URL, password URL-encoded and YAML-escaped.
  * #1338 — verify-full against the install's own CA (self-hosted-db-tls.ts),

package/src/converge-db-trust.test.ts

@@ -0,0 +1,237 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * converge-db-trust.ts (#1373) against a fake gcloud and a fake Cloud SQL
+ * Admin API: which host the converge job dials before the database TLS
+ * step, and whose certificate it accepts there.
+ */
+
+import { describe, expect, it } from "bun:test";
+import {
+  convergeDatabaseTarget,
+  normalizedSha1,
+  serverIdentityProblem,
+} from "./converge-db-trust.js";
+import type { GcloudResult } from "./gcloud.js";
+import type { HttpFetch } from "./runtime-secrets.js";
+
+const DNS = "abc123.p-label.europe-west1.sql-psa.goog";
+const IP = "10.20.0.3";
+const PLACEMENT = { projectId: "p", region: "europe-west1" };
+const OWN_CA = "-----BEGIN CERTIFICATE-----\nper-instance\n-----END CERTIFICATE-----";
+const SHARED_BUNDLE = "-----BEGIN CERTIFICATE-----\nshared\n-----END CERTIFICATE-----";
+const ACTIVE = "a212b6cf51b3736b34a5d4a3194b200db6b3532a";
+const OLD = "0123456789abcdef0123456789abcdef01234567";
+
+const ok = (stdout = ""): GcloudResult => ({ ok: true, stdout, stderr: "", exitCode: 0 });
+const fail = (stderr: string): GcloudResult => ({ ok: false, stdout: "", stderr, exitCode: 1 });
+
+/** The admin service as `gcloud run services describe --format=json` prints it. */
+function adminService(host: string): string {
+  return JSON.stringify({
+    spec: {
+      template: {
+        metadata: {
+          annotations: {
+            "run.googleapis.com/network-interfaces": JSON.stringify([
+              { network: "net", subnetwork: "sub" },
+            ]),
+          },
+        },
+        spec: {
+          containers: [
+            {
+              image: "img@sha256:x",
+              env: [
+                {
+                  name: "ADMIN_DATABASE_URL",
+                  value: `postgresql://admin_role@${host}:5432/cms_admin?sslmode=require`,
+                },
+                {
+                  name: "ADMIN_DATABASE_PASSWORD",
+                  valueFrom: { secretKeyRef: { name: "pw", key: "latest" } },
+                },
+              ],
+            },
+          ],
+        },
+      },
+    },
+  });
+}
+
+function gcloud(opts: { host: string; instance?: object; describeFails?: boolean }) {
+  const calls: string[] = [];
+  const run = async (args: string[]) => {
+    const line = args.join(" ");
+    calls.push(line);
+    if (line.startsWith("run services list")) return ok("caelo-production-admin-abc1234\n");
+    if (line.startsWith("run services describe")) return ok(adminService(opts.host));
+    if (line.startsWith("sql instances list")) return ok("caelo-production-pg-1\n");
+    if (line.startsWith("sql instances describe")) {
+      return opts.describeFails ? fail("PERMISSION_DENIED") : ok(JSON.stringify(opts.instance));
+    }
+    if (line.startsWith("auth print-access-token")) return ok("ya29.token\n");
+    return fail(`unexpected: ${line}`);
+  };
+  return { run, calls };
+}
+
+function api(body: object, status = 200): HttpFetch {
+  return async () => ({ ok: status < 300, status, text: async () => JSON.stringify(body) });
+}
+
+const perInstance = {
+  serverCaCert: { cert: OWN_CA },
+  settings: { ipConfiguration: { serverCaMode: "GOOGLE_MANAGED_INTERNAL_CA" } },
+};
+const shared = {
+  serverCaCert: { cert: OWN_CA },
+  settings: { ipConfiguration: { serverCaMode: "GOOGLE_MANAGED_CAS_CA" } },
+};
+
+describe("convergeDatabaseTarget", () => {
+  it("an install on the DNS name connects with verify-full and asks Cloud SQL nothing", async () => {
+    const g = gcloud({ host: DNS });
+    expect(await convergeDatabaseTarget(PLACEMENT, { run: g.run })).toEqual({
+      kind: "dns",
+      host: DNS,
+    });
+    expect(g.calls.some((c) => c.startsWith("sql "))).toBe(false);
+  });
+
+  it("on the IP with the per-instance CA it trusts exactly that CA", async () => {
+    const g = gcloud({ host: IP, instance: perInstance });
+    const target = await convergeDatabaseTarget(PLACEMENT, {
+      run: g.run,
+      http: async () => {
+        throw new Error("the per-instance CA needs no certificate list");
+      },
+    });
+    expect(target).toEqual({ kind: "ip", host: IP, caPem: OWN_CA, pinnedSha1: null });
+  });
+
+  it("on the IP with the shared CA active it trusts the shared bundle and pins the instance's certificates", async () => {
+    const g = gcloud({ host: IP, instance: shared });
+    const target = await convergeDatabaseTarget(PLACEMENT, {
+      run: g.run,
+      http: api({ activeVersion: ACTIVE, serverCerts: [{ sha1Fingerprint: ACTIVE }] }),
+      readSharedCaBundle: () => SHARED_BUNDLE,
+    });
+    expect(target).toEqual({ kind: "ip", host: IP, caPem: SHARED_BUNDLE, pinnedSha1: [ACTIVE] });
+  });
+
+  it("moved to the shared CA but not rotated in: pinned to the one certificate the API names active", async () => {
+    const g = gcloud({ host: IP, instance: shared });
+    const target = await convergeDatabaseTarget(PLACEMENT, {
+      run: g.run,
+      http: api({ activeVersion: OLD.toUpperCase(), serverCerts: [] }),
+      readSharedCaBundle: () => SHARED_BUNDLE,
+    });
+    expect(target).toEqual({
+      kind: "ip",
+      host: IP,
+      caPem: `${OWN_CA}\n${SHARED_BUNDLE}`,
+      pinnedSha1: [OLD],
+    });
+  });
+
+  it("shared CA: a listing that does not positively name the active certificate is refused (fail closed)", async () => {
+    const answers: { body: unknown; why: RegExp }[] = [
+      { body: {}, why: /does not name the active certificate/ },
+      { body: { serverCerts: [{ sha1Fingerprint: ACTIVE }] }, why: /does not name/ },
+      { body: { activeVersion: "", serverCerts: [] }, why: /does not name/ },
+      { body: { activeVersion: "old-per-instance", serverCerts: [] }, why: /does not name/ },
+      {
+        body: { activeVersion: ACTIVE, serverCerts: [{ sha1Fingerprint: "nope" }] },
+        why: /does not name/,
+      },
+      { body: { activeVersion: ACTIVE, serverCerts: [{}] }, why: /does not name/ },
+      { body: { activeVersion: ACTIVE, serverCerts: "x" }, why: /does not name/ },
+    ];
+    for (const a of answers) {
+      await expect(
+        convergeDatabaseTarget(PLACEMENT, {
+          run: gcloud({ host: IP, instance: shared }).run,
+          http: api(a.body as object),
+          readSharedCaBundle: () => SHARED_BUNDLE,
+        }),
+      ).rejects.toThrow(a.why);
+    }
+    const notJson: HttpFetch = async () => ({ ok: true, status: 200, text: async () => "<html>" });
+    await expect(
+      convergeDatabaseTarget(PLACEMENT, {
+        run: gcloud({ host: IP, instance: shared }).run,
+        http: notJson,
+        readSharedCaBundle: () => SHARED_BUNDLE,
+      }),
+    ).rejects.toThrow(/answered no JSON/);
+  });
+
+  it("refuses a CA mode it cannot reason about", async () => {
+    await expect(
+      convergeDatabaseTarget(PLACEMENT, {
+        run: gcloud({
+          host: IP,
+          instance: {
+            serverCaCert: { cert: OWN_CA },
+            settings: { ipConfiguration: { serverCaMode: "CUSTOMER_MANAGED_CAS_CA" } },
+          },
+        }).run,
+      }),
+    ).rejects.toThrow(/CUSTOMER_MANAGED_CAS_CA/);
+  });
+
+  it("an instance that names no CA mode is on the per-instance default", async () => {
+    const g = gcloud({ host: IP, instance: { serverCaCert: { cert: OWN_CA } } });
+    expect(await convergeDatabaseTarget(PLACEMENT, { run: g.run })).toEqual({
+      kind: "ip",
+      host: IP,
+      caPem: OWN_CA,
+      pinnedSha1: null,
+    });
+  });
+
+  it("refuses to connect when it cannot tell whom to trust", async () => {
+    await expect(
+      convergeDatabaseTarget(PLACEMENT, { run: gcloud({ host: IP, describeFails: true }).run }),
+    ).rejects.toThrow("PERMISSION_DENIED");
+    await expect(
+      convergeDatabaseTarget(PLACEMENT, {
+        run: gcloud({
+          host: IP,
+          instance: {
+            settings: { ipConfiguration: { serverCaMode: "GOOGLE_MANAGED_INTERNAL_CA" } },
+          },
+        }).run,
+      }),
+    ).rejects.toThrow("names no server CA certificate");
+    await expect(
+      convergeDatabaseTarget(PLACEMENT, {
+        run: gcloud({ host: IP, instance: shared }).run,
+        http: api({ error: "denied" }, 403),
+      }),
+    ).rejects.toThrow("HTTP 403");
+  });
+});
+
+describe("serverIdentityProblem", () => {
+  const pinned = { kind: "ip", host: IP, caPem: SHARED_BUNDLE, pinnedSha1: [ACTIVE] } as const;
+
+  it("accepts a pinned certificate in node:tls's colon notation", () => {
+    const colons = ACTIVE.toUpperCase().replace(/(..)(?!$)/g, "$1:");
+    expect(serverIdentityProblem(pinned, colons)).toBeNull();
+  });
+
+  it("refuses a certificate the instance does not list", () => {
+    expect(serverIdentityProblem(pinned, "00:11")).toContain("does not list");
+  });
+
+  it("without a pin the verified chain to the instance's own CA decides", () => {
+    expect(serverIdentityProblem({ ...pinned, pinnedSha1: null }, "00:11")).toBeNull();
+  });
+
+  it("normalizes fingerprints", () => {
+    expect(normalizedSha1("AB:cd:01")).toBe("abcd01");
+  });
+});