@caelo-cms/provisioning 0.11.9 → 0.11.10

package/src/cloudsql-tls.ts

@@ -0,0 +1,352 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * #1338 (gateway hardening L6) — converge a Cloud SQL install to
+ * `sslmode=verify-full` through gcloud, the way `cms-provision upgrade`
+ * converges everything else (stack-converge.ts; it never runs Pulumi).
+ *
+ * Verify-full checks the server certificate's chain AND that it names the
+ * host the client dialled. On Cloud SQL that needs three things the stacks
+ * declare for a new install (stacks/gcp, stacks/gcp-firebase) and this
+ * module brings an existing one to:
+ *
+ *   1. The shared server CA (`GOOGLE_MANAGED_CAS_CA`): "The SAN field
+ *      contains the hostname (DNS name of the instance) for all types of
+ *      instances", while the default per-instance CA names one "only for
+ *      instances enabled with Private Service Connect"
+ *      (docs.cloud.google.com/sql/docs/postgres/authorize-ssl). An instance
+ *      may move from per-instance to shared, not back. After the move the
+ *      instance needs a server certificate from the new CA: "create a new
+ *      server certificate with `gcloud sql ssl server-certs create`", then
+ *      "complete the rotation with `gcloud sql ssl server-certs rotate`"
+ *      (docs.cloud.google.com/sql/docs/postgres/manage-ssl-instance). The
+ *      rotation restarts nothing a client keeps: open connections drop and
+ *      reconnect; every client already uses TLS (`sslmode=require` before).
+ *      `sslMode=ENCRYPTED_ONLY` refuses plaintext from then on.
+ *   2. Resolution of the instance's DNS name (`dnsNames`, the
+ *      PRIVATE_SERVICES_ACCESS entry, `….sql-psa.goog`) inside the VPC:
+ *      "create the DNS record in a private DNS zone in the corresponding
+ *      Virtual Private Cloud (VPC) network"
+ *      (docs.cloud.google.com/sql/docs/postgres/configure-ssl-instance).
+ *   3. The services dialling that name. They read it from their own
+ *      `*_DATABASE_URL`, and so does every job the upgrade derives from the
+ *      admin's (migrations, upgrader, operator access): {@link
+ *      moveServicesToDnsHost} switches the host of the live URLs first,
+ *      keeping their current `sslmode`, so the old image keeps working until
+ *      the roll sets verify-full with the image that carries the CA bundle.
+ *
+ * The CA the clients trust is Google's published shared-CA bundle, shipped
+ * in every image (shared database-tls.ts `CLOUD_SQL_CA_BUNDLE`): it covers
+ * the root and every regional CA, so a server-certificate rotation needs no
+ * change on the client side.
+ *
+ * Every step reads before it writes and is a no-op when already in shape,
+ * so a re-run after a failure picks up where the last one stopped.
+ */
+
+import { isIpLiteral } from "@caelo-cms/shared";
+import { type GcloudRunner, realSleep, runWithRetry, type Sleep } from "./gcloud-retry.js";
+import type { HttpFetch } from "./runtime-secrets.js";
+import {
+  CLOUD_SQL_SERVER_CA_MODE,
+  CLOUD_SQL_SSL_MODE,
+  cloudSqlDnsZoneName,
+  cloudSqlPrivateDnsName,
+} from "./stack-contract.js";
+import type { LiveEnvValue } from "./stack-converge.js";
+
+/** The install's Cloud SQL instance. */
+export interface CloudSqlTlsTarget {
+  readonly projectId: string;
+  readonly env: string;
+  readonly sqlInstance: string;
+}
+
+export interface CloudSqlTlsDeps {
+  readonly run: GcloudRunner;
+  readonly http?: HttpFetch;
+  readonly sleep?: Sleep;
+}
+
+export type CloudSqlTlsResult =
+  | { readonly ok: true; readonly host: string; readonly done: string[] }
+  | { readonly ok: false; readonly error: string; readonly done: string[] };
+
+/** The parts of `gcloud sql instances describe --format=json` this reads. */
+interface InstanceDescription {
+  readonly settings?: {
+    readonly ipConfiguration?: {
+      readonly serverCaMode?: string;
+      readonly sslMode?: string;
+      readonly privateNetwork?: string;
+    };
+  };
+  readonly ipAddresses?: readonly { readonly type?: string; readonly ipAddress?: string }[];
+  readonly dnsNames?: readonly { readonly name?: string; readonly connectionType?: string }[];
+}
+
+const NOT_FOUND = /NOT_FOUND|not found|does not exist|404/i;
+const SQL_ADMIN = "https://sqladmin.googleapis.com/v1";
+
+async function describeInstance(
+  t: CloudSqlTlsTarget,
+  run: GcloudRunner,
+): Promise<{ ok: true; d: InstanceDescription } | { ok: false; error: string }> {
+  const r = await run([
+    "sql",
+    "instances",
+    "describe",
+    t.sqlInstance,
+    `--project=${t.projectId}`,
+    "--format=json",
+  ]);
+  if (!r.ok) return { ok: false, error: `describe ${t.sqlInstance}: ${r.stderr.trim()}` };
+  try {
+    return { ok: true, d: JSON.parse(r.stdout) as InstanceDescription };
+  } catch {
+    return { ok: false, error: `describe ${t.sqlInstance}: gcloud printed no JSON` };
+  }
+}
+
+/** The instance's server certificates and which one is active (Cloud SQL Admin API). */
+async function listServerCertificates(
+  t: CloudSqlTlsTarget,
+  run: GcloudRunner,
+  http: HttpFetch,
+): Promise<
+  { ok: true; activeVersion: string | null; fingerprints: string[] } | { ok: false; error: string }
+> {
+  const token = await run(["auth", "print-access-token"]);
+  const bearer = token.ok ? token.stdout.trim() : "";
+  if (!bearer)
+    return { ok: false, error: `gcloud auth print-access-token: ${token.stderr.trim()}` };
+  const res = await http(
+    `${SQL_ADMIN}/projects/${encodeURIComponent(t.projectId)}/instances/${encodeURIComponent(t.sqlInstance)}/listServerCertificates`,
+    { method: "GET", headers: { Authorization: `Bearer ${bearer}` } },
+  );
+  const text = await res.text();
+  if (!res.ok) {
+    return {
+      ok: false,
+      error: `list server certificates: HTTP ${res.status} ${text.slice(0, 300)}`,
+    };
+  }
+  const body = JSON.parse(text) as {
+    activeVersion?: string;
+    serverCerts?: { sha1Fingerprint?: string }[];
+  };
+  return {
+    ok: true,
+    activeVersion: body.activeVersion ?? null,
+    fingerprints: (body.serverCerts ?? []).map((c) => c.sha1Fingerprint ?? "").filter(Boolean),
+  };
+}
+
+/**
+ * Bring the instance to verify-full: shared CA with an active server
+ * certificate from it, encrypted connections only, its DNS name resolving
+ * in the VPC. Returns the host every client must dial.
+ */
+export async function ensureCloudSqlVerifyFull(
+  t: CloudSqlTlsTarget,
+  deps: CloudSqlTlsDeps,
+): Promise<CloudSqlTlsResult> {
+  const sleep = deps.sleep ?? realSleep;
+  const run = (args: string[]) => runWithRetry(deps.run, sleep, args);
+  const http = deps.http ?? (fetch as unknown as HttpFetch);
+  const done: string[] = [];
+  const fail = (error: string): CloudSqlTlsResult => ({ ok: false, error, done });
+  const project = `--project=${t.projectId}`;
+
+  let described = await describeInstance(t, run);
+  if (!described.ok) return fail(described.error);
+  const ip = described.d.settings?.ipConfiguration;
+  const patch: string[] = [];
+  if (ip?.serverCaMode !== CLOUD_SQL_SERVER_CA_MODE) {
+    patch.push(`--server-ca-mode=${CLOUD_SQL_SERVER_CA_MODE}`);
+  }
+  if (ip?.sslMode !== CLOUD_SQL_SSL_MODE) patch.push(`--ssl-mode=${CLOUD_SQL_SSL_MODE}`);
+  if (patch.length > 0) {
+    const r = await run(["sql", "instances", "patch", t.sqlInstance, project, ...patch, "--quiet"]);
+    if (!r.ok) return fail(`patch ${t.sqlInstance} (${patch.join(" ")}): ${r.stderr.trim()}`);
+    done.push(`Cloud SQL ${t.sqlInstance}: ${patch.join(" ")}`);
+  }
+
+  // The active server certificate must come from the shared CA. Before the
+  // move there is none in the list the shared CA keeps; after an
+  // interrupted run there may be an upcoming one that was never rotated in.
+  const certs = await listServerCertificates(t, run, http);
+  if (!certs.ok) return fail(certs.error);
+  const activeShared =
+    certs.activeVersion !== null && certs.fingerprints.includes(certs.activeVersion);
+  if (!activeShared) {
+    const upcoming = certs.fingerprints.filter((f) => f !== certs.activeVersion);
+    if (upcoming.length === 0) {
+      const c = await run([
+        "sql",
+        "ssl",
+        "server-certs",
+        "create",
+        `--instance=${t.sqlInstance}`,
+        project,
+      ]);
+      if (!c.ok) return fail(`create a server certificate: ${c.stderr.trim()}`);
+      done.push(`Cloud SQL ${t.sqlInstance}: server certificate from the shared CA created`);
+    }
+    const rot = await run([
+      "sql",
+      "ssl",
+      "server-certs",
+      "rotate",
+      `--instance=${t.sqlInstance}`,
+      project,
+      "--quiet",
+    ]);
+    if (!rot.ok) return fail(`rotate to the shared-CA server certificate: ${rot.stderr.trim()}`);
+    done.push(`Cloud SQL ${t.sqlInstance}: rotated onto the shared-CA server certificate`);
+    described = await describeInstance(t, run);
+    if (!described.ok) return fail(described.error);
+  }
+
+  const host = cloudSqlPrivateDnsName(described.d.dnsNames);
+  if (!host) {
+    return fail(
+      `${t.sqlInstance} reports no PRIVATE_SERVICES_ACCESS DNS name (dnsNames) although it uses the shared CA; verify-full needs it. Check the instance's private IP configuration and re-run.`,
+    );
+  }
+  const privateIp = described.d.ipAddresses?.find((a) => a.type === "PRIVATE")?.ipAddress;
+  if (!privateIp) return fail(`${t.sqlInstance} has no private IP address`);
+  const network = ip?.privateNetwork?.split("/").pop();
+  if (!network) return fail(`${t.sqlInstance} names no private network`);
+
+  const zone = cloudSqlDnsZoneName(t.env);
+  const fqdn = `${host}.`;
+  const z = await run(["dns", "managed-zones", "describe", zone, project, "--format=json"]);
+  if (z.ok) {
+    const existing = (JSON.parse(z.stdout) as { dnsName?: string }).dnsName?.toLowerCase();
+    if (existing !== fqdn) {
+      return fail(
+        `the private DNS zone ${zone} serves ${existing ?? "(no name)"}, not the instance's DNS name ${fqdn}. If the instance was replaced, delete the zone (gcloud dns managed-zones delete ${zone} ${project}) after removing its records, then re-run.`,
+      );
+    }
+  } else if (NOT_FOUND.test(z.stderr)) {
+    const c = await run([
+      "dns",
+      "managed-zones",
+      "create",
+      zone,
+      project,
+      `--dns-name=${fqdn}`,
+      "--visibility=private",
+      `--networks=${network}`,
+      `--description=Caelo ${t.env}: the Cloud SQL instance's DNS name inside the VPC (#1338)`,
+    ]);
+    if (!c.ok) return fail(`create the private DNS zone ${zone}: ${c.stderr.trim()}`);
+    done.push(`private DNS zone ${zone} (${fqdn}) on network ${network}`);
+  } else {
+    return fail(`describe the private DNS zone ${zone}: ${z.stderr.trim()}`);
+  }
+
+  const rec = ["dns", "record-sets"];
+  const where = [fqdn, `--zone=${zone}`, "--type=A", project];
+  const a = await run([...rec, "describe", ...where, "--format=json"]);
+  if (a.ok) {
+    const rrdatas = (JSON.parse(a.stdout) as { rrdatas?: string[] }).rrdatas ?? [];
+    if (rrdatas.length !== 1 || rrdatas[0] !== privateIp) {
+      const u = await run([...rec, "update", ...where, "--ttl=300", `--rrdatas=${privateIp}`]);
+      if (!u.ok) return fail(`update the A record of ${fqdn}: ${u.stderr.trim()}`);
+      done.push(`DNS ${fqdn} A ${rrdatas.join(",") || "(none)"} → ${privateIp}`);
+    }
+  } else if (NOT_FOUND.test(a.stderr)) {
+    const c = await run([...rec, "create", ...where, "--ttl=300", `--rrdatas=${privateIp}`]);
+    if (!c.ok) return fail(`create the A record of ${fqdn}: ${c.stderr.trim()}`);
+    done.push(`DNS ${fqdn} A ${privateIp}`);
+  } else {
+    return fail(`describe the A record of ${fqdn}: ${a.stderr.trim()}`);
+  }
+  return { ok: true, host, done };
+}
+
+/** The plain values of `names` in a service's live env (secret refs and unset vars skipped). */
+export function liveDatabaseUrls(
+  liveEnv: ReadonlyMap<string, LiveEnvValue>,
+  names: readonly string[],
+): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const name of names) {
+    const v = liveEnv.get(name);
+    if (v?.kind === "value" && v.value) out[name] = v.value;
+  }
+  return out;
+}
+
+/** Whether a database URL dials an IP literal (an unparsable one does not). */
+export function dialsIpAddress(url: string): boolean {
+  try {
+    return isIpLiteral(new URL(url).hostname);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * `url` with its host replaced by `host`, everything else kept — or null
+ * when it already dials a host name (nothing to move).
+ */
+export function urlOnDnsHost(url: string, host: string): string | null {
+  const parsed = new URL(url);
+  if (!isIpLiteral(parsed.hostname)) return null;
+  parsed.hostname = host;
+  return parsed.toString();
+}
+
+/**
+ * Switch the live database URLs of `services` from the instance's IP to
+ * `host`, keeping their `sslmode` (the running image may not carry the CA
+ * bundle yet; the roll sets verify-full with the image that does). Every
+ * reader of the database host — the rolls, the migration job, the upgrader
+ * and operator-access jobs — takes it from these URLs, so after this they
+ * all dial the name the server's certificate carries.
+ *
+ * @param liveUrls the plain URL values each service carries now
+ */
+export async function moveServicesToDnsHost(
+  target: { readonly projectId: string; readonly region: string },
+  services: readonly {
+    readonly serviceName: string;
+    readonly liveUrls: Readonly<Record<string, string>>;
+  }[],
+  host: string,
+  deps: CloudSqlTlsDeps,
+): Promise<{ ok: true; done: string[] } | { ok: false; error: string; done: string[] }> {
+  const sleep = deps.sleep ?? realSleep;
+  const done: string[] = [];
+  for (const s of services) {
+    const moved = Object.entries(s.liveUrls)
+      .map(([name, url]) => [name, urlOnDnsHost(url, host)] as const)
+      .filter((e): e is readonly [string, string] => e[1] !== null);
+    if (moved.length === 0) continue;
+    const r = await runWithRetry(deps.run, sleep, [
+      "run",
+      "services",
+      "update",
+      s.serviceName,
+      "--region",
+      target.region,
+      `--project=${target.projectId}`,
+      // ^|^ — the values carry `&` and `=`; a custom delimiter keeps gcloud
+      // from splitting them.
+      `--update-env-vars=^|^${moved.map(([n, u]) => `${n}=${u}`).join("|")}`,
+      "--quiet",
+    ]);
+    if (!r.ok) {
+      return {
+        ok: false,
+        error: `move ${s.serviceName} onto the database's DNS name: ${r.stderr.trim()}`,
+        done,
+      };
+    }
+    done.push(`${s.serviceName}: ${moved.map(([n]) => n).join(", ")} → ${host}`);
+  }
+  return { ok: true, done };
+}

package/src/compose.ts

@@ -3,9 +3,9 @@
 /**
  * P14 — production docker-compose.yml generator.
  *
- * Emits the production stack: Postgres + Caddy + the two Caelo services
- * (admin, gateway). Idempotent: same input →
- * byte-identical output. The CLI's `init` / `up` / `upgrade` sub-commands
+ * Emits the production stack: Postgres + Caddy + the Caelo services
+ * (admin, gateway and — #1328 — the MCP-Auth-Proxy). Idempotent: same
+ * input → byte-identical output. The CLI's `init` / `up` / `upgrade` sub-commands
  * write this to `.caelo/docker-compose.yml`.
  *
  * #696 — the admin and gateway run the PREBUILT, cosign-verified release
@@ -27,23 +27,37 @@
  * every fresh `docker compose up` fail.
  */
 
-import { CADDY_REDIRECTS_FILE } from "@caelo-cms/shared";
+import { CADDY_REDIRECTS_FILE, SELF_HOSTED_CA_FILE, verifyFullQuery } from "@caelo-cms/shared";
 import { caddyRedirectReloadCommand } from "./caddy-redirects.js";
 import { DEFAULT_MAX_CONNECTIONS, stackPoolBudget } from "./pool-budget-stack.js";
+import {
+  SELF_HOSTED_DB_HOST,
+  SELF_HOSTED_TLS_DIR,
+  SELF_HOSTED_TLS_FILES,
+} from "./self-hosted-db-tls.js";
 import {
   ADMIN_MEMORY_DEFAULT,
   GATEWAY_TRUSTED_PROXY_HOPS,
+  MCP_AUTH_PROXY_TRUSTED_PROXY_HOPS,
   memoryQuantityMiB,
   publicSiteUrl,
+  REMOTE_MCP_MODE,
+  REMOTE_MCP_MODE_ENV,
   TRUSTED_PROXY_HOPS_ENV,
 } from "./stack-contract.js";
 import { ghcrRepository, isImageDigest, type ReleaseService } from "./upgrade/release-images.js";
 
-/** The admin + gateway release images, each `ghcr.io/caelo-cms/<svc>@sha256:…`. */
+/** The release images, each `ghcr.io/caelo-cms/<svc>@sha256:…`. */
 export interface ReleaseImages {
   readonly mode: "release";
   readonly admin: string;
   readonly gateway: string;
+  /**
+   * #1328 — the MCP-Auth-Proxy. Absent only for a release recorded before
+   * the proxy shipped (a rollback target in release.json): its Compose file
+   * then has no proxy service, exactly as that release ran.
+   */
+  readonly "mcp-auth-proxy"?: string;
 }
 
 /**
@@ -102,11 +116,20 @@ export const MEDIA_ROOT = "/data/media";
 /** The gateway's HTTP port inside the Compose network. */
 export const COMPOSE_GATEWAY_PORT = 8090;
 
+/** #1328 — the MCP-Auth-Proxy's HTTP port inside the Compose network. */
+export const COMPOSE_MCP_AUTH_PROXY_PORT = 8095;
+
+/** #1328 — the Compose service name of the MCP-Auth-Proxy. */
+export const COMPOSE_MCP_AUTH_PROXY_SERVICE = "caelo-mcp-auth-proxy";
+
 /**
- * The Compose services that run Caelo code — what `restore` stops while it
- * rewrites the databases, and what the health checks probe. Workers run
- * inside the admin (CLAUDE.md §11.B Tier 5); there is no orchestrator or
- * runner service.
+ * The Compose services that touch the databases — what `backup` and
+ * `restore` stop while they read or rewrite them. Workers run inside the
+ * admin (CLAUDE.md §11.B Tier 5); there is no orchestrator or runner
+ * service. The MCP-Auth-Proxy (#1328) is deliberately not here: it holds no
+ * database login, so it keeps running, and a release from before it has no
+ * such service for `up` to start. Its health is checked wherever the
+ * install starts (self-hosted-start.ts) or rolls (self-hosted-upgrade.ts).
  */
 export const COMPOSE_APP_SERVICES = ["caelo-admin", "caelo-gateway"] as const;
 
@@ -117,6 +140,7 @@ export const COMPOSE_APP_SERVICES = ["caelo-admin", "caelo-gateway"] as const;
  */
 export const COMPOSE_ADMIN_UPSTREAM = `caelo-admin:${COMPOSE_ADMIN_PORT}`;
 export const COMPOSE_GATEWAY_UPSTREAM = `caelo-gateway:${COMPOSE_GATEWAY_PORT}`;
+export const COMPOSE_MCP_AUTH_PROXY_UPSTREAM = `${COMPOSE_MCP_AUTH_PROXY_SERVICE}:${COMPOSE_MCP_AUTH_PROXY_PORT}`;
 
 /**
  * The Caddy image, pinned to the release the generated Caddyfile is
@@ -213,6 +237,15 @@ export interface ComposeSpec {
    * the gateway never gets it.
    */
   readonly internalSecret: string;
+  /**
+   * #1338 — SHA-256 (hex) of the database server certificate the Postgres
+   * service mounts (`.caelo/postgres-tls/server.crt`). It is part of the
+   * service's environment so a renewed certificate changes the service's
+   * definition and `docker compose up -d` recreates Postgres: a bind-mounted
+   * file whose content changes does not, and Postgres would keep serving
+   * the old certificate until verify-full clients refuse it.
+   */
+  readonly dbTlsServerCertSha256: string;
   readonly anthropicApiKey?: string;
   readonly resendApiKey?: string;
   readonly diskSize: string;
@@ -223,7 +256,7 @@ export interface ComposeSpec {
    * busy host never OOM-kills the admin at the limit.
    */
   readonly adminMemory?: string;
-  /** Which admin + gateway images run (see {@link ComposeImages}). */
+  /** Which Caelo images run (see {@link ComposeImages}). */
   readonly images: ComposeImages;
   /**
    * The host ports Caddy listens on (`80` / `443` for a public install —
@@ -274,6 +307,11 @@ export function generateDockerCompose(spec: ComposeSpec): string {
   if (spec.internalSecret.length < 32) {
     throw new Error("internalSecret is shorter than 32 characters (.caelo/internal-secret.json)");
   }
+  if (!/^[0-9a-f]{64}$/.test(spec.dbTlsServerCertSha256)) {
+    throw new Error(
+      "dbTlsServerCertSha256 is not a SHA-256 hex digest of .caelo/postgres-tls/server.crt",
+    );
+  }
   return `# SPDX-License-Identifier: MPL-2.0
 # Generated by cms-provision — do not edit; re-run \`bunx cms-provision up\`.
 # Domain: ${spec.domain}
@@ -281,10 +319,16 @@ export function generateDockerCompose(spec: ComposeSpec): string {
 name: ${spec.project}
 
 services:
-  postgres:
+  ${SELF_HOSTED_DB_HOST}:
     image: postgres:16-alpine
-    # The pool budget (COMPOSE_POOL_BUDGET) is computed from this number.
-    command: ["postgres", "-c", "max_connections=${DEFAULT_MAX_CONNECTIONS["self-hosted"]}"]
+    # #1338 — TLS with the install's own certificate for the host name
+    # \`${SELF_HOSTED_DB_HOST}\` (self-hosted-db-tls.ts); every client connects with
+    # verify-full. Postgres refuses a key file it does not own, and a bind
+    # mount keeps the host's owner, so the entrypoint copies the pair to a
+    # container-local path owned by postgres first.
+    entrypoint: ${POSTGRES_TLS_ENTRYPOINT}
+    # The pool budget (COMPOSE_POOL_BUDGET) is computed from max_connections.
+    command: ["postgres", "-c", "max_connections=${DEFAULT_MAX_CONNECTIONS["self-hosted"]}", "-c", "ssl=on", "-c", "ssl_cert_file=${POSTGRES_TLS_RUNTIME_DIR}/server.crt", "-c", "ssl_key_file=${POSTGRES_TLS_RUNTIME_DIR}/server.key", "-c", "ssl_min_protocol_version=TLSv1.2"]
     restart: unless-stopped
     environment:
       POSTGRES_PASSWORD: "${escapeYaml(spec.postgresPassword)}"
@@ -292,9 +336,13 @@ services:
       POSTGRES_DB: caelo
       ADMIN_ROLE_PASSWORD: "${escapeYaml(spec.rolePasswords.admin)}"
       PUBLIC_ROLE_PASSWORD: "${escapeYaml(spec.rolePasswords.public)}"
+      # Recreates Postgres when its certificate is renewed (see the spec).
+      CAELO_DB_TLS_SERVER_CERT_SHA256: "${spec.dbTlsServerCertSha256}"
 ${postgresAgent(spec.agent)}    volumes:
       - caelo-pg:/var/lib/postgresql/data
       - ./bootstrap.sh:/docker-entrypoint-initdb.d/00_bootstrap.sh:ro
+      - ./${SELF_HOSTED_TLS_DIR}/${SELF_HOSTED_TLS_FILES.serverCert}:${POSTGRES_TLS_MOUNT_DIR}/server.crt:ro
+      - ./${SELF_HOSTED_TLS_DIR}/${SELF_HOSTED_TLS_FILES.serverKey}:${POSTGRES_TLS_MOUNT_DIR}/server.key:ro
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U caelo"]
       interval: 10s
@@ -317,7 +365,7 @@ ${postgresAgent(spec.agent)}    volumes:
 ${adminService(spec, adminMemoryMiB, env)}
 
 ${gatewayService(spec)}
-
+${mcpAuthProxyService(spec.images)}
 volumes:
   caelo-pg:
   caelo-caddy-data:
@@ -369,10 +417,11 @@ function serviceRuntime(
       ...volumes(["../..:/app", ...mounts]),
     ].join("\n");
   }
-  const digest = ghcrImageDigest(service, images[service]);
+  const ref = images[service] ?? "";
+  const digest = ghcrImageDigest(service, ref);
   if (!digest) {
     throw new Error(
-      `${service} image ${JSON.stringify(images[service])} is not ${ghcrRepository(service)}@sha256:<digest>`,
+      `${service} image ${JSON.stringify(ref)} is not ${ghcrRepository(service)}@sha256:<digest>`,
     );
   }
   // The image's own CMD starts the service; no source is mounted.
@@ -398,6 +447,7 @@ ${serviceRuntime(
     `./${BOOTSTRAP_DIR}:/app/.caelo`,
     `./output:${SITE_OUTPUT_ROOT}`,
     `./${MEDIA_DIR}:${MEDIA_ROOT}`,
+    DATABASE_CA_MOUNT,
   ],
 )}
     restart: unless-stopped
@@ -422,7 +472,7 @@ ${env("RESEND_API_KEY", spec.resendApiKey)}`;
 
 function gatewayService(spec: ComposeSpec): string {
   return `  caelo-gateway:
-${serviceRuntime(spec.images, "gateway", "bun run apps/api-gateway/src/server.ts")}
+${serviceRuntime(spec.images, "gateway", "bun run apps/api-gateway/src/server.ts", [DATABASE_CA_MOUNT])}
     restart: unless-stopped
     depends_on:
       postgres: { condition: service_healthy }
@@ -434,19 +484,70 @@ ${serviceRuntime(spec.images, "gateway", "bun run apps/api-gateway/src/server.ts
       # Caddy is the one proxy in front (client-ip.ts in the gateway).
       ${TRUSTED_PROXY_HOPS_ENV}: "${GATEWAY_TRUSTED_PROXY_HOPS["self-hosted"]}"
 ${poolEnvLines("gateway")}
-      # #806 — no remote MCP here: it rides Google IAP, which this install
-      # does not have. CAELO_ADMIN_URL / CAELO_MCP_PUBLIC_URL /
-      # CAELO_OPERATOR_OAUTH_CLIENT_FILE stay unset, so the gateway's /api/mcp/*
-      # answers 503 remote_mcp_not_configured and Settings → MCP says
-      # "not available on this install".
       NODE_ENV: production`;
 }
 
-/** A compose-network database URL, password URL-encoded and YAML-escaped. */
+/**
+ * #1328 — the MCP-Auth-Proxy: remote MCP's sign-in broker, its own
+ * container so it never shares a process with visitor handlers. Remote MCP
+ * needs the admin behind Google IAP, which a Compose install does not have,
+ * so it runs with `CAELO_REMOTE_MCP=unavailable` and answers every request
+ * but its health check with one fixed 404 (E14 a of #763: the service
+ * exists on every provider, its attack surface one branch). Caddy serves it
+ * on its own host (`mcp.<domain>`, caddy.ts), never on the site's origin.
+ *
+ * It gets no database URL, no secret and no mount: there is nothing it
+ * could read. The container is locked down on top — read-only root
+ * filesystem, every capability dropped, no privilege escalation — because
+ * it is the Caelo service that holds operators' Google tokens wherever
+ * remote MCP runs. Empty for a release recorded before the proxy shipped
+ * ({@link ReleaseImages}).
+ */
+function mcpAuthProxyService(images: ComposeImages): string {
+  if (images.mode === "release" && images["mcp-auth-proxy"] === undefined) return "";
+  return `
+  ${COMPOSE_MCP_AUTH_PROXY_SERVICE}:
+${serviceRuntime(images, "mcp-auth-proxy", "bun run apps/mcp-auth-proxy/src/server.ts")}
+    restart: unless-stopped
+    read_only: true
+    tmpfs: ["/tmp"]
+    cap_drop: ["ALL"]
+    security_opt: ["no-new-privileges:true"]
+    environment:
+      CAELO_PROVIDER: "self-hosted"
+      ${REMOTE_MCP_MODE_ENV}: "${REMOTE_MCP_MODE["self-hosted"]}"
+      MCP_AUTH_PROXY_PORT: "${COMPOSE_MCP_AUTH_PROXY_PORT}"
+      # Caddy is the one proxy in front (@caelo-cms/shared client-address.ts).
+      ${TRUSTED_PROXY_HOPS_ENV}: "${MCP_AUTH_PROXY_TRUSTED_PROXY_HOPS["self-hosted"]}"
+      NODE_ENV: production
+`;
+}
+
+/**
+ * A compose-network database URL, password URL-encoded and YAML-escaped.
+ * #1338 — verify-full against the install's own CA (self-hosted-db-tls.ts),
+ * mounted at shared `SELF_HOSTED_CA_FILE` ({@link DATABASE_CA_MOUNT}).
+ */
 function dbUrl(role: string, password: string, database: string): string {
-  return escapeYaml(`postgres://${role}:${encodeURIComponent(password)}@postgres:5432/${database}`);
+  return escapeYaml(
+    `postgres://${role}:${encodeURIComponent(password)}@${SELF_HOSTED_DB_HOST}:5432/${database}?${verifyFullQuery(SELF_HOSTED_CA_FILE)}`,
+  );
 }
 
+/** Where the server certificate + key are bind-mounted into Postgres (host-owned). */
+const POSTGRES_TLS_MOUNT_DIR = "/caelo-tls";
+/** Where the entrypoint copies them, owned by postgres (Postgres refuses others' keys). */
+const POSTGRES_TLS_RUNTIME_DIR = "/var/lib/postgresql/caelo-tls";
+/** The Postgres entrypoint: copy the TLS pair into place, then the image's own entrypoint. */
+const POSTGRES_TLS_ENTRYPOINT = JSON.stringify([
+  "/bin/sh",
+  "-c",
+  `install -d -o postgres -g postgres -m 0700 ${POSTGRES_TLS_RUNTIME_DIR} && install -o postgres -g postgres -m 0644 ${POSTGRES_TLS_MOUNT_DIR}/server.crt ${POSTGRES_TLS_RUNTIME_DIR}/server.crt && install -o postgres -g postgres -m 0600 ${POSTGRES_TLS_MOUNT_DIR}/server.key ${POSTGRES_TLS_RUNTIME_DIR}/server.key && exec docker-entrypoint.sh "$$@"`,
+  "--",
+]);
+/** The install's database CA, read-only, where every verify-full URL names it. */
+const DATABASE_CA_MOUNT = `./${SELF_HOSTED_TLS_DIR}/${SELF_HOSTED_TLS_FILES.caCert}:${SELF_HOSTED_CA_FILE}:ro`;
+
 function escapeYaml(s: string): string {
   // Conservative escape for double-quoted YAML strings.
   return s.replace(/\\/g, "\\\\").replace(/"/g, '\\"');

package/src/converge-checkpoint.ts

@@ -44,7 +44,14 @@ export const ConvergeCheckpointSchema = InstallJobStatusSchema.extend({
       backupId: z.string().min(1),
     })
     .optional(),
-  priorRevisions: z.object({ admin: z.string().min(1), gateway: z.string().min(1) }).optional(),
+  priorRevisions: z
+    .object({
+      admin: z.string().min(1),
+      gateway: z.string().min(1),
+      // #1328 — absent in a checkpoint an older release wrote.
+      "mcp-auth-proxy": z.string().min(1).optional(),
+    })
+    .optional(),
 });
 /**
  * The ops-bucket prefix of the converge job's own files — outside the

package/src/converge-job-gcp.ts

@@ -42,7 +42,7 @@ import {
 } from "./install-state.js";
 import { runMigrationsViaCloudRunJob } from "./migration-runner.js";
 import { installIapAllowlist } from "./operator-access.js";
-import { fetchVerifiedReleaseManifest } from "./release-manifest.js";
+import { fetchVerifiedReleaseManifest, manifestServiceImages } from "./release-manifest.js";
 import type { HttpFetch } from "./runtime-secrets.js";
 import { publicSiteUrl, STAGING_HOST_ENV } from "./stack-contract.js";
 import { attachInstall } from "./state-attach.js";
@@ -314,16 +314,7 @@ export function gcpConvergeDeps(
         sleep: realSleep,
         reporter,
         // The signed manifest's digests — never a registry tag lookup.
-        resolveImages: async () => ({
-          ok: true,
-          images: {
-            admin: { digest: m.digests.admin, imageRef: releaseImageRef("admin", m.digests.admin) },
-            gateway: {
-              digest: m.digests.gateway,
-              imageRef: releaseImageRef("gateway", m.digests.gateway),
-            },
-          },
-        }),
+        resolveImages: async () => manifestServiceImages(m),
         verifyImages: verifyCosignAll,
         migrate: (imageRef) =>
           runMigrationsViaCloudRunJob(