@caelo-cms/provisioning 0.10.46 → 0.10.48

package/src/cloudsql-iam-user.ts

@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * A Cloud SQL IAM database user for one of the install's in-cloud jobs
+ * (the operator-access sync job, the upgrader job): IAM database
+ * authentication switched on for the instance, the job's service account
+ * as a `cloud_iam_service_account` user, and that user made a member of
+ * exactly one database role a migration created for it. No password ever
+ * exists; the job logs in with its short-lived access token.
+ */
+
+import type { GcloudResult } from "./gcloud.js";
+import { gcpNamePrefix } from "./gcp-names.js";
+
+type Run = (args: string[]) => Promise<GcloudResult>;
+
+/** The flags of a Cloud SQL instance with `cloudsql.iam_authentication` on. */
+export function withIamAuthFlag(
+  flags: readonly { name: string; value: string }[],
+): { name: string; value: string }[] | null {
+  if (flags.some((f) => f.name === "cloudsql.iam_authentication" && f.value === "on")) return null;
+  return [
+    ...flags.filter((f) => f.name !== "cloudsql.iam_authentication"),
+    { name: "cloudsql.iam_authentication", value: "on" },
+  ];
+}
+
+export type CloudSqlIamUserResult =
+  | { readonly ok: true; readonly done: string[] }
+  | { readonly ok: false; readonly done: string[]; readonly error: string };
+
+/**
+ * Ensure `serviceAccountEmail` logs in to the install's Cloud SQL instance
+ * as an IAM database user holding `databaseRole`. Idempotent; never throws
+ * for a gcloud failure.
+ */
+export async function ensureCloudSqlIamUser(
+  run: Run,
+  opts: {
+    readonly projectId: string;
+    readonly env: string;
+    readonly serviceAccountEmail: string;
+    readonly databaseRole: string;
+  },
+): Promise<CloudSqlIamUserResult> {
+  const project = `--project=${opts.projectId}`;
+  const done: string[] = [];
+  const fail = (step: string, r: GcloudResult): CloudSqlIamUserResult => ({
+    ok: false,
+    done,
+    error: `${step}: ${r.stderr.trim()}`,
+  });
+  const instances = await run([
+    "sql",
+    "instances",
+    "list",
+    project,
+    `--filter=name~^${gcpNamePrefix(opts.env)}-pg`,
+    "--format=value(name)",
+  ]);
+  if (!instances.ok) return fail("list Cloud SQL instances", instances);
+  const names = instances.stdout
+    .split("\n")
+    .map((s) => s.trim())
+    .filter(Boolean);
+  if (names.length !== 1) {
+    return {
+      ok: false,
+      done,
+      error: `expected exactly one ${gcpNamePrefix(opts.env)}-pg* Cloud SQL instance, found ${names.length} (${names.join(", ")})`,
+    };
+  }
+  const instance = names[0] as string;
+  const flagsJson = await run([
+    "sql",
+    "instances",
+    "describe",
+    instance,
+    project,
+    "--format=json(settings.databaseFlags)",
+  ]);
+  if (!flagsJson.ok) return fail(`read the flags of ${instance}`, flagsJson);
+  const flags =
+    (
+      JSON.parse(flagsJson.stdout) as {
+        settings?: { databaseFlags?: { name: string; value: string }[] };
+      }
+    ).settings?.databaseFlags ?? [];
+  const nextFlags = withIamAuthFlag(flags);
+  if (nextFlags) {
+    // `--database-flags` replaces the whole list, so every existing flag is
+    // passed again. cloudsql.iam_authentication needs no restart.
+    const patch = await run([
+      "sql",
+      "instances",
+      "patch",
+      instance,
+      project,
+      `--database-flags=${nextFlags.map((f) => `${f.name}=${f.value}`).join(",")}`,
+      "--quiet",
+    ]);
+    if (!patch.ok) return fail(`enable IAM database authentication on ${instance}`, patch);
+    done.push(`${instance}: cloudsql.iam_authentication=on`);
+  }
+  const dbUser = opts.serviceAccountEmail.replace(/\.gserviceaccount\.com$/, "");
+  const users = await run([
+    "sql",
+    "users",
+    "list",
+    `--instance=${instance}`,
+    project,
+    "--format=value(name)",
+  ]);
+  if (!users.ok) return fail(`list database users of ${instance}`, users);
+  if (
+    !users.stdout
+      .split("\n")
+      .map((s) => s.trim())
+      .includes(dbUser)
+  ) {
+    const create = await run([
+      "sql",
+      "users",
+      "create",
+      dbUser,
+      `--instance=${instance}`,
+      "--type=cloud_iam_service_account",
+      project,
+      "--quiet",
+    ]);
+    if (!create.ok) return fail(`create database user ${dbUser}`, create);
+  }
+  const assign = await run([
+    "sql",
+    "users",
+    "assign-roles",
+    dbUser,
+    `--instance=${instance}`,
+    "--type=cloud_iam_service_account",
+    `--database-roles=${opts.databaseRole}`,
+    project,
+    "--quiet",
+  ]);
+  if (!assign.ok) return fail(`give ${dbUser} the ${opts.databaseRole} database role`, assign);
+  done.push(`database user ${dbUser} (${opts.databaseRole})`);
+  return { ok: true, done };
+}

package/src/compose.ts

@@ -0,0 +1,334 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * P14 — production docker-compose.yml generator.
+ *
+ * Emits the production stack: Postgres + pgBackRest sidecar + Caddy +
+ * MinIO + the two Caelo services (admin, gateway). Idempotent: same input →
+ * byte-identical output. The CLI's `init` / `up` / `upgrade` sub-commands
+ * write this to `.caelo/docker-compose.yml`.
+ *
+ * #696 — the admin and gateway run the PREBUILT, cosign-verified release
+ * images, pinned by digest (CLAUDE.md §11.C: "a PR that requires the user to
+ * run docker build is a regression"). Nothing of a Caelo source checkout is
+ * mounted. The digest pin is what `upgrade` moves; a tag would let a
+ * `docker compose pull` change the running release behind the operator's
+ * back. Contributors who want their working tree in the containers ask for
+ * {@link DevImages} explicitly (`init --dev`) — never the default.
+ */
+
+import { ADMIN_MEMORY_DEFAULT, memoryQuantityMiB, publicSiteUrl } from "./stack-contract.js";
+import { ghcrRepository, isImageDigest, type ReleaseService } from "./upgrade/release-images.js";
+
+/** The admin + gateway release images, each `ghcr.io/caelo-cms/<svc>@sha256:…`. */
+export interface ReleaseImages {
+  readonly mode: "release";
+  readonly admin: string;
+  readonly gateway: string;
+}
+
+/**
+ * Contributor-only: run the admin + gateway from the Caelo checkout the
+ * `.caelo/` dir lives in (bind mount on `oven/bun`). Never for an end-user
+ * install — it needs the repo and builds nothing reproducible.
+ */
+export interface DevImages {
+  readonly mode: "dev";
+}
+
+export type ComposeImages = ReleaseImages | DevImages;
+
+/** The admin's HTTP port inside the Compose network (the Caddyfile proxies to it). */
+export const COMPOSE_ADMIN_PORT = 5173;
+/** `.caelo/<this>/` holds the pending Owner-bootstrap token the admin consumes. */
+export const BOOTSTRAP_DIR = "bootstrap";
+
+/**
+ * Where the host's `.caelo/output/` appears in the admin (read-write, the
+ * generator's output root) and in Caddy (read-only). The seeded deploy
+ * targets' `out_dir` (`output/<env>`) resolves below it, which is where the
+ * Caddyfile's site roots point (`/srv/caelo/output/<env>/current`).
+ */
+export const SITE_OUTPUT_ROOT = "/srv/caelo";
+
+/** `.caelo/<this>/` holds the uploaded media (the admin's local media adapter root). */
+export const MEDIA_DIR = "media";
+/** Where {@link MEDIA_DIR} appears in the admin container (its `MEDIA_ROOT_DIR`). */
+export const MEDIA_ROOT = "/data/media";
+
+/** The gateway's HTTP port inside the Compose network. */
+export const COMPOSE_GATEWAY_PORT = 8090;
+
+/**
+ * `ghcr.io/caelo-cms/<service>@<digest>` — the only image reference shape a
+ * release-mode Compose file accepts.
+ */
+export function ghcrImageRef(service: ReleaseService, digest: string): string {
+  if (!isImageDigest(digest)) throw new Error(`not an image digest: ${JSON.stringify(digest)}`);
+  return `${ghcrRepository(service)}@${digest}`;
+}
+
+/**
+ * The digest of a release image reference, or null when `ref` is not a
+ * digest-pinned reference to that service's Caelo repository.
+ */
+export function ghcrImageDigest(service: ReleaseService, ref: string): string | null {
+  const prefix = `${ghcrRepository(service)}@`;
+  if (!ref.startsWith(prefix)) return null;
+  const digest = ref.slice(prefix.length);
+  return isImageDigest(digest) ? digest : null;
+}
+
+/**
+ * The application roles' passwords (bootstrap.sh creates the roles with
+ * them). One per role, so the gateway — which gets only the public and
+ * gateway logins — never holds admin_role's (#613, CLAUDE.md §2).
+ */
+export interface RolePasswords {
+  readonly admin: string;
+  readonly public: string;
+  readonly gateway: string;
+}
+
+export interface ComposeSpec {
+  readonly domain: string;
+  /** The Postgres superuser's; only the database container uses it. */
+  readonly postgresPassword: string; // generated by cms-provision; persisted in .caelo/secrets
+  readonly rolePasswords: RolePasswords;
+  readonly minioRootUser: string;
+  readonly minioRootPassword: string;
+  /**
+   * P18 — project KEK (32 bytes hex) for at-rest encryption of AI provider
+   * keys + future secrets. Generated per install by cms-provision; persisted
+   * in `.caelo/secrets`. NEVER prompt the operator for this — auto-generated
+   * via the same randomHex(32) the GCP stack uses.
+   */
+  readonly caeloSecretKek: string;
+  readonly anthropicApiKey?: string;
+  readonly resendApiKey?: string;
+  readonly diskSize: string;
+  /**
+   * #553 — memory reserved for the admin (it runs the Lighthouse quality
+   * audit). Same `adminMemory` knob + default as every adapter
+   * (stack-contract.ts); compose reserves it rather than capping it, so a
+   * busy host never OOM-kills the admin at the limit.
+   */
+  readonly adminMemory?: string;
+  /** Which admin + gateway images run (see {@link ComposeImages}). */
+  readonly images: ComposeImages;
+  /**
+   * #696 Phase 4c — the host update agent's database login, when the agent
+   * is set up: Postgres is then also published on `127.0.0.1:<dbPort>` (never
+   * on another interface) for the agent on the host, and bootstrap.sh
+   * creates the `caelo_agent` role from `AGENT_ROLE_PASSWORD`.
+   */
+  readonly agent?: ComposeAgent;
+}
+
+/** The host agent's side of the Compose file (see {@link ComposeSpec.agent}). */
+export interface ComposeAgent {
+  readonly dbPort: number;
+  readonly dbPassword: string;
+}
+
+/** The only interface the agent's Postgres port is published on. */
+export const AGENT_DB_BIND_ADDRESS = "127.0.0.1";
+
+export function generateDockerCompose(spec: ComposeSpec): string {
+  const adminMemory = spec.adminMemory ?? ADMIN_MEMORY_DEFAULT;
+  const adminMemoryMiB = memoryQuantityMiB(adminMemory);
+  if (adminMemoryMiB === null) {
+    throw new Error(`adminMemory "${adminMemory}" is not a memory quantity (e.g. 2Gi)`);
+  }
+  const env = (k: string, v: string | undefined): string =>
+    v ? `      ${k}: "${escapeYaml(v)}"` : "";
+  if (spec.agent && !isAgentDbPort(spec.agent.dbPort)) {
+    throw new Error(`agent dbPort ${spec.agent.dbPort} is not a TCP port between 1024 and 65535`);
+  }
+  return `# SPDX-License-Identifier: MPL-2.0
+# Generated by cms-provision — do not edit; re-run \`bunx cms-provision up\`.
+# Domain: ${spec.domain}
+
+services:
+  postgres:
+    image: postgres:16-alpine
+    container_name: caelo-postgres
+    restart: unless-stopped
+    environment:
+      POSTGRES_PASSWORD: "${escapeYaml(spec.postgresPassword)}"
+      POSTGRES_USER: caelo
+      POSTGRES_DB: caelo
+      ADMIN_ROLE_PASSWORD: "${escapeYaml(spec.rolePasswords.admin)}"
+      PUBLIC_ROLE_PASSWORD: "${escapeYaml(spec.rolePasswords.public)}"
+      GATEWAY_ROLE_PASSWORD: "${escapeYaml(spec.rolePasswords.gateway)}"
+${postgresAgent(spec.agent)}    volumes:
+      - caelo-pg:/var/lib/postgresql/data
+      - ./bootstrap.sh:/docker-entrypoint-initdb.d/00_bootstrap.sh:ro
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U caelo"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  pgbackrest:
+    image: pgbackrest/pgbackrest:latest
+    container_name: caelo-pgbackrest
+    restart: unless-stopped
+    depends_on: [postgres]
+    volumes:
+      - caelo-pg-backups:/var/lib/pgbackrest
+      - ./pgbackrest.conf:/etc/pgbackrest.conf:ro
+
+  minio:
+    image: minio/minio:latest
+    container_name: caelo-minio
+    restart: unless-stopped
+    command: server /data --console-address ":9001"
+    environment:
+      MINIO_ROOT_USER: "${escapeYaml(spec.minioRootUser)}"
+      MINIO_ROOT_PASSWORD: "${escapeYaml(spec.minioRootPassword)}"
+    volumes:
+      - caelo-minio:/data
+
+  caddy:
+    image: caddy:2-alpine
+    container_name: caelo-caddy
+    restart: unless-stopped
+    ports: ["80:80", "443:443"]
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile:ro
+      - caelo-caddy-data:/data
+      - caelo-caddy-config:/config
+      - ./output:${SITE_OUTPUT_ROOT}:ro
+
+${adminService(spec, adminMemoryMiB, env)}
+
+${gatewayService(spec)}
+
+volumes:
+  caelo-pg:
+  caelo-pg-backups:
+  caelo-minio:
+  caelo-caddy-data:
+  caelo-caddy-config:
+`;
+}
+
+type EnvLine = (k: string, v: string | undefined) => string;
+
+/** Whether `port` may carry the agent's Postgres (unprivileged TCP port). */
+export function isAgentDbPort(port: number): boolean {
+  return Number.isInteger(port) && port >= 1024 && port <= 65535;
+}
+
+/**
+ * The agent's lines in the postgres service: its role password for
+ * bootstrap.sh, and the port on the host's loopback interface only — the
+ * agent runs on the host, and nothing else may reach the database.
+ */
+function postgresAgent(agent: ComposeAgent | undefined): string {
+  if (!agent) return "";
+  return [
+    `      AGENT_ROLE_PASSWORD: "${escapeYaml(agent.dbPassword)}"`,
+    "    # The host update agent (cms-provision agent): loopback only.",
+    `    ports: ["${AGENT_DB_BIND_ADDRESS}:${agent.dbPort}:5432"]`,
+    "",
+  ].join("\n");
+}
+
+/**
+ * How a Caelo service's container starts: its image and, in dev mode, the
+ * checkout. `mounts` are extra bind mounts the service needs in both modes.
+ */
+function serviceRuntime(
+  images: ComposeImages,
+  service: ReleaseService,
+  devCommand: string,
+  mounts: readonly string[] = [],
+): string {
+  const volumes = (list: readonly string[]) =>
+    list.length === 0 ? [] : ["    volumes:", ...list.map((m) => `      - ${m}`)];
+  if (images.mode === "dev") {
+    return [
+      "    # DEV MODE (cms-provision init --dev): runs the Caelo checkout this",
+      "    # .caelo/ dir lives in. Contributors only — never an end-user install.",
+      "    image: oven/bun:1.4.2",
+      "    working_dir: /app",
+      `    command: ${devCommand}`,
+      ...volumes(["../..:/app", ...mounts]),
+    ].join("\n");
+  }
+  const digest = ghcrImageDigest(service, images[service]);
+  if (!digest) {
+    throw new Error(
+      `${service} image ${JSON.stringify(images[service])} is not ${ghcrRepository(service)}@sha256:<digest>`,
+    );
+  }
+  // The image's own CMD starts the service; no source is mounted.
+  return [`    image: ${ghcrImageRef(service, digest)}`, ...volumes(mounts)].join("\n");
+}
+
+function adminService(spec: ComposeSpec, adminMemoryMiB: number, env: EnvLine): string {
+  return `  caelo-admin:
+${serviceRuntime(
+  spec.images,
+  "admin",
+  `sh -c "cd apps/admin && bunx svelte-kit sync && bun --bun vite dev --host 0.0.0.0 --port ${COMPOSE_ADMIN_PORT}"`,
+  // The Owner-bootstrap token handoff: `init` writes the token to
+  // `.caelo/bootstrap/pending-token.json`; the admin reads (and deletes) it
+  // from `/app/.caelo/` on first boot (hooks.server.ts). Only that dir is
+  // mounted — never `.caelo/` itself, which holds config.json's superuser
+  // and MinIO passwords. Without the token /setup accepts any first Owner.
+  // The static site: the admin builds into it (CAELO_OUTPUT_ROOT below),
+  // Caddy serves it read-only from the same host dir. On the host, so a
+  // recreated admin (upgrade) keeps the published site.
+  // Uploaded media (MEDIA_ROOT_DIR below) likewise lives on the host.
+  [
+    `./${BOOTSTRAP_DIR}:/app/.caelo`,
+    `./output:${SITE_OUTPUT_ROOT}`,
+    `./${MEDIA_DIR}:${MEDIA_ROOT}`,
+  ],
+)}
+    container_name: caelo-admin
+    restart: unless-stopped
+    depends_on:
+      postgres: { condition: service_healthy }
+    mem_reservation: ${adminMemoryMiB}m
+    environment:
+      ADMIN_DATABASE_URL: "${dbUrl("admin_role", spec.rolePasswords.admin, "cms_admin")}"
+      PUBLIC_ADMIN_DATABASE_URL: "${dbUrl("admin_role", spec.rolePasswords.admin, "cms_public")}"
+      CAELO_SECRET_KEK: "${escapeYaml(spec.caeloSecretKek)}"
+      CAELO_SITE_URL: "${escapeYaml(publicSiteUrl(spec.domain))}"
+      PORT: "${COMPOSE_ADMIN_PORT}"
+      CAELO_OUTPUT_ROOT: "${SITE_OUTPUT_ROOT}"
+      MEDIA_ROOT_DIR: "${MEDIA_ROOT}"
+      CAELO_PROVIDER: "self-hosted"
+      NODE_ENV: production
+${env("ANTHROPIC_API_KEY", spec.anthropicApiKey)}
+${env("RESEND_API_KEY", spec.resendApiKey)}`;
+}
+
+function gatewayService(spec: ComposeSpec): string {
+  return `  caelo-gateway:
+${serviceRuntime(spec.images, "gateway", "bun run apps/api-gateway/src/server.ts")}
+    container_name: caelo-gateway
+    restart: unless-stopped
+    depends_on:
+      postgres: { condition: service_healthy }
+    environment:
+      # Its own two logins only — no authoring credential (#613, CLAUDE.md §2).
+      GATEWAY_DATABASE_URL: "${dbUrl("gateway_role", spec.rolePasswords.gateway, "cms_admin")}"
+      PUBLIC_DATABASE_URL: "${dbUrl("public_role", spec.rolePasswords.public, "cms_public")}"
+      GATEWAY_PORT: "${COMPOSE_GATEWAY_PORT}"
+      NODE_ENV: production`;
+}
+
+/** A compose-network database URL, password URL-encoded and YAML-escaped. */
+function dbUrl(role: string, password: string, database: string): string {
+  return escapeYaml(`postgres://${role}:${encodeURIComponent(password)}@postgres:5432/${database}`);
+}
+
+function escapeYaml(s: string): string {
+  // Conservative escape for double-quoted YAML strings.
+  return s.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
+}