@caelo-cms/provisioning 0.10.48 → 0.10.50

package/src/compose.ts

@@ -3,8 +3,8 @@
 /**
  * P14 — production docker-compose.yml generator.
  *
- * Emits the production stack: Postgres + pgBackRest sidecar + Caddy +
- * MinIO + the two Caelo services (admin, gateway). Idempotent: same input →
+ * Emits the production stack: Postgres + Caddy + the two Caelo services
+ * (admin, gateway). Idempotent: same input →
  * byte-identical output. The CLI's `init` / `up` / `upgrade` sub-commands
  * write this to `.caelo/docker-compose.yml`.
  *
@@ -15,8 +15,20 @@
  * `docker compose pull` change the running release behind the operator's
  * back. Contributors who want their working tree in the containers ask for
  * {@link DevImages} explicitly (`init --dev`) — never the default.
+ *
+ * #707 — no `container_name`: Compose names containers per project, so an
+ * install can share a host with other stacks (a fixed `caelo-postgres`
+ * collides with the contributor dev stack's container of that name). Every
+ * CLI command addresses containers by service name (`docker compose exec`).
+ * Backups are `pg_dump`s plus the media and site files (self-hosted-backup.ts);
+ * there is no pgBackRest sidecar — it never had access to the data volume.
+ * No MinIO either: nothing in Caelo used it (media live in `.caelo/media/`),
+ * and its `minio/minio` image is no longer published, so the service made
+ * every fresh `docker compose up` fail.
  */
 
+import { CADDY_REDIRECTS_FILE } from "@caelo-cms/shared";
+import { caddyRedirectReloadCommand } from "./caddy-redirects.js";
 import { ADMIN_MEMORY_DEFAULT, memoryQuantityMiB, publicSiteUrl } from "./stack-contract.js";
 import { ghcrRepository, isImageDigest, type ReleaseService } from "./upgrade/release-images.js";
 
@@ -51,6 +63,13 @@ export const BOOTSTRAP_DIR = "bootstrap";
  */
 export const SITE_OUTPUT_ROOT = "/srv/caelo";
 
+/**
+ * #587 — every site's redirect snippet as Caddy sees it (one per deploy
+ * target below {@link SITE_OUTPUT_ROOT}); the Caddy container reloads when
+ * any of them changes (caddy-redirects.ts).
+ */
+export const COMPOSE_REDIRECT_SNIPPETS = `${SITE_OUTPUT_ROOT}/output/*/current/${CADDY_REDIRECTS_FILE}`;
+
 /** `.caelo/<this>/` holds the uploaded media (the admin's local media adapter root). */
 export const MEDIA_DIR = "media";
 /** Where {@link MEDIA_DIR} appears in the admin container (its `MEDIA_ROOT_DIR`). */
@@ -59,6 +78,66 @@ export const MEDIA_ROOT = "/data/media";
 /** The gateway's HTTP port inside the Compose network. */
 export const COMPOSE_GATEWAY_PORT = 8090;
 
+/**
+ * The Compose services that run Caelo code — what `restore` stops while it
+ * rewrites the databases, and what the health checks probe. Workers run
+ * inside the admin (CLAUDE.md §11.B Tier 5); there is no orchestrator or
+ * runner service.
+ */
+export const COMPOSE_APP_SERVICES = ["caelo-admin", "caelo-gateway"] as const;
+
+/**
+ * Where Caddy reaches the admin and the gateway: their Compose service
+ * names. Caddy runs in its own container, so `localhost` there is Caddy
+ * itself (issue #707).
+ */
+export const COMPOSE_ADMIN_UPSTREAM = `caelo-admin:${COMPOSE_ADMIN_PORT}`;
+export const COMPOSE_GATEWAY_UPSTREAM = `caelo-gateway:${COMPOSE_GATEWAY_PORT}`;
+
+/**
+ * The Caddy image, pinned to the release the generated Caddyfile is
+ * validated against (caddy-validate.test.ts runs `caddy validate` on this
+ * exact image in CI). A floating `caddy:2-alpine` could pick up a Caddy
+ * that rejects the config on the next `docker compose pull`.
+ */
+export const CADDY_IMAGE = "caddy:2.11.7-alpine";
+
+/** The host ports Caddy publishes for HTTP and HTTPS. */
+export interface PublishedPorts {
+  readonly http: number;
+  readonly https: number;
+}
+
+/**
+ * The Compose project an install's containers, network and volumes belong
+ * to, emitted as the file's top-level `name`. Without it Compose names the
+ * project after the directory holding the file — `.caelo` → `caelo` for
+ * every install, so two installs on one host would share containers and
+ * the database volume. New installs get one per domain; installs from
+ * before this was stored keep `caelo`, the name their volumes carry.
+ */
+export function composeProjectName(domain: string): string {
+  return `caelo-${domain
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")}`;
+}
+
+/** Whether `name` is a Compose project name (lowercase, digits, `-`, `_`; leading letter or digit). */
+export function isComposeProjectName(name: string): boolean {
+  return /^[a-z0-9][a-z0-9_-]*$/.test(name);
+}
+
+/** `https://<domain>`, with the published HTTPS port when it is not 443. */
+export function selfHostedSiteUrl(domain: string, httpsPort: number): string {
+  return httpsPort === 443 ? publicSiteUrl(domain) : `${publicSiteUrl(domain)}:${httpsPort}`;
+}
+
+/** Whether `port` can be published on the host (any TCP port). */
+export function isPublishablePort(port: number): boolean {
+  return Number.isInteger(port) && port >= 1 && port <= 65535;
+}
+
 /**
  * `ghcr.io/caelo-cms/<service>@<digest>` — the only image reference shape a
  * release-mode Compose file accepts.
@@ -95,8 +174,6 @@ export interface ComposeSpec {
   /** The Postgres superuser's; only the database container uses it. */
   readonly postgresPassword: string; // generated by cms-provision; persisted in .caelo/secrets
   readonly rolePasswords: RolePasswords;
-  readonly minioRootUser: string;
-  readonly minioRootPassword: string;
   /**
    * P18 — project KEK (32 bytes hex) for at-rest encryption of AI provider
    * keys + future secrets. Generated per install by cms-provision; persisted
@@ -116,6 +193,13 @@ export interface ComposeSpec {
   readonly adminMemory?: string;
   /** Which admin + gateway images run (see {@link ComposeImages}). */
   readonly images: ComposeImages;
+  /**
+   * The host ports Caddy listens on (`80` / `443` for a public install —
+   * ACME's HTTP and TLS-ALPN challenges need exactly those).
+   */
+  readonly ports: PublishedPorts;
+  /** The Compose project ({@link composeProjectName}). */
+  readonly project: string;
   /**
    * #696 Phase 4c — the host update agent's database login, when the agent
    * is set up: Postgres is then also published on `127.0.0.1:<dbPort>` (never
@@ -142,6 +226,15 @@ export function generateDockerCompose(spec: ComposeSpec): string {
   }
   const env = (k: string, v: string | undefined): string =>
     v ? `      ${k}: "${escapeYaml(v)}"` : "";
+  for (const port of [spec.ports.http, spec.ports.https]) {
+    if (!isPublishablePort(port)) throw new Error(`port ${port} is not a TCP port`);
+  }
+  if (!isComposeProjectName(spec.project)) {
+    throw new Error(`${JSON.stringify(spec.project)} is not a Compose project name`);
+  }
+  if (spec.ports.http === spec.ports.https) {
+    throw new Error(`HTTP and HTTPS cannot share port ${spec.ports.http}`);
+  }
   if (spec.agent && !isAgentDbPort(spec.agent.dbPort)) {
     throw new Error(`agent dbPort ${spec.agent.dbPort} is not a TCP port between 1024 and 65535`);
   }
@@ -149,10 +242,11 @@ export function generateDockerCompose(spec: ComposeSpec): string {
 # Generated by cms-provision — do not edit; re-run \`bunx cms-provision up\`.
 # Domain: ${spec.domain}
 
+name: ${spec.project}
+
 services:
   postgres:
     image: postgres:16-alpine
-    container_name: caelo-postgres
     restart: unless-stopped
     environment:
       POSTGRES_PASSWORD: "${escapeYaml(spec.postgresPassword)}"
@@ -170,31 +264,13 @@ ${postgresAgent(spec.agent)}    volumes:
       timeout: 5s
       retries: 5
 
-  pgbackrest:
-    image: pgbackrest/pgbackrest:latest
-    container_name: caelo-pgbackrest
-    restart: unless-stopped
-    depends_on: [postgres]
-    volumes:
-      - caelo-pg-backups:/var/lib/pgbackrest
-      - ./pgbackrest.conf:/etc/pgbackrest.conf:ro
-
-  minio:
-    image: minio/minio:latest
-    container_name: caelo-minio
-    restart: unless-stopped
-    command: server /data --console-address ":9001"
-    environment:
-      MINIO_ROOT_USER: "${escapeYaml(spec.minioRootUser)}"
-      MINIO_ROOT_PASSWORD: "${escapeYaml(spec.minioRootPassword)}"
-    volumes:
-      - caelo-minio:/data
-
   caddy:
-    image: caddy:2-alpine
-    container_name: caelo-caddy
+    image: ${CADDY_IMAGE}
+    # #587 — Caddy plus a loop that reloads it when a deploy changes a
+    # site's _redirects.caddy import (caddy-redirects.ts).
+    ${caddyRedirectReloadCommand(COMPOSE_REDIRECT_SNIPPETS)}
     restart: unless-stopped
-    ports: ["80:80", "443:443"]
+    ports: ["${spec.ports.http}:80", "${spec.ports.https}:443"]
     volumes:
       - ./Caddyfile:/etc/caddy/Caddyfile:ro
       - caelo-caddy-data:/data
@@ -207,8 +283,6 @@ ${gatewayService(spec)}
 
 volumes:
   caelo-pg:
-  caelo-pg-backups:
-  caelo-minio:
   caelo-caddy-data:
   caelo-caddy-config:
 `;
@@ -278,7 +352,7 @@ ${serviceRuntime(
   // `.caelo/bootstrap/pending-token.json`; the admin reads (and deletes) it
   // from `/app/.caelo/` on first boot (hooks.server.ts). Only that dir is
   // mounted — never `.caelo/` itself, which holds config.json's superuser
-  // and MinIO passwords. Without the token /setup accepts any first Owner.
+  // password. Without the token /setup accepts any first Owner.
   // The static site: the admin builds into it (CAELO_OUTPUT_ROOT below),
   // Caddy serves it read-only from the same host dir. On the host, so a
   // recreated admin (upgrade) keeps the published site.
@@ -289,7 +363,6 @@ ${serviceRuntime(
     `./${MEDIA_DIR}:${MEDIA_ROOT}`,
   ],
 )}
-    container_name: caelo-admin
     restart: unless-stopped
     depends_on:
       postgres: { condition: service_healthy }
@@ -298,7 +371,7 @@ ${serviceRuntime(
       ADMIN_DATABASE_URL: "${dbUrl("admin_role", spec.rolePasswords.admin, "cms_admin")}"
       PUBLIC_ADMIN_DATABASE_URL: "${dbUrl("admin_role", spec.rolePasswords.admin, "cms_public")}"
       CAELO_SECRET_KEK: "${escapeYaml(spec.caeloSecretKek)}"
-      CAELO_SITE_URL: "${escapeYaml(publicSiteUrl(spec.domain))}"
+      CAELO_SITE_URL: "${escapeYaml(selfHostedSiteUrl(spec.domain, spec.ports.https))}"
       PORT: "${COMPOSE_ADMIN_PORT}"
       CAELO_OUTPUT_ROOT: "${SITE_OUTPUT_ROOT}"
       MEDIA_ROOT_DIR: "${MEDIA_ROOT}"
@@ -311,7 +384,6 @@ ${env("RESEND_API_KEY", spec.resendApiKey)}`;
 function gatewayService(spec: ComposeSpec): string {
   return `  caelo-gateway:
 ${serviceRuntime(spec.images, "gateway", "bun run apps/api-gateway/src/server.ts")}
-    container_name: caelo-gateway
     restart: unless-stopped
     depends_on:
       postgres: { condition: service_healthy }

package/src/gcp-names.ts

@@ -133,3 +133,26 @@ export function upgraderServiceAccountId(env: string): string {
 export function upgraderServiceAccountEmail(projectId: string, env: string): string {
   return `${upgraderServiceAccountId(env)}@${projectId}.iam.gserviceaccount.com`;
 }
+
+/**
+ * #587 — id of the project custom role that lets the admin write the
+ * redirect rules into the `gcp` stack's URL map (url-map-redirects.ts).
+ * Role ids allow letters, digits, `_` and `.`.
+ */
+export function urlMapRedirectsRoleId(env: string): string {
+  return `caelo_${env.replace(/[^A-Za-z0-9]/g, "_")}_urlMapRedirects`;
+}
+
+/**
+ * #587 — permissions of {@link urlMapRedirectsRoleId}: read-modify-write of
+ * the URL map and waiting for the operation. A URL-map update must be
+ * allowed to "use" every backend the map references, hence the `.use` pair.
+ */
+export const URL_MAP_REDIRECTS_PERMISSIONS: readonly string[] = [
+  "compute.urlMaps.list",
+  "compute.urlMaps.get",
+  "compute.urlMaps.update",
+  "compute.backendBuckets.use",
+  "compute.backendServices.use",
+  "compute.globalOperations.get",
+];

package/src/gcp-url-map.test.ts

@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * #587 — the `gcp` stack's URL map under Pulumi mocks: host routing, the
+ * public path matcher the admin writes redirect rules into, the field the
+ * stack ignores after creation; the admin's custom role is the CLI's.
+ */
+
+import { beforeAll, describe, expect, it } from "bun:test";
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { GCP_PUBLIC_BASE_PATHS, isGcpRedirectRule, withGcpRedirects } from "@caelo-cms/shared";
+import * as pulumi from "@pulumi/pulumi";
+import { URL_MAP_REDIRECTS_PERMISSIONS, urlMapRedirectsRoleId } from "./gcp-names.js";
+
+interface Registered {
+  readonly type: string;
+  readonly name: string;
+  readonly inputs: Record<string, unknown>;
+}
+const registered: Registered[] = [];
+
+beforeAll(() => {
+  pulumi.runtime.setMocks(
+    {
+      newResource: (args) => {
+        registered.push({ type: args.type, name: args.name, inputs: args.inputs });
+        return { id: `${args.name}-id`, state: args.inputs };
+      },
+      call: (args) => args.inputs,
+    },
+    "caelo",
+    "production",
+    false,
+  );
+});
+
+const resolved = <T>(o: pulumi.Output<T>): Promise<T> => new Promise((r) => o.apply(r));
+
+describe("gcp stack URL map (Pulumi mocks)", () => {
+  it("routes the site host to the bucket with /api/* to the gateway, the admin host to the admin", async () => {
+    // Imported after setMocks so the resource registers against the mock monitor.
+    const { createSiteUrlMap, PUBLIC_PATH_MATCHER } = await import("./gcp-url-map.js");
+    const map = createSiteUrlMap(
+      "caelo-production-url-map",
+      {
+        domain: "example.com",
+        adminDomain: "admin.example.com",
+        staticBackend: "bucket-id",
+        gatewayBackend: "gateway-id",
+        adminBackend: "admin-id",
+      },
+      {},
+    );
+    const [hostRules, pathMatchers, defaultService] = await Promise.all([
+      resolved(map.hostRules),
+      resolved(map.pathMatchers),
+      resolved(map.defaultService),
+    ]);
+    expect(defaultService).toBe("bucket-id");
+    expect(hostRules).toEqual([
+      { hosts: ["example.com"], pathMatcher: PUBLIC_PATH_MATCHER },
+      { hosts: ["admin.example.com"], pathMatcher: "admin" },
+    ]);
+    expect(pathMatchers).toEqual([
+      {
+        name: PUBLIC_PATH_MATCHER,
+        defaultService: "bucket-id",
+        pathRules: [{ paths: [...GCP_PUBLIC_BASE_PATHS], service: "gateway-id" }],
+      },
+      { name: "admin", defaultService: "admin-id" },
+    ]);
+    expect(registered.find((r) => r.name === "caelo-production-url-map")?.type).toBe(
+      "gcp:compute/uRLMap:URLMap",
+    );
+
+    // The rules the admin writes on Confirm-publish fit this matcher and keep
+    // its routing — the base the Stage-time budget check assumes.
+    const first = pathMatchers?.[0];
+    if (!first) throw new Error("no public path matcher");
+    const withRedirects = withGcpRedirects(
+      {
+        name: first.name,
+        pathRules: (first.pathRules ?? []).map((r) => ({
+          paths: r.paths ?? [],
+          service: r.service ?? "",
+        })),
+      },
+      {
+        redirects: [
+          {
+            fromPath: "/old",
+            matchPaths: ["/old", "/old/"],
+            location: "/new/",
+            statusCode: 301,
+            preserveQuery: true,
+          },
+        ],
+        gone: [],
+        shadowedByBuild: [],
+      },
+    );
+    expect(withRedirects.pathRules?.filter((r) => !isGcpRedirectRule(r))).toEqual([
+      { paths: ["/api/*"], service: "gateway-id" },
+    ]);
+  });
+
+  it("ignores the public path matcher's path rules after creation (the admin owns the redirects)", async () => {
+    const { SITE_URL_MAP_IGNORE_CHANGES, siteUrlMapArgs, PUBLIC_PATH_MATCHER } = await import(
+      "./gcp-url-map.js"
+    );
+    expect(SITE_URL_MAP_IGNORE_CHANGES).toEqual(["pathMatchers[0].pathRules"]);
+    const args = siteUrlMapArgs({
+      domain: "d",
+      adminDomain: "a.d",
+      staticBackend: "s",
+      gatewayBackend: "g",
+      adminBackend: "a",
+    });
+    // Index 0 must stay the public matcher, or the ignore would hit the admin's.
+    expect((args.pathMatchers as { name: string }[])[0]?.name).toBe(PUBLIC_PATH_MATCHER);
+  });
+});
+
+describe("gcp stack wiring (source)", () => {
+  const src = readFileSync(resolve(import.meta.dir, "../stacks/gcp/index.ts"), "utf8");
+
+  it("builds the URL map through createSiteUrlMap", () => {
+    expect(src).toMatch(/const urlMap = createSiteUrlMap\(\s*`\$\{namePrefix\}-url-map`/);
+    expect(src).not.toMatch(/new gcp\.compute\.URLMap\(\s*`\$\{namePrefix\}-url-map`/);
+  });
+
+  it("leaves the admin's URL-map redirects role to the CLI (installer + upgrade)", () => {
+    expect(src).not.toContain("IAMCustomRole");
+    expect(src).not.toContain("urlMapRedirects");
+  });
+
+  it("names a valid custom role id with only the permissions the writer needs", () => {
+    expect(urlMapRedirectsRoleId("production")).toBe("caelo_production_urlMapRedirects");
+    expect(urlMapRedirectsRoleId("production")).toMatch(/^[A-Za-z0-9_.]{3,64}$/);
+    expect([...URL_MAP_REDIRECTS_PERMISSIONS].sort()).toEqual([
+      "compute.backendBuckets.use",
+      "compute.backendServices.use",
+      "compute.globalOperations.get",
+      "compute.urlMaps.get",
+      "compute.urlMaps.list",
+      "compute.urlMaps.update",
+    ]);
+  });
+});

package/src/gcp-url-map.ts

@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * The `gcp` stack's URL map, and who may write its redirect rules (#587).
+ *
+ * The admin writes the redirects table into the site host's path matcher
+ * as `urlRedirect` path rules on every Confirm-publish (admin-core
+ * deploy/gcp-url-map-redirects.ts). Those rules change at runtime, not with
+ * `pulumi up`, so the stack must not own them: the URL map ignores changes
+ * to the public path matcher's `pathRules` after creation. Otherwise a
+ * `pulumi refresh` + `up` would wipe every redirect until the next publish.
+ * The admin's writer keeps every non-redirect rule (`/api/*`) as it finds
+ * it.
+ *
+ * The admin's run SA gets a project custom role with only the permissions
+ * that read-modify-write needs (`URL_MAP_REDIRECTS_PERMISSIONS`, gcp-names.ts);
+ * the CLI creates it (url-map-redirects-access.ts), not the stack.
+ *
+ * Imported by the stack from `dist/`; keep it free of CLI dependencies.
+ */
+
+import { GCP_PUBLIC_BASE_PATHS } from "@caelo-cms/shared";
+import * as gcp from "@pulumi/gcp";
+import type * as pulumi from "@pulumi/pulumi";
+
+/** Name of the site host's path matcher. */
+export const PUBLIC_PATH_MATCHER = "public";
+
+/**
+ * The URL map's `ignoreChanges`: the public path matcher (index 0, see
+ * {@link siteUrlMapArgs}) holds the admin-managed redirect rules.
+ */
+export const SITE_URL_MAP_IGNORE_CHANGES: readonly string[] = ["pathMatchers[0].pathRules"];
+
+/** Inputs of {@link siteUrlMapArgs}. */
+export interface SiteUrlMapInputs {
+  readonly domain: string;
+  readonly adminDomain: string;
+  readonly staticBackend: pulumi.Input<string>;
+  readonly gatewayBackend: pulumi.Input<string>;
+  readonly adminBackend: pulumi.Input<string>;
+}
+
+/**
+ * The URL map's arguments: the site host to the static bucket with `/api/*`
+ * to the gateway; the admin host to the IAP-gated admin. The public path
+ * matcher comes first — {@link SITE_URL_MAP_IGNORE_CHANGES} addresses it by
+ * index.
+ */
+export function siteUrlMapArgs(i: SiteUrlMapInputs): gcp.compute.URLMapArgs {
+  return {
+    defaultService: i.staticBackend,
+    hostRules: [
+      { hosts: [i.domain], pathMatcher: PUBLIC_PATH_MATCHER },
+      { hosts: [i.adminDomain], pathMatcher: "admin" },
+    ],
+    pathMatchers: [
+      {
+        name: PUBLIC_PATH_MATCHER,
+        defaultService: i.staticBackend,
+        pathRules: [{ paths: [...GCP_PUBLIC_BASE_PATHS], service: i.gatewayBackend }],
+      },
+      {
+        name: "admin",
+        defaultService: i.adminBackend,
+      },
+    ],
+  };
+}
+
+/** The stack's URL map (see the module comment for the ignored field). */
+export function createSiteUrlMap(
+  name: string,
+  inputs: SiteUrlMapInputs,
+  opts: pulumi.CustomResourceOptions,
+): gcp.compute.URLMap {
+  return new gcp.compute.URLMap(name, siteUrlMapArgs(inputs), {
+    ...opts,
+    ignoreChanges: [...(opts.ignoreChanges ?? []), ...SITE_URL_MAP_IGNORE_CHANGES],
+  });
+}