@caelo-cms/provisioning 0.11.5 → 0.11.7

package/stacks/aws/Pulumi.yaml

@@ -28,6 +28,18 @@ config:
     type: string
     default: db.t4g.small
     description: RDS instance class. db.t4g.small is the cheapest Multi-AZ-capable Graviton class.
+  caelo-aws:maxConnections:
+    type: string
+    default: "100"
+    description: RDS max_connections (parameter group); every service's pool size is computed from it (docs/dev/connection-budget.md).
+  caelo-aws:adminMaxInstances:
+    type: string
+    default: "5"
+    description: Most admin tasks at once; part of the connection budget.
+  caelo-aws:gatewayMaxInstances:
+    type: string
+    default: "10"
+    description: Most gateway tasks at once; part of the connection budget (2 connections each).
   caelo-aws:fargateCpu:
     type: string
     default: "512"

package/stacks/aws/README.md

@@ -7,6 +7,7 @@ Pulumi stack provisioning Caelo on AWS managed services. Implements the shared `
 | Concern | AWS resource |
 |---|---|
 | Managed Postgres | RDS Postgres 16 Multi-AZ + automated backups + encryption (`db.t4g.small` default) |
+| Connection budget | `max_connections` pinned by a parameter group (`maxConnections`, default 100); the pool sizes for `adminMaxInstances` 5 / `gatewayMaxInstances` 10 are exported as `poolBudgetEnvOut` for the task definitions (#749): cms_admin 7, admin cms_public 4, gateway 2 per task (docs/dev/connection-budget.md) |
 | Blob storage | Two S3 buckets (`caelo-<env>-media` + `caelo-<env>-static`) |
 | CDN | CloudFront distribution with two origins (S3 static + S3 media); static files are served as files, no edge compute on the request path |
 | Container runtime | ECS Fargate cluster + four services (admin / gateway / orchestrator / runner) |

package/stacks/aws/index.ts

@@ -39,6 +39,11 @@ import {
   secretsReadPolicy,
 } from "../../dist/aws-task-roles.js";
 import { generateBootstrapToken } from "../../dist/bootstrap-token.js";
+import {
+  DEFAULT_GATEWAY_MAX_INSTANCES,
+  DEFAULT_MAX_CONNECTIONS,
+  stackPoolBudget,
+} from "../../dist/pool-budget-stack.js";
 import { ADMIN_MEMORY_DEFAULT, memoryQuantityMiB } from "../../dist/stack-contract.js";
 import {
   CLOUDFRONT_SVG_BEHAVIORS,
@@ -68,6 +73,25 @@ const _fargateMemoryMb = cfg.get("fargateMemoryMb") ?? "1024";
 // ignored, and the admin task definition sizes itself from it.
 const adminMemory = cfg.get("adminMemory") ?? ADMIN_MEMORY_DEFAULT;
 const _adminMemoryMiB = memoryQuantityMiB(adminMemory);
+// #1254 — the connection budget. RDS derives max_connections from instance
+// memory; the parameter group below pins it, so the pool sizes computed
+// here hold on every instance class. The task definitions (#749) set
+// `poolBudgetEnvOut` on the admin and gateway tasks and scale to the same
+// instance counts.
+const maxConnections = Number.parseInt(
+  cfg.get("maxConnections") ?? String(DEFAULT_MAX_CONNECTIONS.aws),
+  10,
+);
+const adminMaxInstances = Number.parseInt(cfg.get("adminMaxInstances") ?? "5", 10);
+const gatewayMaxInstances = Number.parseInt(
+  cfg.get("gatewayMaxInstances") ?? String(DEFAULT_GATEWAY_MAX_INSTANCES),
+  10,
+);
+const poolBudget = stackPoolBudget("aws", {
+  maxConnections,
+  adminMaxInstances,
+  gatewayMaxInstances,
+});
 if (_adminMemoryMiB === null) {
   throw new Error(`caelo-aws:adminMemory "${adminMemory}" is not a memory quantity (e.g. 2Gi)`);
 }
@@ -141,10 +165,20 @@ const dbSecurityGroup = new aws.ec2.SecurityGroup(`${namePrefix}-db-sg`, {
   egress: [{ protocol: "-1", fromPort: 0, toPort: 0, cidrBlocks: ["0.0.0.0/0"] }],
 });
 
+// #1254 — max_connections is the input of the pool budget, so it is set,
+// never left to the instance class's memory-derived default.
+const dbParameters = new aws.rds.ParameterGroup(`${namePrefix}-pg-params`, {
+  family: "postgres16",
+  parameters: [
+    { name: "max_connections", value: String(maxConnections), applyMethod: "pending-reboot" },
+  ],
+});
+
 const db = new aws.rds.Instance(`${namePrefix}-pg`, {
   engine: "postgres",
   engineVersion: "16.4",
   instanceClass: rdsInstanceClass,
+  parameterGroupName: dbParameters.name,
   allocatedStorage: 20,
   storageEncrypted: true,
   multiAz: env === "production",
@@ -529,6 +563,8 @@ export const rdsAddressOut = db.address;
 // The task roles the admin-side and gateway task definitions run as.
 export const adminTaskRoleArnOut = taskRoles.admin.arn;
 export const gatewayTaskRoleArnOut = taskRoles.gateway.arn;
+// #1254 — the pool env the admin and gateway task definitions set.
+export const poolBudgetEnvOut = poolBudget.env;
 
 // Note: the bootstrap token (and other secrets like postgres-password,
 // CSRF/cookie secrets, Anthropic key) are stored in Pulumi's encrypted

package/stacks/azure/Pulumi.yaml

@@ -38,3 +38,7 @@ config:
     type: string
     default: Standard_B2s
     description: Postgres flexible-server SKU. B2s = burstable; bump to GP_Standard_D2s_v3 for production.
+  caelo-azure:maxConnections:
+    type: string
+    default: "429"
+    description: Postgres max_connections (server parameter; must not exceed the SKU's limit, 429 on B2s). Every app's pool size is computed from it (docs/dev/connection-budget.md).

package/stacks/azure/README.md

@@ -7,6 +7,7 @@ Pulumi stack provisioning Caelo on Azure managed services. Implements the shared
 | Concern | Azure resource |
 |---|---|
 | Managed Postgres | Azure DB for PostgreSQL flexible server (zone-redundant in production) + automated backups (geo-redundant in production) |
+| Connection budget | `max_connections` pinned as a server parameter (`maxConnections`, default 429, the Standard_B2s limit); the admin, orchestrator and runner apps (10 replicas each) get a cms_admin pool of 7 and a cms_public pool of 4, the gateway a pool of 2 (docs/dev/connection-budget.md) |
 | Blob storage | Storage Account with a private `media` container and static-website hosting (`$web`) |
 | CDN / Edge | Azure Front Door Standard: the primary domain serves the static website as files, with a Front Door managed certificate; no edge compute on the request path |
 | Container runtime | Four Container Apps (admin / gateway / orchestrator / runner) in one managed environment |

package/stacks/azure/index.ts

@@ -39,6 +39,7 @@ import * as pulumi from "@pulumi/pulumi";
 import type { CloudAdapterOutputs, DnsRecord } from "../../dist/adapter.js";
 import { azureFrontDoorDnsRecords } from "../../dist/azure-front-door.js";
 import { generateBootstrapToken } from "../../dist/bootstrap-token.js";
+import { DEFAULT_MAX_CONNECTIONS, stackPoolBudget } from "../../dist/pool-budget-stack.js";
 import {
   ADMIN_MEMORY_DEFAULT,
   azureContainerAppResources,
@@ -67,6 +68,22 @@ if (legacyLocation !== undefined) {
 }
 const location = cfg.require("region");
 const flexibleServerSku = cfg.get("flexibleServerSku") ?? "Standard_B2s";
+// #1254 — the server's max_connections, set explicitly below (the default
+// Standard_B2s allows up to 429); every app's pool size is computed from it.
+const maxConnections = Number.parseInt(
+  cfg.get("maxConnections") ?? String(DEFAULT_MAX_CONNECTIONS.azure),
+  10,
+);
+/** Replicas each Container App may scale to (see containerApp). */
+const MAX_REPLICAS = 10;
+// The admin, orchestrator and runner apps all open the admin's two pools,
+// so the budget counts their replicas together as admin instances.
+const ADMIN_SIDE_APPS = 3;
+const poolBudget = stackPoolBudget("azure", {
+  maxConnections,
+  adminMaxInstances: ADMIN_SIDE_APPS * MAX_REPLICAS,
+  gatewayMaxInstances: MAX_REPLICAS,
+});
 // #553 — the admin runs the Lighthouse quality audit; same knob + default
 // as every adapter (stack-contract.ts). Container Apps only accepts fixed
 // CPU/memory pairs (2 GiB per vCPU), so the admin's CPU follows its memory
@@ -149,6 +166,18 @@ const pgServer = new azure.dbforpostgresql.Server(`${namePrefix}-pg`, {
   network: { publicNetworkAccess: "Disabled" },
 });
 
+// #1254 — pinned so the pool budget never depends on the SKU's default.
+const _pgMaxConnections = new azure.dbforpostgresql.Configuration(
+  `${namePrefix}-pg-max-connections`,
+  {
+    resourceGroupName: rg.name,
+    serverName: pgServer.name,
+    configurationName: "max_connections",
+    value: String(maxConnections),
+    source: "user-override",
+  },
+);
+
 const _pgAdmin = new azure.dbforpostgresql.Database(`${namePrefix}-cms-admin-db`, {
   resourceGroupName: rg.name,
   serverName: pgServer.name,
@@ -425,7 +454,7 @@ function containerApp(args: ContainerAppArgs): azure.app.ContainerApp {
       },
     },
     template: {
-      scale: { minReplicas: env === "production" ? 1 : 0, maxReplicas: 10 },
+      scale: { minReplicas: env === "production" ? 1 : 0, maxReplicas: MAX_REPLICAS },
       containers: [
         {
           name: args.serviceName,
@@ -453,6 +482,7 @@ function containerApp(args: ContainerAppArgs): azure.app.ContainerApp {
 const adminDatabaseEnv = [
   { name: "ADMIN_DATABASE_URL", value: adminDatabaseUrl },
   { name: "PUBLIC_ADMIN_DATABASE_URL", value: publicAdminDatabaseUrl },
+  ...poolBudget.env.admin,
 ];
 const adminApp = containerApp({
   serviceName: "admin",
@@ -464,7 +494,10 @@ const adminApp = containerApp({
 });
 const _gatewayApp = containerApp({
   serviceName: "gateway",
-  databaseEnv: [{ name: "PUBLIC_DATABASE_URL", value: publicDatabaseUrl }],
+  databaseEnv: [
+    { name: "PUBLIC_DATABASE_URL", value: publicDatabaseUrl },
+    ...poolBudget.env.gateway,
+  ],
   // Which X-Forwarded-For entry is the visitor (api-gateway client-ip.ts).
   // #806 — no remote-MCP env: remote MCP rides Google IAP (gcp stacks
   // only); unset, the gateway answers /api/mcp/* with 503

package/stacks/gcp/Pulumi.yaml

@@ -57,22 +57,30 @@ config:
     type: string
     default: "5"
     description: |
-      Admin Cloud Run max instances. Default 5 caps the connection-pool
-      footprint at ~100 Postgres conns (Bun SQL pool max=10, two pools per
-      admin instance). Bump alongside maxConnections + cloudSqlTier when
-      sizing for heavy editorial traffic.
+      Admin Cloud Run max instances. Each instance gets an equal share of
+      the connection budget (docs/dev/connection-budget.md): at the
+      defaults, a cms_admin pool of 7. Bump alongside maxConnections +
+      cloudSqlTier when sizing for heavy editorial traffic; a scale-out the
+      server cannot afford fails at preview with the max_connections it needs.
   caelo-gcp:maxConnections:
     type: string
     default: "100"
     description: |
-      Postgres max_connections database flag. Default 100 matches the safe
-      ceiling for adminMaxInstances=5 + light gateway traffic. Cloud SQL's
-      db-f1-micro defaults to 25 — too low for any sustained admin load.
-      Tune up alongside cloudSqlTier when scaling out.
+      Postgres max_connections database flag; every service's pool size is
+      computed from it (#1254). Cloud SQL's db-f1-micro defaults to 25 —
+      too low for any sustained admin load. Tune up alongside cloudSqlTier
+      when scaling out.
   caelo-gcp:gatewayMinInstances:
     type: string
     default: "0"
     description: Gateway Cloud Run min instances. 0 = scale to zero. 1 = no cold start (+~$15/mo).
+  caelo-gcp:gatewayMaxInstances:
+    type: string
+    default: "10"
+    description: |
+      Gateway Cloud Run max instances. Part of the connection budget (2
+      connections each); 10 serve ~800 concurrent visitor requests. Raise
+      together with maxConnections.
   caelo-gcp:iapAllowlist:
     type: string
     default: ""

package/stacks/gcp/README.md

@@ -26,11 +26,15 @@ pulumi config set caelo-gcp:project my-gcp-project
 # Required (#607): no default, fixed after install.
 pulumi config set caelo-gcp:region europe-west1
 
-# Optional: scale up beyond the safe-by-default ceiling. The defaults
-# size for a small editorial install (admin maxScale 5 × Bun SQL pool
-# max 10 × 2 pools = 100 conns, matching Postgres max_connections=100).
+# Optional: scale up beyond the safe-by-default ceiling. Every service's
+# pool size is computed from max_connections and the instance counts
+# (docs/dev/connection-budget.md): at the defaults (max_connections 100,
+# 5 admin, 10 gateway instances) each admin instance gets a cms_admin
+# pool of 7. A scale-out the server cannot afford fails at preview with
+# the max_connections it needs.
 # pulumi config set caelo-gcp:adminMaxInstances 10
 # pulumi config set caelo-gcp:maxConnections 200
+# pulumi config set caelo-gcp:gatewayMaxInstances 20
 
 # Build + push images. Operator does this once + on every release.
 gcloud auth configure-docker us-central1-docker.pkg.dev

package/stacks/gcp/index.ts

@@ -31,6 +31,11 @@ import {
   staticPublisherServiceAccountId,
 } from "../../dist/gcp-names.js";
 import { createSiteUrlMap } from "../../dist/gcp-url-map.js";
+import {
+  DEFAULT_GATEWAY_MAX_INSTANCES,
+  DEFAULT_MAX_CONNECTIONS,
+  stackPoolBudget,
+} from "../../dist/pool-budget-stack.js";
 import { mcpArmorRuleArgs } from "../../dist/remote-mcp-edge.js";
 import {
   ADMIN_MEMORY_DEFAULT,
@@ -96,13 +101,14 @@ const adminMinInstances = Number.parseInt(cfg.get("adminMinInstances") ?? "0", 1
 // #553 — the admin also runs the Lighthouse quality audit; same knob and
 // default on every adapter (stack-contract.ts ADMIN_MEMORY_DEFAULT).
 const adminMemory = cfg.get("adminMemory") ?? ADMIN_MEMORY_DEFAULT;
-// adminMaxInstances default is intentionally low. Each admin process holds
-// TWO Bun SQL pools (admin_role on cms_admin + admin_role on cms_public),
-// each with default pool-max=10 → up to ~20 connections per instance under
-// sustained load. With db-f1-micro's tunable max_connections, even 5 admin
-// instances saturate the pool (5 × 20 = 100 = our default ceiling below).
-// Operators on larger SQL tiers should bump maxConnections AND
-// adminMaxInstances together.
+// adminMaxInstances default is intentionally low. Every admin instance
+// holds two pools (cms_admin + cms_public) plus a LISTEN connection, and
+// all of them share the server's max_connections with the gateway, the
+// generator and the jobs. #1254 sizes the pools from that budget
+// (src/pool-budget-stack.ts) instead of Bun's default 10 per pool; a
+// scale-out the server cannot afford fails at preview, naming the
+// connections it needs. Operators on larger SQL tiers bump maxConnections
+// AND adminMaxInstances together.
 //
 // Why 5, not 10 (Cloud Run's default)? An incident on production showed
 // admin scaling to 10 under bursty editor traffic, demanding 200 slots
@@ -111,11 +117,23 @@ const adminMemory = cfg.get("adminMemory") ?? ADMIN_MEMORY_DEFAULT;
 // installs don't trip the same wall.
 const adminMaxInstances = Number.parseInt(cfg.get("adminMaxInstances") ?? "5", 10);
 const gatewayMinInstances = Number.parseInt(cfg.get("gatewayMinInstances") ?? "0", 10);
-// Postgres connection slot ceiling. Cloud SQL's db-f1-micro defaults to 25,
-// which 1 admin pool exhausts under load (Bun SQL pool-max=10, two pools per
-// admin instance, plus gateway + migrations + backup-agent overhead). 100 is
-// the safe floor for a small editorial install — tune up alongside SQL tier.
-const maxConnections = Number.parseInt(cfg.get("maxConnections") ?? "100", 10);
+// #1254 — the gateway's scale-out is part of the connection budget: each
+// instance holds a cms_public pool.
+const gatewayMaxInstances = Number.parseInt(
+  cfg.get("gatewayMaxInstances") ?? String(DEFAULT_GATEWAY_MAX_INSTANCES),
+  10,
+);
+// Postgres connection slot ceiling, set as a database flag below (Cloud
+// SQL's db-f1-micro defaults to 25). The pool budget is computed from it.
+const maxConnections = Number.parseInt(
+  cfg.get("maxConnections") ?? String(DEFAULT_MAX_CONNECTIONS.gcp),
+  10,
+);
+const poolBudget = stackPoolBudget("gcp", {
+  maxConnections,
+  adminMaxInstances,
+  gatewayMaxInstances,
+});
 const wafAdaptiveProtection = cfg.getBoolean("wafAdaptiveProtection") ?? false;
 const backupRetentionDays = Number.parseInt(cfg.get("backupRetentionDays") ?? "7", 10);
 // IAP allowlist: comma-list of "user:<email>" or "group:<group@domain>".
@@ -292,11 +310,9 @@ const sqlInstance = new gcp.sql.DatabaseInstance(
       diskSize: 20,
       diskType: "PD_SSD",
       diskAutoresize: true,
-      // db-f1-micro defaults max_connections=25, which a single
-      // admin Cloud Run instance can exhaust under load (Bun SQL
-      // pool-max=10, two pools per admin process). Override at
-      // instance level so the cap matches the Cloud Run scale-out
-      // headroom defined by adminMaxInstances + gatewayMaxInstances.
+      // db-f1-micro defaults max_connections=25. Set explicitly: the
+      // services' pool sizes are computed from this exact number
+      // (poolBudget above), so the server must have it.
       databaseFlags: [
         { name: "max_connections", value: maxConnections.toString() },
         // The operator-access sync job logs in as its service account (Cloud
@@ -787,13 +803,18 @@ const adminSvc = cloudRunService({
   // in Cloud Run logs. 3600s is the platform max; no cost penalty for
   // unused timeout (Cloud Run bills running time, not allocated time).
   timeout: "3600s",
-  contractEnv: [...adminEnvContract(envContractInputs), ...remoteMcpEnv.admin],
+  contractEnv: [
+    ...adminEnvContract(envContractInputs),
+    ...remoteMcpEnv.admin,
+    // #1254 — pool sizes from the connection budget.
+    ...poolBudget.env.admin,
+  ],
   mediaVolume: adminMediaVolumeTemplate(adminMediaVolume(project, env)),
 });
 const gatewaySvc = cloudRunService({
   serviceName: "gateway",
   minInstances: gatewayMinInstances,
-  maxInstances: 100,
+  maxInstances: gatewayMaxInstances,
   memory: "512Mi",
   // #806 — remote MCP: chat turns stream through the gateway for minutes,
   // so it gets the admin's 3600 s instead of Cloud Run's 300 s default
@@ -802,7 +823,11 @@ const gatewaySvc = cloudRunService({
   timeout: `${GATEWAY_TIMEOUT_SECONDS}s`,
   // public_role on cms_public only: no cms_admin login of any role
   // (#613, #635, CLAUDE.md §2).
-  contractEnv: [...gatewayEnvContract(envContractInputs), ...remoteMcpEnv.gateway],
+  contractEnv: [
+    ...gatewayEnvContract(envContractInputs),
+    ...remoteMcpEnv.gateway,
+    ...poolBudget.env.gateway,
+  ],
 });
 
 // =========================================================================

package/stacks/gcp-firebase/Pulumi.yaml

@@ -63,11 +63,15 @@ config:
   caelo-gcp-firebase:maxConnections:
     type: string
     default: "100"
-    description: Postgres max_connections.
+    description: Postgres max_connections; every service's pool size is computed from it (docs/dev/connection-budget.md).
   caelo-gcp-firebase:gatewayMinInstances:
     type: string
     default: "0"
     description: Gateway Cloud Run min instances.
+  caelo-gcp-firebase:gatewayMaxInstances:
+    type: string
+    default: "10"
+    description: Gateway Cloud Run max instances; part of the connection budget (2 connections each).
   caelo-gcp-firebase:iapAllowlist:
     type: string
     default: ""

package/stacks/gcp-firebase/index.ts

@@ -37,6 +37,11 @@ import {
   MCP_IAP_SERVICE_ACCOUNT_ID,
   runServiceAccountId,
 } from "../../dist/gcp-names.js";
+import {
+  DEFAULT_GATEWAY_MAX_INSTANCES,
+  DEFAULT_MAX_CONNECTIONS,
+  stackPoolBudget,
+} from "../../dist/pool-budget-stack.js";
 import {
   ADMIN_MEMORY_DEFAULT,
   adminEnvContract,
@@ -93,7 +98,21 @@ const adminMinInstances = Number.parseInt(cfg.get("adminMinInstances") ?? "0", 1
 const adminMemory = cfg.get("adminMemory") ?? ADMIN_MEMORY_DEFAULT;
 const adminMaxInstances = Number.parseInt(cfg.get("adminMaxInstances") ?? "5", 10);
 const gatewayMinInstances = Number.parseInt(cfg.get("gatewayMinInstances") ?? "0", 10);
-const maxConnections = Number.parseInt(cfg.get("maxConnections") ?? "100", 10);
+// #1254 — part of the connection budget (stacks/gcp has the long comment).
+const gatewayMaxInstances = Number.parseInt(
+  cfg.get("gatewayMaxInstances") ?? String(DEFAULT_GATEWAY_MAX_INSTANCES),
+  10,
+);
+const maxConnections = Number.parseInt(
+  cfg.get("maxConnections") ?? String(DEFAULT_MAX_CONNECTIONS["gcp-firebase"]),
+  10,
+);
+// #1254 — every service's pool sizes, from the server's max_connections.
+const poolBudget = stackPoolBudget("gcp-firebase", {
+  maxConnections,
+  adminMaxInstances,
+  gatewayMaxInstances,
+});
 const backupRetentionDays = Number.parseInt(cfg.get("backupRetentionDays") ?? "7", 10);
 const iapAllowlistRaw = cfg.get("iapAllowlist");
 const iapAllowlist =
@@ -655,7 +674,7 @@ const envContractInputs = {
 const gatewaySvc = cloudRunService({
   serviceName: "gateway",
   minInstances: gatewayMinInstances,
-  maxInstances: 100,
+  maxInstances: gatewayMaxInstances,
   memory: "512Mi",
   // #806 — remote MCP: chat turns stream through the gateway for minutes
   // (stack-contract.ts GATEWAY_TIMEOUT_SECONDS). The Firebase Hosting
@@ -670,7 +689,11 @@ const gatewaySvc = cloudRunService({
   // request passes its rate limits, captcha and honeypot; staging requests
   // pass the staging access gate.
   ingress: "INGRESS_TRAFFIC_ALL",
-  contractEnv: [...gatewayEnvContract(envContractInputs), ...remoteMcpEnv.gateway],
+  contractEnv: [
+    ...gatewayEnvContract(envContractInputs),
+    ...remoteMcpEnv.gateway,
+    ...poolBudget.env.gateway,
+  ],
 });
 
 const adminSvc = cloudRunService({
@@ -693,6 +716,7 @@ const adminSvc = cloudRunService({
       gatewayService: gatewaySvc.name,
     }),
     ...remoteMcpEnv.admin,
+    ...poolBudget.env.admin,
   ],
   mediaVolume: adminMediaVolumeTemplate(adminMediaVolume(project, env)),
 });

package/stacks/self-hosted/README.md

@@ -32,6 +32,10 @@ pulumi stack output bootstrapUrl
 
 Open that URL once Caddy has issued the cert (~30 s).
 
+## Connection budget
+
+The generated Compose file starts Postgres with `max_connections=100` and sizes every pool from it (#1254, docs/dev/connection-budget.md): the admin gets a cms_admin pool of 77 and a cms_public pool of 4, the gateway a pool of 2, and static builds 2 per pool.
+
 ## Destroy
 
 ```bash