@caelo-cms/provisioning 0.11.10 → 0.11.12

package/stacks/gcp/index.ts

@@ -33,8 +33,10 @@ import {
 } from "../../dist/gcp-names.js";
 import { createSiteUrlMap } from "../../dist/gcp-url-map.js";
 import {
+  DEFAULT_ADMIN_MAX_INSTANCES,
   DEFAULT_GATEWAY_MAX_INSTANCES,
   DEFAULT_MAX_CONNECTIONS,
+  DEFAULT_STAGING_GATEWAY_MAX_INSTANCES,
   stackPoolBudget,
 } from "../../dist/pool-budget-stack.js";
 import {
@@ -119,7 +121,10 @@ const adminMemory = cfg.get("adminMemory") ?? ADMIN_MEMORY_DEFAULT;
 // against the original 25-slot Postgres default — connection exhaustion
 // took the admin offline. v0.2.49 bakes the post-incident shape so fresh
 // installs don't trip the same wall.
-const adminMaxInstances = Number.parseInt(cfg.get("adminMaxInstances") ?? "5", 10);
+const adminMaxInstances = Number.parseInt(
+  cfg.get("adminMaxInstances") ?? String(DEFAULT_ADMIN_MAX_INSTANCES),
+  10,
+);
 const gatewayMinInstances = Number.parseInt(cfg.get("gatewayMinInstances") ?? "0", 10);
 // #1254 — the gateway's scale-out is part of the connection budget: each
 // instance holds a cms_public pool.
@@ -137,6 +142,8 @@ const poolBudget = stackPoolBudget("gcp", {
   maxConnections,
   adminMaxInstances,
   gatewayMaxInstances,
+  // #1329 — the staging gateway (P3b-2) is budgeted from this release on.
+  stagingGatewayMaxInstances: DEFAULT_STAGING_GATEWAY_MAX_INSTANCES,
 });
 const wafAdaptiveProtection = cfg.getBoolean("wafAdaptiveProtection") ?? false;
 const backupRetentionDays = Number.parseInt(cfg.get("backupRetentionDays") ?? "7", 10);
@@ -281,6 +288,8 @@ const runtimeSecretIds: Record<RuntimeSecret, pulumi.Input<string>> = {
   "tool-approval-secret": gcpSecretId(env, "tool-approval-secret"),
   "staging-gate-secret": gcpSecretId(env, "staging-gate-secret"),
   "public-role-password": gcpSecretId(env, "public-role-password"),
+  // #1329 — the staging gateway's login, never the live one's.
+  "public-staging-role-password": gcpSecretId(env, "public-staging-role-password"),
   // #914 / #806 — the Owner-set Google sign-in client (a placeholder until
   // then; IAP login, IAP programmatic access and the remote-MCP broker) and
   // the broker's seal key, both CLI-generated.
@@ -297,6 +306,14 @@ const publicRolePassword = pulumi.secret(
     { provider: gcpProvider },
   ).secretData,
 );
+// #1329 — public_staging_role's own password (CLI-generated like the one
+// above): the staging gateway's only login, on cms_public_staging only.
+const publicStagingRolePassword = pulumi.secret(
+  gcp.secretmanager.getSecretVersionOutput(
+    { project, secret: gcpSecretId(env, "public-staging-role-password") },
+    { provider: gcpProvider },
+  ).secretData,
+);
 
 // =========================================================================
 // Tier 4 — Cloud SQL Postgres (private IP only; HA + retention configurable)
@@ -371,6 +388,14 @@ const pgPublicUser = new gcp.sql.User(
   { instance: sqlInstance.name, name: "public_role", password: publicRolePassword },
   opts,
 );
+// #1329 — the staging gateway's login: an Admin API user like public_role,
+// so the same membership revoke (public-role-isolation.ts) and the same
+// audit apply; migration 0530 gives it CONNECT on cms_public_staging only.
+new gcp.sql.User(
+  `${namePrefix}-pg-public-staging`,
+  { instance: sqlInstance.name, name: "public_staging_role", password: publicStagingRolePassword },
+  opts,
+);
 
 const cmsAdminDb = new gcp.sql.Database(
   `${namePrefix}-cms-admin-db`,
@@ -382,6 +407,13 @@ const cmsPublicDb = new gcp.sql.Database(
   { instance: sqlInstance.name, name: "cms_public" },
   opts,
 );
+// #1329 — staging's own data: the twin of cms_public on the same instance
+// (no extra cost), reached by the staging gateway only.
+const cmsPublicStagingDb = new gcp.sql.Database(
+  `${namePrefix}-cms-public-staging-db`,
+  { instance: sqlInstance.name, name: "cms_public_staging" },
+  opts,
+);
 
 // #1338 — the instance's DNS name, resolved to its private IP by a private
 // zone in the VPC (Cloud SQL assigns the name, it does not resolve it).
@@ -767,6 +799,8 @@ const envContractInputs = {
     admin: dbUrls.admin,
     publicAdmin: dbUrls.publicAdmin,
     public: dbUrls.public,
+    publicStagingAdmin: dbUrls.publicStagingAdmin,
+    publicStaging: dbUrls.publicStaging,
   },
 } as const;
 
@@ -841,6 +875,7 @@ function cloudRunService(args: CloudRunArgs): gcp.cloudrunv2.Service {
         pgAdminUser,
         cmsAdminDb,
         cmsPublicDb,
+        cmsPublicStagingDb,
         // The env contract names the buckets as strings; keep them ordered
         // before the services that use them.
         mediaBucket,
@@ -1155,6 +1190,18 @@ new gcp.iap.WebBackendServiceIamMember(
 // (packages/provisioning/src/capture-job-setup.ts), for the same reason as
 // the audit job above.
 
+// =========================================================================
+// Scheduler tick — deliberately NOT here
+// =========================================================================
+//
+// #1140 — one Cloud Scheduler job starts a Cloud Workflows execution every
+// `schedulerIntervalMinutes` (1–15, default 5); the workflow signs an IAP
+// credential as the scheduler's own service account and POSTs the admin's
+// /api/internal/scheduler/tick, where the admin runs its periodic work and
+// plugin workers from its due-table. The CLI owns the account, its grants,
+// the workflow and the job (packages/provisioning/src/scheduler-setup.ts),
+// for the same reason as the audit job above.
+
 // =========================================================================
 // Operator access — deliberately NOT here
 // =========================================================================
@@ -1325,6 +1372,7 @@ const out: CloudAdapterOutputs = {
   // Password-less — the password lives only in Secret Manager.
   adminDatabaseUrl: dbUrls.admin as unknown as string,
   publicDatabaseUrl: dbUrls.public as unknown as string,
+  publicStagingDatabaseUrl: dbUrls.publicStaging as unknown as string,
   mediaStorageUrl: pulumi.interpolate`gs://${mediaBucket.name}` as unknown as string,
   mediaCdnBaseUrl: pulumi.interpolate`https://${domain}/media` as unknown as string,
   bootstrapUrl:
@@ -1338,6 +1386,7 @@ const out: CloudAdapterOutputs = {
 
 export const adminDatabaseUrlOut = out.adminDatabaseUrl;
 export const publicDatabaseUrlOut = out.publicDatabaseUrl;
+export const publicStagingDatabaseUrlOut = out.publicStagingDatabaseUrl;
 export const mediaStorageUrlOut = out.mediaStorageUrl;
 export const mediaCdnBaseUrlOut = out.mediaCdnBaseUrl;
 export const bootstrapUrlOut = out.bootstrapUrl;

package/stacks/gcp-firebase/Pulumi.yaml

@@ -58,7 +58,7 @@ config:
     description: Admin Cloud Run memory. 2Gi leaves room for the Lighthouse quality audit next to editor requests; billed only while an instance runs.
   caelo-gcp-firebase:adminMaxInstances:
     type: string
-    default: "5"
+    default: "4"
     description: Admin Cloud Run max instances.
   caelo-gcp-firebase:maxConnections:
     type: string

package/stacks/gcp-firebase/index.ts

@@ -41,8 +41,10 @@ import {
   runServiceAccountId,
 } from "../../dist/gcp-names.js";
 import {
+  DEFAULT_ADMIN_MAX_INSTANCES,
   DEFAULT_GATEWAY_MAX_INSTANCES,
   DEFAULT_MAX_CONNECTIONS,
+  DEFAULT_STAGING_GATEWAY_MAX_INSTANCES,
   stackPoolBudget,
 } from "../../dist/pool-budget-stack.js";
 import {
@@ -109,7 +111,10 @@ const adminMinInstances = Number.parseInt(cfg.get("adminMinInstances") ?? "0", 1
 // #553 — the admin also runs the Lighthouse quality audit; same knob and
 // default on every adapter (stack-contract.ts ADMIN_MEMORY_DEFAULT).
 const adminMemory = cfg.get("adminMemory") ?? ADMIN_MEMORY_DEFAULT;
-const adminMaxInstances = Number.parseInt(cfg.get("adminMaxInstances") ?? "5", 10);
+const adminMaxInstances = Number.parseInt(
+  cfg.get("adminMaxInstances") ?? String(DEFAULT_ADMIN_MAX_INSTANCES),
+  10,
+);
 const gatewayMinInstances = Number.parseInt(cfg.get("gatewayMinInstances") ?? "0", 10);
 // #1254 — part of the connection budget (stacks/gcp has the long comment).
 const gatewayMaxInstances = Number.parseInt(
@@ -125,6 +130,8 @@ const poolBudget = stackPoolBudget("gcp-firebase", {
   maxConnections,
   adminMaxInstances,
   gatewayMaxInstances,
+  // #1329 — the staging gateway (P3b-2) is budgeted from this release on.
+  stagingGatewayMaxInstances: DEFAULT_STAGING_GATEWAY_MAX_INSTANCES,
 });
 const backupRetentionDays = Number.parseInt(cfg.get("backupRetentionDays") ?? "7", 10);
 const iapAllowlistRaw = cfg.get("iapAllowlist");
@@ -262,6 +269,8 @@ const runtimeSecretIds: Record<RuntimeSecret, pulumi.Input<string>> = {
   "tool-approval-secret": gcpSecretId(env, "tool-approval-secret"),
   "staging-gate-secret": gcpSecretId(env, "staging-gate-secret"),
   "public-role-password": gcpSecretId(env, "public-role-password"),
+  // #1329 — the staging gateway's login, never the live one's.
+  "public-staging-role-password": gcpSecretId(env, "public-staging-role-password"),
   // #914 / #806 — the Owner-set Google sign-in client (a placeholder until
   // then; IAP login, IAP programmatic access and the remote-MCP broker) and
   // the broker's seal key, both CLI-generated.
@@ -278,6 +287,14 @@ const publicRolePassword = pulumi.secret(
     { provider: gcpProvider },
   ).secretData,
 );
+// #1329 — public_staging_role's own password (CLI-generated like the one
+// above): the staging gateway's only login, on cms_public_staging only.
+const publicStagingRolePassword = pulumi.secret(
+  gcp.secretmanager.getSecretVersionOutput(
+    { project, secret: gcpSecretId(env, "public-staging-role-password") },
+    { provider: gcpProvider },
+  ).secretData,
+);
 
 // =========================================================================
 // Tier 4 — Cloud SQL Postgres (private IP only; HA configurable)
@@ -334,6 +351,13 @@ const cmsPublicDb = new gcp.sql.Database(
   { instance: sqlInstance.name, name: "cms_public" },
   opts,
 );
+// #1329 — staging's own data: the twin of cms_public on the same instance
+// (no extra cost), reached by the staging gateway only.
+const cmsPublicStagingDb = new gcp.sql.Database(
+  `${namePrefix}-cms-public-staging`,
+  { instance: sqlInstance.name, name: "cms_public_staging" },
+  opts,
+);
 
 const pgAdminUser = new gcp.sql.User(
   `${namePrefix}-admin-user`,
@@ -352,6 +376,14 @@ new gcp.sql.User(
   { instance: sqlInstance.name, name: "public_role", password: publicRolePassword },
   opts,
 );
+// #1329 — the staging gateway's login: an Admin API user like public_role,
+// so the same membership revoke (public-role-isolation.ts) and the same
+// audit apply; migration 0530 gives it CONNECT on cms_public_staging only.
+new gcp.sql.User(
+  `${namePrefix}-public-staging-user`,
+  { instance: sqlInstance.name, name: "public_staging_role", password: publicStagingRolePassword },
+  opts,
+);
 
 // #1338 — the instance's DNS name, resolved to its private IP by a private
 // zone in the VPC (Cloud SQL assigns the name, it does not resolve it).
@@ -729,6 +761,7 @@ function cloudRunService(args: CloudRunArgs): gcp.cloudrunv2.Service {
         pgAdminUser,
         cmsAdminDb,
         cmsPublicDb,
+        cmsPublicStagingDb,
         // The env contract names the media bucket as a string; keep it
         // ordered before the services that use it.
         mediaBucket,
@@ -756,6 +789,8 @@ const envContractInputs = {
     admin: dbUrls.admin,
     publicAdmin: dbUrls.publicAdmin,
     public: dbUrls.public,
+    publicStagingAdmin: dbUrls.publicStagingAdmin,
+    publicStaging: dbUrls.publicStaging,
   },
 } as const;
 
@@ -1129,6 +1164,18 @@ new gcp.iap.WebCloudRunServiceIamMember(
 // (packages/provisioning/src/capture-job-setup.ts), for the same reason as
 // the audit job above.
 
+// =========================================================================
+// Scheduler tick — deliberately NOT here
+// =========================================================================
+//
+// #1140 — one Cloud Scheduler job starts a Cloud Workflows execution every
+// `schedulerIntervalMinutes` (1–15, default 5); the workflow signs an IAP
+// credential as the scheduler's own service account and POSTs the admin's
+// /api/internal/scheduler/tick, where the admin runs its periodic work and
+// plugin workers from its due-table. The CLI owns the account, its grants,
+// the workflow and the job (packages/provisioning/src/scheduler-setup.ts),
+// for the same reason as the audit job above.
+
 // =========================================================================
 // Operator access — deliberately NOT here
 // =========================================================================
@@ -1213,6 +1260,7 @@ const out: CloudAdapterOutputs = {
   // Password-less — the password lives only in Secret Manager.
   adminDatabaseUrl: dbUrls.admin as unknown as string,
   publicDatabaseUrl: dbUrls.public as unknown as string,
+  publicStagingDatabaseUrl: dbUrls.publicStaging as unknown as string,
   mediaStorageUrl: pulumi.interpolate`gs://${mediaBucket.name}` as unknown as string,
   // Firebase Hosting serves media via its own CDN; we proxy through
   // the Firebase site's apex domain for canonical URLs.
@@ -1229,6 +1277,7 @@ const out: CloudAdapterOutputs = {
 
 export const adminDatabaseUrlOut = out.adminDatabaseUrl;
 export const publicDatabaseUrlOut = out.publicDatabaseUrl;
+export const publicStagingDatabaseUrlOut = out.publicStagingDatabaseUrl;
 export const mediaStorageUrlOut = out.mediaStorageUrl;
 export const mediaCdnBaseUrlOut = out.mediaCdnBaseUrl;
 export const bootstrapUrlOut = out.bootstrapUrl;

package/stacks/self-hosted/Pulumi.yaml

@@ -29,3 +29,6 @@ config:
     type: string
     default: "2Gi"
     description: Memory reserved for the admin container (it runs the Lighthouse quality audit).
+  caelo-self-hosted:schedulerIntervalMinutes:
+    type: integer
+    description: Minutes between scheduler ticks (1-15, default 5; #1140). On an install `cms-provision init` set up, `.caelo/config.json` records it and this key must match.

package/stacks/self-hosted/README.md

@@ -33,6 +33,10 @@ pulumi stack output bootstrapUrl
 
 Open that URL once Caddy has issued the cert (~30 s).
 
+## Scheduler tick
+
+The Compose file runs a `caelo-scheduler` container (the admin image, no database access, only the internal secret it signs with) that POSTs the admin's `/api/internal/scheduler/tick` every `schedulerIntervalMinutes` (#1140; 1–15, default 5). The admin runs its periodic work and plugin workers on that tick. Set the interval with `cms-provision init --scheduler-interval-minutes 2` (recorded in `.caelo/config.json`, which every CLI render reads) or, on an install without a `config.json`, with `pulumi config set caelo-self-hosted:schedulerIntervalMinutes 2`. The stack reads `.caelo/config.json` too, so a `pulumi up` keeps the recorded interval; a Pulumi key that disagrees with it is refused. An install from before the scheduler gets the container on its next `upgrade`, which rewrites the Compose file.
+
 ## Connection budget
 
 The generated Compose file starts Postgres with `max_connections=100` and sizes every pool from it (#1254, docs/dev/connection-budget.md): the admin gets a cms_admin pool of 77 and a cms_public pool of 4, the gateway a pool of 2, and static builds 2 per pool.

package/stacks/self-hosted/index.ts

@@ -34,7 +34,12 @@ import {
   generateDockerCompose,
   SITE_OUTPUT_ROOT,
 } from "../../dist/compose.js";
+import { MIGRATION_TARGETS } from "../../dist/migration-targets.js";
 import { selfHostedBootstrapScript } from "../../dist/self-hosted-roles.js";
+import {
+  recordedSchedulerIntervalMinutes,
+  selfHostedSchedulerIntervalMinutes,
+} from "../../dist/self-hosted-scheduler-interval.js";
 
 const cfg = new pulumi.Config();
 const domain = cfg.require("domain");
@@ -44,6 +49,17 @@ const ownerEmail = cfg.require("ownerEmail");
 const caeloDir = cfg.get("caeloDir") ?? "./.caelo";
 // #553 — same knob as every adapter; undefined = stack-contract default.
 const adminMemory = cfg.get("adminMemory");
+// #1140 — how often the scheduler container calls the admin's tick (1–15
+// minutes, default 5). `init --scheduler-interval-minutes` records it in
+// <caeloDir>/config.json, which the CLI renders from; the stack reads the
+// same record so a `pulumi up` does not put it back to the default, and
+// refuses a Pulumi key that disagrees with it.
+const caeloConfigPath = resolve(caeloDir, "config.json");
+const schedulerIntervalMinutes = selfHostedSchedulerIntervalMinutes(
+  cfg.getNumber("schedulerIntervalMinutes"),
+  recordedSchedulerIntervalMinutes(caeloConfigPath),
+  caeloConfigPath,
+);
 // #696 — the prebuilt, cosign-verified release images, pinned by digest
 // (`ghcr.io/caelo-cms/<svc>@sha256:…`; the generator refuses anything else).
 // `cms-provision init` resolves + verifies them and records them in
@@ -69,21 +85,30 @@ const postgresPassword = pulumi.secret(randomHex(32));
 // public_role one (#635: no cms_admin login).
 const adminRolePassword = pulumi.secret(randomHex(32));
 const publicRolePassword = pulumi.secret(randomHex(32));
+// #1329 — the staging gateway's own login on cms_public_staging.
+const publicStagingRolePassword = pulumi.secret(randomHex(32));
 // P18 — the KEK that encrypts stored provider keys.
 const caeloSecretKek = pulumi.secret(randomHex(32));
 
 const composeYaml = pulumi
-  .all([postgresPassword, adminRolePassword, publicRolePassword, caeloSecretKek])
-  .apply(([pgPw, adminPw, publicPw, kek]) =>
+  .all([
+    postgresPassword,
+    adminRolePassword,
+    publicRolePassword,
+    publicStagingRolePassword,
+    caeloSecretKek,
+  ])
+  .apply(([pgPw, adminPw, publicPw, publicStagingPw, kek]) =>
     generateDockerCompose({
       domain,
       postgresPassword: pgPw,
-      rolePasswords: { admin: adminPw, public: publicPw },
+      rolePasswords: { admin: adminPw, public: publicPw, publicStaging: publicStagingPw },
       caeloSecretKek: kek,
       anthropicApiKey: process.env.ANTHROPIC_API_KEY,
       resendApiKey: process.env.RESEND_API_KEY,
       diskSize: "20Gi",
       ...(adminMemory ? { adminMemory } : {}),
+      ...(schedulerIntervalMinutes !== undefined ? { schedulerIntervalMinutes } : {}),
       images: {
         mode: "release",
         admin: adminImage,
@@ -144,7 +169,7 @@ const composeUp = new local.Command(
     // install's admin image, then everything else.
     create: pulumi.interpolate`set -e
 docker compose -f ${composePath} up -d --wait postgres
-for target in admin public; do
+for target in ${MIGRATION_TARGETS.join(" ")}; do
   docker compose -f ${composePath} run --rm --no-deps -T caelo-admin bun --bun /app/packages/migrations/src/migrate.ts "$target"
 done
 docker compose -f ${composePath} up -d --remove-orphans`,