@wtfalch/foundry 0.7.1 → 0.9.0

package/dist/plan/platform-providers.js

@@ -1,11 +1,12 @@
 import { createHash, createPrivateKey, generateKeyPairSync, randomBytes, randomUUID, sign, } from 'node:crypto';
 import { isIPv4 } from 'node:net';
-import { VERSIONS } from '../bootstrap/versions.js';
+import { VERSIONS, floatingTagBlocked } from '../bootstrap/versions.js';
 import { redactSecrets } from '../calls-sfu-deploy-support.js';
 import { ensureCloudflaredConnector } from './cloudflared-connector.js';
 import { ACCESS_POLICY_NAME, isNotFoundError, isRunningOrStarting } from './executor.js';
-import { JOBS_SCHEDULER_KEY, PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY, PLATFORM_AI_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_AI_DEPLOY_PENDING_SCOPE_KEY, PLATFORM_AI_INTERNAL_URL_SCOPE_KEY, PLATFORM_AI_KEY, PLATFORM_AI_MANAGEMENT_KEY_CONFLICT_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_AI_MIGRATE_BLOCKED_SCOPE_KEY, PLATFORM_AI_ORGANISATION_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_SET_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_BROKER_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_ARCHON_CONNECTOR_APP_SCOPE_KEY, PLATFORM_ARCHON_GATE_CLOSED_SCOPE_KEY, PLATFORM_ARCHON_KEY, PLATFORM_ARCHON_MASTER_CLOUDFLARE_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_URL_REF, PLATFORM_AUTH_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY, PLATFORM_AUTH_BROKER_DEPLOYED_SCOPE_KEY, PLATFORM_AUTH_BROKER_SCOPE_KEY, PLATFORM_AUTH_CONNECTOR_APP_SCOPE_KEY, PLATFORM_AUTH_KEY, PLATFORM_AUTH_LOGIN_CLIENT_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_LOGIN_CLIENT_USER_SCOPE_KEY, PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_KEY, PLATFORM_FILES_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_FILES_MANAGEMENT_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_ZITADEL_PROJECT_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY, PLATFORM_INTEGRATIONS_KEY_SET_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_ADMIN_KEY_SCOPE_KEY, PLATFORM_MAIL_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_MAIL_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_MAIL_CONNECTOR_APP_SCOPE_KEY, PLATFORM_MAIL_INTERNAL_URL_SCOPE_KEY, PLATFORM_MAIL_KEY, parseBrokerDeployRecord, parseLoginClientRecord, platformMailMxHost, } from './platform-diff.js';
+import { JOBS_SCHEDULER_KEY, PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY, PLATFORM_AI_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_AI_DEPLOY_PENDING_SCOPE_KEY, PLATFORM_AI_INTERNAL_URL_SCOPE_KEY, PLATFORM_AI_KEY, PLATFORM_AI_MANAGEMENT_KEY_CONFLICT_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_AI_MIGRATE_BLOCKED_SCOPE_KEY, PLATFORM_AI_ORGANISATION_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_SET_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_BROKER_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_ARCHON_CONNECTOR_APP_SCOPE_KEY, PLATFORM_ARCHON_GATE_CLOSED_SCOPE_KEY, PLATFORM_ARCHON_KEY, PLATFORM_ARCHON_MASTER_CLOUDFLARE_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_URL_REF, PLATFORM_AUTH_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY, PLATFORM_AUTH_BROKER_DEPLOYED_SCOPE_KEY, PLATFORM_AUTH_BROKER_SCOPE_KEY, PLATFORM_AUTH_CONNECTOR_APP_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_KEY, PLATFORM_AUTH_LOGIN_CLIENT_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_LOGIN_CLIENT_USER_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_APPLIED_IMAGE_TAG_SCOPE_KEY, PLATFORM_BACKUP_DATABASE_SCOPE_KEYS, PLATFORM_BACKUP_SERVICE_KEYS, PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_KEY, PLATFORM_FILES_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_FILES_MANAGEMENT_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_ZITADEL_PROJECT_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY, PLATFORM_INTEGRATIONS_KEY_SET_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_ADMIN_KEY_SCOPE_KEY, PLATFORM_MAIL_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_MAIL_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_MAIL_CONNECTOR_APP_SCOPE_KEY, PLATFORM_MAIL_INTERNAL_URL_SCOPE_KEY, PLATFORM_MAIL_KEY, parseBrokerDeployRecord, parseLoginClientRecord, platformMailMxHost, } from './platform-diff.js';
 import { applyCallsSfuPlatformStep } from './platform-sfu.js';
+import { CreateInstanceError, createZitadelInstance, generateSystemApiKeypair, instanceWithoutBearerMessage, systemApiUsersEnv, } from './zitadel-system-api.js';
 /** The key `runPlatformSteps` records the apex zone's id under (`system:kind:key`). */
 function apexZoneKey(apex) {
     return `cloudflare:zone:${apex}`;
@@ -93,6 +94,10 @@ const JOBS_SCHEDULER_REPO = 'wtfalch/jobs';
  * #175 adds `coolify:app` keyed {@link PLATFORM_ARCHON_KEY}, Archon itself
  * ({@link provisionPlatformArchon}), once a host passes
  * {@link PlatformArchonDeps}.
+ *
+ * #257 adds `coolify:backup:<service>`, a nightly R2 backup of each
+ * platform database ({@link provisionPlatformBackup}), once a host passes
+ * {@link PlatformBackupDeps}.
  */
 export class FoundryPlatformStepExecutor {
     cloudflare;
@@ -106,6 +111,7 @@ export class FoundryPlatformStepExecutor {
     files;
     ai;
     archon;
+    backup;
     constructor(cloudflare, declaredOrgStore, options = {}) {
         this.cloudflare = cloudflare;
         this.declaredOrgStore = declaredOrgStore;
@@ -118,6 +124,7 @@ export class FoundryPlatformStepExecutor {
         this.files = options.files;
         this.ai = options.ai;
         this.archon = options.archon;
+        this.backup = options.backup;
     }
     async blockedSteps(platform, recordedIds = {}) {
         const blocked = {};
@@ -131,8 +138,19 @@ export class FoundryPlatformStepExecutor {
         }
         else if (platform.auth.enabled && this.auth) {
             const { reason: authBlocked, otherInputsMissing } = await authBrokerBlockedReason(this.auth, recordedIds);
-            if (authBlocked)
+            if (authBlocked) {
                 blocked.auth = authBlocked;
+            }
+            else {
+                // #248: the same "block, not throw" rule mail's own tag check
+                // follows. `VERSIONS.zitadel` is a real pin today, so this never
+                // fires in practice -- it is here so a future bump that lands a
+                // floating tag is caught the same way mail's would be.
+                const zitadelTagBlocked = floatingTagBlocked('ZITADEL', VERSIONS.zitadel, 'v4.17.3');
+                if (zitadelTagBlocked) {
+                    blocked.auth = `the platform's sign-in is blocked, not planned: ${zitadelTagBlocked}`;
+                }
+            }
             // #216 round 5: see `PlanPlatformOptions.blocked.authOtherInputsMissing`.
             if (otherInputsMissing.length > 0)
                 blocked.authOtherInputsMissing = otherInputsMissing;
@@ -198,6 +216,14 @@ export class FoundryPlatformStepExecutor {
                 }
             }
         }
+        // Only once a database actually exists to back up -- the same
+        // "nothing to report yet" rule `planBackups` (`platform-diff.ts`)
+        // itself follows for "no database", so an otherwise-unrelated platform
+        // with no backup deps wired carries no standing blocked note.
+        if (!this.backup &&
+            PLATFORM_BACKUP_SERVICE_KEYS.some((service) => Boolean(recordedIds[PLATFORM_BACKUP_DATABASE_SCOPE_KEYS[service]]))) {
+            blocked.backupExecutorMissing = true;
+        }
         return blocked;
     }
     /**
@@ -271,6 +297,12 @@ export class FoundryPlatformStepExecutor {
         if (action.system === 'coolify' && action.kind === 'jobs') {
             return this.coolifyJobs(action, context);
         }
+        if (action.system === 'coolify' && action.kind === 'backup') {
+            if (!this.backup) {
+                throw new Error(`platform:backup:${action.key} needs FoundryPlatformStepExecutor to be constructed with its backup dependencies (#257)`);
+            }
+            return provisionPlatformBackup(action, context, this.backup);
+        }
         if (action.system === 'sfu' && action.kind === 'livekit') {
             return this.executeSfu();
         }
@@ -464,7 +496,7 @@ const AUTH_BROKER_PORT = '3000';
  * falls through to the login app's own ingress rule below (login v2 answers
  * everything else, including the root).
  */
-const AUTH_ZITADEL_PATH_PATTERN = '^/(oidc|oauth|saml|management|admin|system|auth|assets|ui/console|\\.well-known)(/|$)';
+const AUTH_ZITADEL_PATH_PATTERN = '^/(oidc|oauth|saml|management|admin|system|auth|assets|debug/healthz|ui/console|\\.well-known)(/|$)';
 /**
  * `@wtfalch/auth`'s own `brokerUrl` default, `<issuer>/api`
  * (`packages/auth/src/config.ts`) -- `deploy/proxy/Caddyfile`'s own
@@ -484,6 +516,14 @@ const PLATFORM_AUTH_ADMIN_USER_NAME = 'foundry-admin';
 /** #216 (building on #209/#215): the narrower `IAM_LOGIN_CLIENT` machine user for the login app and the broker. */
 const PLATFORM_AUTH_LOGIN_CLIENT_USER_NAME = 'foundry-login-client';
 const PLATFORM_AUTH_ORG_NAME = 'foundry';
+/**
+ * #283: the ref the new instance's IAM_OWNER machine token (a PAT) is
+ * delivered under, once. A host reads it back to build `zitadelForInstance`
+ * (see {@link provisionPlatformAuth}'s doc comment). Distinct from the admin
+ * key delivered under `zitadel:platform:admin`.
+ */
+export const PLATFORM_AUTH_INSTANCE_BEARER_REF = 'zitadel:platform:instanceBearer';
+const PLATFORM_AUTH_BEARER_USER_NAME = 'foundry-instance-owner';
 /**
  * Retries `action` up to `attempts` times, sleeping increasingly between
  * tries -- the same shape the admin machine user's own delivery retry (just
@@ -727,8 +767,24 @@ async function liveLoginClientTokenHash(coolify, appId) {
  *   first), so it will not start (`parseApps` throws on an empty list) until
  *   then either. Safe to run again: the next apply that registers an app
  *   redeploys the broker with a non-empty list.
- * - Whoever supplies `deps.zitadel` (or `zitadelForInstance`) must hold a
- *   bearer token for the new instance, which foundry cannot read from it.
+ * - ~~Whoever supplies `deps.zitadel` (or `zitadelForInstance`) must hold a
+ *   bearer token for the new instance, which foundry cannot read from it.~~
+ *   Closed by #283: see "The instance bearer" below.
+ *
+ * The instance bearer (#283): the container creates no instance of its own
+ * (`ZITADEL_FIRSTINSTANCE_SKIP`). foundry generates an RSA keypair, puts the
+ * public half in `ZITADEL_SYSTEMAPIUSERS` (a system API user with
+ * SYSTEM_OWNER only), and once the container answers calls ZITADEL's system
+ * API `CreateInstance` with a machine owner (IAM_OWNER of the new instance)
+ * signed by the private half, which lives in memory for that run only. The
+ * owner's PAT comes back once and is handed to
+ * {@link PlatformAuthDeps.deliverSecret} under
+ * {@link PLATFORM_AUTH_INSTANCE_BEARER_REF}, before anything else uses it. A
+ * host builds `zitadelForInstance` as `(url) => createZitadelAdminMinter({
+ * url, token: <the secret delivered under that ref> })` plus its own org
+ * calls; `zitadelForInstance` is first called after the delivery. No human
+ * admin is created: a person who needs console access is added by the host
+ * through that bearer.
  *
  * #197: a `cloudflared` connector for this tunnel is run
  * ({@link ensureCloudflaredConnector}), so `hostname` answers.
@@ -809,6 +865,15 @@ async function provisionPlatformAuth(context, deps) {
     await deps.coolify.updateApplication(zitadelAppId, {
         custom_internal_name: PLATFORM_AUTH_ZITADEL_INTERNAL_NAME,
     });
+    // #251: only on create, never unconditionally -- nothing here patches
+    // `docker_registry_image_tag` on an app `findOrCreatePlatformApp` found
+    // already running, so that is the only moment this tag is actually live.
+    // Recording it every run, the way the app id above is, would make
+    // `versionNotAppliedLine` compare a bumped `VERSIONS.zitadel` against a
+    // value that was never applied to the running app.
+    if (zitadelApp.justCreated) {
+        await record(PLATFORM_AUTH_ZITADEL_APPLIED_IMAGE_TAG_SCOPE_KEY, AUTH_ZITADEL_IMAGE_TAG);
+    }
     // ---- the zitadel-login app -----------------------------------------------
     const loginApp = await findOrCreatePlatformApp(deps.coolify, PLATFORM_AUTH_LOGIN_APP_NAME, environment, async () => {
         const server = await resolvePlatformServer(deps.coolify);
@@ -859,7 +924,19 @@ async function provisionPlatformAuth(context, deps) {
     // same rule `executor.ts`'s `ensureAuthCookieSecret` follows for a value
     // this package mints itself rather than receiving from a provider.
     const masterkey = await ensureMasterkey(deps.coolify, zitadelAppId, databaseInUse);
-    const adminPassword = await ensureGeneratedEnv(deps.coolify, zitadelAppId, 'ZITADEL_ADMIN_PASSWORD', generateAdminPassword);
+    // #283: the instance is created once, with the system API, by a platform
+    // whose admin machine user is not recorded yet (one that already has it was
+    // built from a first instance and keeps it). The keypair is regenerated on
+    // any run that still has to create it, so the private half never needs to
+    // be held anywhere between runs.
+    const lostInstance = recorded[PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY];
+    if (lostInstance &&
+        !recorded[PLATFORM_AUTH_INSTANCE_SCOPE_KEY] &&
+        !recorded[PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY]) {
+        throw new Error(instanceWithoutBearerMessage(hostname, `instance ${lostInstance}`));
+    }
+    const needsInstance = !recorded[PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY] && !recorded[PLATFORM_AUTH_INSTANCE_SCOPE_KEY];
+    const systemKeys = needsInstance ? generateSystemApiKeypair() : undefined;
     const zitadelEnvs = [
         { key: 'ZITADEL_PORT', value: AUTH_ZITADEL_PORT },
         { key: 'ZITADEL_EXTERNALDOMAIN', value: hostname },
@@ -867,8 +944,7 @@ async function provisionPlatformAuth(context, deps) {
         { key: 'ZITADEL_EXTERNALSECURE', value: 'true' },
         { key: 'ZITADEL_TLS_ENABLED', value: 'false' },
         { key: 'ZITADEL_DATABASE_POSTGRES_DSN', value: db.internal_db_url },
-        { key: 'ZITADEL_FIRSTINSTANCE_ORG_HUMAN_USERNAME', value: 'zitadel-admin' },
-        { key: 'ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORDCHANGEREQUIRED', value: 'false' },
+        { key: 'ZITADEL_FIRSTINSTANCE_SKIP', value: 'true' },
         { key: 'ZITADEL_DEFAULTINSTANCE_FEATURES_LOGINV2_REQUIRED', value: 'true' },
         { key: 'ZITADEL_DEFAULTINSTANCE_FEATURES_LOGINV2_BASEURI', value: `https://${hostname}/` },
         { key: 'ZITADEL_OIDC_DEFAULTLOGINURLV2', value: `https://${hostname}/login?authRequest=` },
@@ -881,8 +957,12 @@ async function provisionPlatformAuth(context, deps) {
     ];
     if (masterkey)
         zitadelEnvs.push({ key: 'ZITADEL_MASTERKEY', value: masterkey });
-    if (adminPassword)
-        zitadelEnvs.push({ key: 'ZITADEL_ADMIN_PASSWORD', value: adminPassword });
+    if (systemKeys) {
+        zitadelEnvs.push({
+            key: 'ZITADEL_SYSTEMAPIUSERS',
+            value: systemApiUsersEnv(systemKeys.publicKey),
+        });
+    }
     const loginEnvs = [
         {
             key: 'ZITADEL_API_URL',
@@ -1011,6 +1091,50 @@ async function provisionPlatformAuth(context, deps) {
             // a caller that reaches the step anyway.
             throw new Error(`platform:auth: the admin key of ZITADEL machine user ${lost} was minted but never delivered; delete that user and the recorded key ${PLATFORM_AUTH_ADMIN_UNDELIVERED_SCOPE_KEY} to mint a new one`);
         }
+        if (systemKeys) {
+            // ZITADEL answers /debug/healthz without an instance; the openid
+            // document below only exists once the instance does.
+            await waitForInstance(`https://${hostname}/debug/healthz`, runtime, sleep);
+            // Sent exactly once. Anything that may have reached ZITADEL leaves a
+            // marker first, so the next run refuses instead of creating again.
+            let created;
+            try {
+                created = await createZitadelInstance({
+                    customDomain: hostname,
+                    privateKey: systemKeys.privateKey,
+                    instanceName: PLATFORM_AUTH_ORG_NAME,
+                    firstOrgName: PLATFORM_AUTH_ORG_NAME,
+                    owner: { userName: PLATFORM_AUTH_BEARER_USER_NAME, name: 'Foundry (instance owner)' },
+                }, { fetch: runtime.fetch, now: runtime.now ?? Date.now });
+            }
+            catch (error) {
+                if (error instanceof CreateInstanceError && error.kind !== 'rejected') {
+                    await record(PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY, 'unknown');
+                }
+                throw error;
+            }
+            const { instanceId, pat } = created;
+            // Recorded before the PAT leaves this function, like the admin key below.
+            await record(PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY, instanceId);
+            const attempts = runtime.deliverAttempts ?? 3;
+            for (let attempt = 1;; attempt += 1) {
+                try {
+                    await deps.deliverSecret(PLATFORM_AUTH_INSTANCE_BEARER_REF, pat);
+                    break;
+                }
+                catch (error) {
+                    if (attempt >= attempts) {
+                        const reason = redactSecrets(error instanceof Error ? error.message : String(error), [
+                            pat,
+                        ]);
+                        throw new Error(`platform:auth: the owner token of ZITADEL instance ${instanceId} was created but could not be delivered after ${attempts} tries (${reason}); ${instanceWithoutBearerMessage(hostname, `instance ${instanceId}`)}`);
+                    }
+                    await sleep(1000 * attempt);
+                }
+            }
+            // The marker stays recorded but is ignored once this is set.
+            await record(PLATFORM_AUTH_INSTANCE_SCOPE_KEY, instanceId);
+        }
         // Minting needs a running instance behind the tunnel: the first apply
         // gets here right after the deploy.
         await waitForInstance(`https://${hostname}/.well-known/openid-configuration`, runtime, sleep);
@@ -1520,12 +1644,17 @@ async function provisionPlatformMail(context, deps) {
     await record(PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY, imageTag);
     return { resourceId: appId };
 }
-/** Why the Stalwart image tag cannot be used, or `null`. A floating tag cannot be re-pulled by comparing tags. */
+/**
+ * Why the Stalwart image tag cannot be used, or `null`. #248: a thin wrapper
+ * over the estate-wide {@link floatingTagBlocked} -- the mail step's own
+ * message, naming `gitRef` since that is the declared field this tag
+ * actually comes from.
+ */
 function mailImageTagBlocked(tag) {
-    if (tag.trim() === '' || tag.trim().toLowerCase() === 'latest') {
-        return `the platform's mail server is blocked, not planned: the Stalwart image tag "${tag}" floats; declare a pinned release (for example v0.16.24) as the mail gitRef`;
-    }
-    return null;
+    const reason = floatingTagBlocked('Stalwart', tag, 'v0.16.24');
+    return reason
+        ? `the platform's mail server is blocked, not planned: ${reason} as the mail gitRef`
+        : null;
 }
 /** Why the server's address cannot be the MX A record's content, or `null`. */
 function mailServerIpBlocked(ip) {
@@ -1933,7 +2062,7 @@ async function provisionPlatformArchon(context, deps) {
         // response was lost, or one made for another hostname: its client id
         // cannot be read back (`listApps` does not carry it), so it is deleted and
         // made again, as `zitadelApp` does.
-        const orphan = (await zitadel.listApps(org.id, project.id)).find((a) => a.name === PLATFORM_ARCHON_ZITADEL_APP &&
+        const orphan = (await zitadel.listApps(org.id, project.id)).apps.find((a) => a.name === PLATFORM_ARCHON_ZITADEL_APP &&
             a.appType === 'web' &&
             a.state !== 'APP_STATE_REMOVED');
         if (orphan) {
@@ -3597,10 +3726,6 @@ async function ensureMasterkey(coolify, appId, databaseInUse) {
     }
     return randomBytes(24).toString('base64');
 }
-/** `docs/deploy.md`: "ZITADEL_ADMIN_PASSWORD: 8+, upper, lower, digit, symbol". */
-function generateAdminPassword() {
-    return `${randomBytes(18).toString('base64')}Aa1!`;
-}
 /**
  * #171: how the host signs a files management request with the key this step
  * delivers under {@link FILES_MANAGEMENT_KEY_REF} (files'
@@ -3836,7 +3961,7 @@ async function provisionPlatformFiles(context, deps) {
         // An app by this name here has no recorded secret (no undelivered id, or
         // the throw above would have fired): a crash between the create and the
         // record. Nothing was delivered, so it is deleted and made again.
-        const stale = (await zitadel.listApps(org.id, project.id)).filter((app) => app.name === FILES_INTROSPECTION_APP_NAME && app.state !== 'APP_STATE_REMOVED');
+        const stale = (await zitadel.listApps(org.id, project.id)).apps.filter((app) => app.name === FILES_INTROSPECTION_APP_NAME && app.state !== 'APP_STATE_REMOVED');
         for (const app of stale) {
             try {
                 await zitadel.deleteApp(org.id, project.id, app.id);
@@ -4339,6 +4464,61 @@ async function waitForIntegrationsMigration(coolify, appId, runtime, sleep) {
         await sleep(interval);
     }
 }
+/** The name every platform database's backup is registered against -- one shared R2 bucket, one storage entry. */
+const PLATFORM_BACKUP_S3_STORAGE_NAME = 'platform-backups';
+/** The uuid `provisionPlatformBackup` records once {@link PLATFORM_BACKUP_S3_STORAGE_NAME} is registered on Coolify, so a later apply never re-registers it. */
+const PLATFORM_BACKUP_S3_STORAGE_SCOPE_KEY = 'coolify_platform_backup_s3_storage';
+/** Coolify's own name for "once a day" -- `createDatabaseBackup`'s `frequency`. */
+const PLATFORM_BACKUP_DEFAULT_FREQUENCY = 'daily';
+/**
+ * #257: schedules a nightly R2 backup of one platform database --
+ * `planBackups` (`platform-diff.ts`) plans this step once per service, each
+ * carrying that service's own database id as `action.desired.databaseId`
+ * (recorded by that service's own `coolify:app` step; never minted here).
+ * The R2 bucket is registered as a Coolify S3 storage once, by name, and
+ * its uuid recorded under {@link PLATFORM_BACKUP_S3_STORAGE_SCOPE_KEY} so
+ * every service's own backup step -- and a rerun of this one -- reuses it
+ * rather than registering a second storage entry under the same name.
+ * `createDatabaseBackup` itself is the idempotency boundary for the
+ * schedule: an existing enabled, `save_s3` schedule is left alone rather
+ * than replaced, since Coolify has no "upsert" for this call and a second
+ * schedule would dump the same database twice a night.
+ */
+async function provisionPlatformBackup(action, context, deps) {
+    const { coolify, r2 } = deps;
+    const databaseId = action.desired.databaseId;
+    const recorded = context.recordedIds();
+    let storageUuid = recorded[PLATFORM_BACKUP_S3_STORAGE_SCOPE_KEY];
+    if (!storageUuid) {
+        const found = (await coolify.listS3Storages()).find((storage) => storage.name === PLATFORM_BACKUP_S3_STORAGE_NAME);
+        if (found) {
+            storageUuid = found.uuid;
+        }
+        else {
+            const created = await coolify.createS3Storage({
+                name: PLATFORM_BACKUP_S3_STORAGE_NAME,
+                endpoint: r2.endpoint,
+                bucket: r2.bucket,
+                region: r2.region ?? 'auto',
+                key: r2.accessKeyId,
+                secret: r2.secretAccessKey,
+                is_usable: true,
+            });
+            storageUuid = created.uuid;
+        }
+        context.recordScopeId(PLATFORM_BACKUP_S3_STORAGE_SCOPE_KEY, storageUuid);
+    }
+    const existing = (await coolify.listDatabaseBackups(databaseId)).find((backup) => backup.enabled && backup.save_s3);
+    if (!existing) {
+        await coolify.createDatabaseBackup(databaseId, {
+            frequency: deps.frequency ?? PLATFORM_BACKUP_DEFAULT_FREQUENCY,
+            enabled: true,
+            save_s3: true,
+            s3_storage_uuid: storageUuid,
+        });
+    }
+    return undefined;
+}
 /**
  * The real {@link PlatformCloudflareReader}: the platform's apex zone as
  * `planApexZone` expects to see it, and nothing else. The zone's DNS records
@@ -4393,23 +4573,55 @@ export class ProvidersPlatformCoolifyReader {
         this.platformStore = platformStore;
     }
     async listResources() {
-        const recordedId = (await this.platformStore.getServiceIds())[JOBS_APP_KEY];
-        if (!recordedId)
-            return [];
-        try {
-            const app = await this.client.getApplication(recordedId);
-            return [
-                {
+        const recorded = await this.platformStore.getServiceIds();
+        const resources = [];
+        const recordedAppId = recorded[JOBS_APP_KEY];
+        if (recordedAppId) {
+            try {
+                const app = await this.client.getApplication(recordedAppId);
+                resources.push({
                     system: 'coolify',
                     kind: 'jobs',
                     key: JOBS_SCHEDULER_KEY,
                     config: { uuid: app.uuid, status: app.status },
-                },
-            ];
+                });
+            }
+            catch {
+                // A recorded id Coolify no longer has reads as not-live (see this
+                // class's own doc comment); the scheduler, not every other resource
+                // this reader checks.
+            }
         }
-        catch {
-            return [];
+        // #257: one backup-schedule resource per service whose database is
+        // recorded, found the only way a schedule can be: listed off that
+        // database itself, since Coolify names no schedule of its own.
+        for (const service of PLATFORM_BACKUP_SERVICE_KEYS) {
+            const databaseId = recorded[PLATFORM_BACKUP_DATABASE_SCOPE_KEYS[service]];
+            if (!databaseId)
+                continue;
+            try {
+                const backups = await this.client.listDatabaseBackups(databaseId);
+                const scheduled = backups.find((backup) => backup.enabled && backup.save_s3);
+                if (scheduled) {
+                    resources.push({
+                        system: 'coolify',
+                        kind: 'backup',
+                        key: service,
+                        config: { uuid: scheduled.uuid, frequency: scheduled.frequency },
+                    });
+                }
+            }
+            catch {
+                // A Coolify error (or a recorded database id Coolify no longer has)
+                // reads as not-live, the same as the jobs app above -- `databaseId`
+                // is still recorded, so `planBackups` plans the step again rather
+                // than reporting it blocked; `provisionPlatformBackup`'s own
+                // `listDatabaseBackups` check (run again there) is what keeps a
+                // transient failure here from scheduling a second backup once the
+                // first one did succeed.
+            }
         }
+        return resources;
     }
 }
 /**

package/dist/plan/zitadel-system-api.d.ts

@@ -0,0 +1,64 @@
+/**
+ * #283: foundry's way into a brand-new ZITADEL instance. ZITADEL's system API
+ * (`POST /system/v1/instances/_create`, permission `system.instance.write`,
+ * https://github.com/zitadel/zitadel/blob/main/proto/zitadel/system.proto)
+ * creates the instance with a machine owner (IAM_OWNER of that instance) and
+ * returns that owner's personal access token (`pat`) once, inline. foundry
+ * authenticates to the system API with a short-lived RS256 JWT signed by a
+ * keypair it generates: the public half goes into the container as a system
+ * API user, the private half stays in memory for one run. The calls follow
+ * ZITADEL's source and proto and have not been run against a live instance.
+ */
+/** The system API user's name; it is the JWT's `iss` and `sub`. */
+export declare const SYSTEM_API_USER_NAME = "foundry";
+/** A fresh RSA keypair: the public key (PEM) for ZITADEL, the private key (PKCS#8 PEM) for signing. */
+export declare function generateSystemApiKeypair(): {
+    publicKey: string;
+    privateKey: string;
+};
+/**
+ * The value of `ZITADEL_SYSTEMAPIUSERS`: JSON in ZITADEL's `SystemAPIUsers`
+ * shape (`cmd/defaults.yaml`), `KeyData` being the base64 of the public key
+ * PEM. With no `Memberships` ZITADEL grants exactly System/SYSTEM_OWNER.
+ */
+export declare function systemApiUsersEnv(publicKeyPem: string): string;
+/** A 1-minute RS256 JWT for the system API: `iss` = `sub` = the user name, `aud` = the instance URL. */
+export declare function signSystemApiJwt(privateKeyPem: string, audience: string, nowMs: number): string;
+/**
+ * How a create failed. `rejected`: ZITADEL refused it for good before doing
+ * anything (bad key, bad request), so nothing exists. `exists`: an instance
+ * for the domain already exists. `uncertain`: the request may have reached
+ * ZITADEL (timeout, network error, 5xx, an unreadable answer), so an instance
+ * may exist. The caller records a marker for the last two.
+ */
+export declare class CreateInstanceError extends Error {
+    readonly kind: 'rejected' | 'exists' | 'uncertain';
+    constructor(kind: 'rejected' | 'exists' | 'uncertain', message: string);
+}
+/** What a person does when an instance exists without a delivered bearer. Nothing is deleted automatically. */
+export declare function instanceWithoutBearerMessage(hostname: string, detail: string): string;
+export interface CreateInstanceRuntime {
+    fetch?: typeof fetch;
+    now: () => number;
+    /** Fetch timeout for the one create call (default 30s). */
+    timeoutMs?: number;
+}
+/**
+ * Creates the platform's ZITADEL instance on `customDomain` with one machine
+ * owner and returns its id and the owner's PAT. Sends exactly one request,
+ * never retries, and signs a fresh 60-second JWT for it. The PAT is returned
+ * to the caller only, never logged or put in an error.
+ */
+export declare function createZitadelInstance(input: {
+    customDomain: string;
+    privateKey: string;
+    instanceName: string;
+    firstOrgName: string;
+    owner: {
+        userName: string;
+        name: string;
+    };
+}, runtime: CreateInstanceRuntime): Promise<{
+    instanceId: string;
+    pat: string;
+}>;

package/dist/plan/zitadel-system-api.js

@@ -0,0 +1,125 @@
+import { createSign, generateKeyPairSync } from 'node:crypto';
+/**
+ * #283: foundry's way into a brand-new ZITADEL instance. ZITADEL's system API
+ * (`POST /system/v1/instances/_create`, permission `system.instance.write`,
+ * https://github.com/zitadel/zitadel/blob/main/proto/zitadel/system.proto)
+ * creates the instance with a machine owner (IAM_OWNER of that instance) and
+ * returns that owner's personal access token (`pat`) once, inline. foundry
+ * authenticates to the system API with a short-lived RS256 JWT signed by a
+ * keypair it generates: the public half goes into the container as a system
+ * API user, the private half stays in memory for one run. The calls follow
+ * ZITADEL's source and proto and have not been run against a live instance.
+ */
+/** The system API user's name; it is the JWT's `iss` and `sub`. */
+export const SYSTEM_API_USER_NAME = 'foundry';
+/** A fresh RSA keypair: the public key (PEM) for ZITADEL, the private key (PKCS#8 PEM) for signing. */
+export function generateSystemApiKeypair() {
+    return generateKeyPairSync('rsa', {
+        modulusLength: 2048,
+        publicKeyEncoding: { type: 'spki', format: 'pem' },
+        privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
+    });
+}
+/**
+ * The value of `ZITADEL_SYSTEMAPIUSERS`: JSON in ZITADEL's `SystemAPIUsers`
+ * shape (`cmd/defaults.yaml`), `KeyData` being the base64 of the public key
+ * PEM. With no `Memberships` ZITADEL grants exactly System/SYSTEM_OWNER.
+ */
+export function systemApiUsersEnv(publicKeyPem) {
+    return JSON.stringify({
+        [SYSTEM_API_USER_NAME]: { KeyData: Buffer.from(publicKeyPem).toString('base64') },
+    });
+}
+const b64url = (input) => Buffer.from(input).toString('base64url');
+/** A 1-minute RS256 JWT for the system API: `iss` = `sub` = the user name, `aud` = the instance URL. */
+export function signSystemApiJwt(privateKeyPem, audience, nowMs) {
+    const iat = Math.floor(nowMs / 1000);
+    const head = b64url(JSON.stringify({ alg: 'RS256', typ: 'JWT' }));
+    const body = b64url(JSON.stringify({
+        iss: SYSTEM_API_USER_NAME,
+        sub: SYSTEM_API_USER_NAME,
+        aud: audience,
+        iat,
+        exp: iat + 60,
+    }));
+    const signature = createSign('RSA-SHA256').update(`${head}.${body}`).sign(privateKeyPem);
+    return `${head}.${body}.${b64url(signature)}`;
+}
+/**
+ * How a create failed. `rejected`: ZITADEL refused it for good before doing
+ * anything (bad key, bad request), so nothing exists. `exists`: an instance
+ * for the domain already exists. `uncertain`: the request may have reached
+ * ZITADEL (timeout, network error, 5xx, an unreadable answer), so an instance
+ * may exist. The caller records a marker for the last two.
+ */
+export class CreateInstanceError extends Error {
+    kind;
+    constructor(kind, message) {
+        super(message);
+        this.kind = kind;
+        this.name = 'CreateInstanceError';
+    }
+}
+/** What a person does when an instance exists without a delivered bearer. Nothing is deleted automatically. */
+export function instanceWithoutBearerMessage(hostname, detail) {
+    return `platform:auth: a ZITADEL instance may already exist for ${hostname} without a delivered bearer (${detail}). To recover, delete the instance with the system API (DELETE /system/v1/instances/{id}) or redeploy the ZITADEL database, remove the recorded key zitadel_platform_instance_undelivered, then apply again`;
+}
+/**
+ * Creates the platform's ZITADEL instance on `customDomain` with one machine
+ * owner and returns its id and the owner's PAT. Sends exactly one request,
+ * never retries, and signs a fresh 60-second JWT for it. The PAT is returned
+ * to the caller only, never logged or put in an error.
+ */
+export async function createZitadelInstance(input, runtime) {
+    const fetchImpl = runtime.fetch ?? globalThis.fetch;
+    const url = `https://${input.customDomain}`;
+    let response;
+    try {
+        response = await fetchImpl(`${url}/system/v1/instances/_create`, {
+            method: 'POST',
+            signal: AbortSignal.timeout(runtime.timeoutMs ?? 30_000),
+            headers: {
+                authorization: `Bearer ${signSystemApiJwt(input.privateKey, url, runtime.now())}`,
+                'content-type': 'application/json',
+            },
+            body: JSON.stringify({
+                instanceName: input.instanceName,
+                firstOrgName: input.firstOrgName,
+                customDomain: input.customDomain,
+                machine: {
+                    userName: input.owner.userName,
+                    name: input.owner.name,
+                    personalAccessToken: {
+                        expirationDate: new Date(runtime.now() + 10 * 365 * 86_400_000).toISOString(),
+                    },
+                },
+            }),
+        });
+    }
+    catch {
+        throw new CreateInstanceError('uncertain', instanceWithoutBearerMessage(input.customDomain, 'the create request timed out or failed'));
+    }
+    if (response.status === 401) {
+        throw new CreateInstanceError('rejected', 'platform:auth: system API key rejected (HTTP 401); nothing was created');
+    }
+    if (response.status === 409) {
+        throw new CreateInstanceError('exists', instanceWithoutBearerMessage(input.customDomain, 'ZITADEL answered 409, already exists'));
+    }
+    if (!response.ok) {
+        // A 4xx is a refusal before anything was created; a 5xx may not be.
+        throw new CreateInstanceError(response.status < 500 ? 'rejected' : 'uncertain', response.status < 500
+            ? `platform:auth: ZITADEL refused to create the instance (HTTP ${response.status}); nothing was created`
+            : instanceWithoutBearerMessage(input.customDomain, `ZITADEL answered HTTP ${response.status}`));
+    }
+    let body = {};
+    try {
+        body = (await response.json());
+    }
+    catch {
+        // Handled below.
+    }
+    if (!body.instanceId || !body.pat) {
+        throw new CreateInstanceError('uncertain', instanceWithoutBearerMessage(input.customDomain, 'the answer had no instance id or token'));
+    }
+    return { instanceId: body.instanceId, pat: body.pat };
+}