@wtfalch/foundry 0.8.0 → 0.10.0

package/dist/plan/platform-providers.js

@@ -4,8 +4,9 @@ import { VERSIONS, floatingTagBlocked } from '../bootstrap/versions.js';
 import { redactSecrets } from '../calls-sfu-deploy-support.js';
 import { ensureCloudflaredConnector } from './cloudflared-connector.js';
 import { ACCESS_POLICY_NAME, isNotFoundError, isRunningOrStarting } from './executor.js';
-import { JOBS_SCHEDULER_KEY, PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY, PLATFORM_AI_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_AI_DEPLOY_PENDING_SCOPE_KEY, PLATFORM_AI_INTERNAL_URL_SCOPE_KEY, PLATFORM_AI_KEY, PLATFORM_AI_MANAGEMENT_KEY_CONFLICT_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_AI_MIGRATE_BLOCKED_SCOPE_KEY, PLATFORM_AI_ORGANISATION_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_SET_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_BROKER_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_ARCHON_CONNECTOR_APP_SCOPE_KEY, PLATFORM_ARCHON_GATE_CLOSED_SCOPE_KEY, PLATFORM_ARCHON_KEY, PLATFORM_ARCHON_MASTER_CLOUDFLARE_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_URL_REF, PLATFORM_AUTH_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY, PLATFORM_AUTH_BROKER_DEPLOYED_SCOPE_KEY, PLATFORM_AUTH_BROKER_SCOPE_KEY, PLATFORM_AUTH_CONNECTOR_APP_SCOPE_KEY, PLATFORM_AUTH_KEY, PLATFORM_AUTH_LOGIN_CLIENT_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_LOGIN_CLIENT_USER_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_APPLIED_IMAGE_TAG_SCOPE_KEY, PLATFORM_BACKUP_DATABASE_SCOPE_KEYS, PLATFORM_BACKUP_SERVICE_KEYS, PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_KEY, PLATFORM_FILES_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_FILES_MANAGEMENT_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_ZITADEL_PROJECT_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY, PLATFORM_INTEGRATIONS_KEY_SET_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_ADMIN_KEY_SCOPE_KEY, PLATFORM_MAIL_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_MAIL_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_MAIL_CONNECTOR_APP_SCOPE_KEY, PLATFORM_MAIL_INTERNAL_URL_SCOPE_KEY, PLATFORM_MAIL_KEY, parseBrokerDeployRecord, parseLoginClientRecord, platformMailMxHost, } from './platform-diff.js';
+import { JOBS_SCHEDULER_KEY, PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY, PLATFORM_AI_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_AI_DEPLOY_PENDING_SCOPE_KEY, PLATFORM_AI_INTERNAL_URL_SCOPE_KEY, PLATFORM_AI_KEY, PLATFORM_AI_MANAGEMENT_KEY_CONFLICT_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_AI_MIGRATE_BLOCKED_SCOPE_KEY, PLATFORM_AI_ORGANISATION_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_SET_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_BROKER_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_ARCHON_CONNECTOR_APP_SCOPE_KEY, PLATFORM_ARCHON_GATE_CLOSED_SCOPE_KEY, PLATFORM_ARCHON_KEY, PLATFORM_ARCHON_MASTER_CLOUDFLARE_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_URL_REF, PLATFORM_AUTH_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY, PLATFORM_AUTH_BROKER_DEPLOYED_SCOPE_KEY, PLATFORM_AUTH_BROKER_SCOPE_KEY, PLATFORM_AUTH_CONNECTOR_APP_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_KEY, PLATFORM_AUTH_LOGIN_CLIENT_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_LOGIN_CLIENT_USER_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_APPLIED_IMAGE_TAG_SCOPE_KEY, PLATFORM_BACKUP_DATABASE_SCOPE_KEYS, PLATFORM_BACKUP_SERVICE_KEYS, PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_KEY, PLATFORM_FILES_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_FILES_MANAGEMENT_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_ZITADEL_PROJECT_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY, PLATFORM_INTEGRATIONS_KEY_SET_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_ADMIN_KEY_SCOPE_KEY, PLATFORM_MAIL_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_MAIL_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_MAIL_CONNECTOR_APP_SCOPE_KEY, PLATFORM_MAIL_INTERNAL_URL_SCOPE_KEY, PLATFORM_MAIL_KEY, parseBrokerDeployRecord, parseLoginClientRecord, platformMailMxHost, } from './platform-diff.js';
 import { applyCallsSfuPlatformStep } from './platform-sfu.js';
+import { CreateInstanceError, createZitadelInstance, generateSystemApiKeypair, instanceWithoutBearerMessage, systemApiUsersEnv, } from './zitadel-system-api.js';
 /** The key `runPlatformSteps` records the apex zone's id under (`system:kind:key`). */
 function apexZoneKey(apex) {
     return `cloudflare:zone:${apex}`;
@@ -247,6 +248,18 @@ export class FoundryPlatformStepExecutor {
             if (loginClientDrifted)
                 result.auth = true;
         }
+        // #257 fix round 2: only once a platform database actually exists to
+        // back up -- the same "nothing to report yet" gate `blockedSteps`'s own
+        // `backupExecutorMissing` check follows above, so a platform that has
+        // not applied any service yet never pays for a live Coolify read here.
+        if (this.backup &&
+            PLATFORM_BACKUP_SERVICE_KEYS.some((service) => Boolean(recordedIds[PLATFORM_BACKUP_DATABASE_SCOPE_KEYS[service]]))) {
+            const backupStorage = await platformBackupStorageDrifted(this.backup.coolify, this.backup.r2, recordedIds);
+            if (backupStorage.drifted)
+                result.backupStorage = true;
+            if (backupStorage.blocked)
+                result.backupStorageBlocked = backupStorage.blocked;
+        }
         return result;
     }
     async execute(action, context) {
@@ -302,6 +315,12 @@ export class FoundryPlatformStepExecutor {
             }
             return provisionPlatformBackup(action, context, this.backup);
         }
+        if (action.system === 'coolify' && action.kind === 'backup-storage') {
+            if (!this.backup) {
+                throw new Error('platform:backup-storage needs FoundryPlatformStepExecutor to be constructed with its backup dependencies (#257)');
+            }
+            return provisionPlatformBackupStorage(context, this.backup);
+        }
         if (action.system === 'sfu' && action.kind === 'livekit') {
             return this.executeSfu();
         }
@@ -495,7 +514,7 @@ const AUTH_BROKER_PORT = '3000';
  * falls through to the login app's own ingress rule below (login v2 answers
  * everything else, including the root).
  */
-const AUTH_ZITADEL_PATH_PATTERN = '^/(oidc|oauth|saml|management|admin|system|auth|assets|ui/console|\\.well-known)(/|$)';
+const AUTH_ZITADEL_PATH_PATTERN = '^/(oidc|oauth|saml|management|admin|system|auth|assets|debug/healthz|ui/console|\\.well-known)(/|$)';
 /**
  * `@wtfalch/auth`'s own `brokerUrl` default, `<issuer>/api`
  * (`packages/auth/src/config.ts`) -- `deploy/proxy/Caddyfile`'s own
@@ -515,6 +534,14 @@ const PLATFORM_AUTH_ADMIN_USER_NAME = 'foundry-admin';
 /** #216 (building on #209/#215): the narrower `IAM_LOGIN_CLIENT` machine user for the login app and the broker. */
 const PLATFORM_AUTH_LOGIN_CLIENT_USER_NAME = 'foundry-login-client';
 const PLATFORM_AUTH_ORG_NAME = 'foundry';
+/**
+ * #283: the ref the new instance's IAM_OWNER machine token (a PAT) is
+ * delivered under, once. A host reads it back to build `zitadelForInstance`
+ * (see {@link provisionPlatformAuth}'s doc comment). Distinct from the admin
+ * key delivered under `zitadel:platform:admin`.
+ */
+export const PLATFORM_AUTH_INSTANCE_BEARER_REF = 'zitadel:platform:instanceBearer';
+const PLATFORM_AUTH_BEARER_USER_NAME = 'foundry-instance-owner';
 /**
  * Retries `action` up to `attempts` times, sleeping increasingly between
  * tries -- the same shape the admin machine user's own delivery retry (just
@@ -758,8 +785,24 @@ async function liveLoginClientTokenHash(coolify, appId) {
  *   first), so it will not start (`parseApps` throws on an empty list) until
  *   then either. Safe to run again: the next apply that registers an app
  *   redeploys the broker with a non-empty list.
- * - Whoever supplies `deps.zitadel` (or `zitadelForInstance`) must hold a
- *   bearer token for the new instance, which foundry cannot read from it.
+ * - ~~Whoever supplies `deps.zitadel` (or `zitadelForInstance`) must hold a
+ *   bearer token for the new instance, which foundry cannot read from it.~~
+ *   Closed by #283: see "The instance bearer" below.
+ *
+ * The instance bearer (#283): the container creates no instance of its own
+ * (`ZITADEL_FIRSTINSTANCE_SKIP`). foundry generates an RSA keypair, puts the
+ * public half in `ZITADEL_SYSTEMAPIUSERS` (a system API user with
+ * SYSTEM_OWNER only), and once the container answers calls ZITADEL's system
+ * API `CreateInstance` with a machine owner (IAM_OWNER of the new instance)
+ * signed by the private half, which lives in memory for that run only. The
+ * owner's PAT comes back once and is handed to
+ * {@link PlatformAuthDeps.deliverSecret} under
+ * {@link PLATFORM_AUTH_INSTANCE_BEARER_REF}, before anything else uses it. A
+ * host builds `zitadelForInstance` as `(url) => createZitadelAdminMinter({
+ * url, token: <the secret delivered under that ref> })` plus its own org
+ * calls; `zitadelForInstance` is first called after the delivery. No human
+ * admin is created: a person who needs console access is added by the host
+ * through that bearer.
  *
  * #197: a `cloudflared` connector for this tunnel is run
  * ({@link ensureCloudflaredConnector}), so `hostname` answers.
@@ -899,7 +942,19 @@ async function provisionPlatformAuth(context, deps) {
     // same rule `executor.ts`'s `ensureAuthCookieSecret` follows for a value
     // this package mints itself rather than receiving from a provider.
     const masterkey = await ensureMasterkey(deps.coolify, zitadelAppId, databaseInUse);
-    const adminPassword = await ensureGeneratedEnv(deps.coolify, zitadelAppId, 'ZITADEL_ADMIN_PASSWORD', generateAdminPassword);
+    // #283: the instance is created once, with the system API, by a platform
+    // whose admin machine user is not recorded yet (one that already has it was
+    // built from a first instance and keeps it). The keypair is regenerated on
+    // any run that still has to create it, so the private half never needs to
+    // be held anywhere between runs.
+    const lostInstance = recorded[PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY];
+    if (lostInstance &&
+        !recorded[PLATFORM_AUTH_INSTANCE_SCOPE_KEY] &&
+        !recorded[PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY]) {
+        throw new Error(instanceWithoutBearerMessage(hostname, `instance ${lostInstance}`));
+    }
+    const needsInstance = !recorded[PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY] && !recorded[PLATFORM_AUTH_INSTANCE_SCOPE_KEY];
+    const systemKeys = needsInstance ? generateSystemApiKeypair() : undefined;
     const zitadelEnvs = [
         { key: 'ZITADEL_PORT', value: AUTH_ZITADEL_PORT },
         { key: 'ZITADEL_EXTERNALDOMAIN', value: hostname },
@@ -907,8 +962,7 @@ async function provisionPlatformAuth(context, deps) {
         { key: 'ZITADEL_EXTERNALSECURE', value: 'true' },
         { key: 'ZITADEL_TLS_ENABLED', value: 'false' },
         { key: 'ZITADEL_DATABASE_POSTGRES_DSN', value: db.internal_db_url },
-        { key: 'ZITADEL_FIRSTINSTANCE_ORG_HUMAN_USERNAME', value: 'zitadel-admin' },
-        { key: 'ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORDCHANGEREQUIRED', value: 'false' },
+        { key: 'ZITADEL_FIRSTINSTANCE_SKIP', value: 'true' },
         { key: 'ZITADEL_DEFAULTINSTANCE_FEATURES_LOGINV2_REQUIRED', value: 'true' },
         { key: 'ZITADEL_DEFAULTINSTANCE_FEATURES_LOGINV2_BASEURI', value: `https://${hostname}/` },
         { key: 'ZITADEL_OIDC_DEFAULTLOGINURLV2', value: `https://${hostname}/login?authRequest=` },
@@ -921,8 +975,12 @@ async function provisionPlatformAuth(context, deps) {
     ];
     if (masterkey)
         zitadelEnvs.push({ key: 'ZITADEL_MASTERKEY', value: masterkey });
-    if (adminPassword)
-        zitadelEnvs.push({ key: 'ZITADEL_ADMIN_PASSWORD', value: adminPassword });
+    if (systemKeys) {
+        zitadelEnvs.push({
+            key: 'ZITADEL_SYSTEMAPIUSERS',
+            value: systemApiUsersEnv(systemKeys.publicKey),
+        });
+    }
     const loginEnvs = [
         {
             key: 'ZITADEL_API_URL',
@@ -1051,6 +1109,50 @@ async function provisionPlatformAuth(context, deps) {
             // a caller that reaches the step anyway.
             throw new Error(`platform:auth: the admin key of ZITADEL machine user ${lost} was minted but never delivered; delete that user and the recorded key ${PLATFORM_AUTH_ADMIN_UNDELIVERED_SCOPE_KEY} to mint a new one`);
         }
+        if (systemKeys) {
+            // ZITADEL answers /debug/healthz without an instance; the openid
+            // document below only exists once the instance does.
+            await waitForInstance(`https://${hostname}/debug/healthz`, runtime, sleep);
+            // Sent exactly once. Anything that may have reached ZITADEL leaves a
+            // marker first, so the next run refuses instead of creating again.
+            let created;
+            try {
+                created = await createZitadelInstance({
+                    customDomain: hostname,
+                    privateKey: systemKeys.privateKey,
+                    instanceName: PLATFORM_AUTH_ORG_NAME,
+                    firstOrgName: PLATFORM_AUTH_ORG_NAME,
+                    owner: { userName: PLATFORM_AUTH_BEARER_USER_NAME, name: 'Foundry (instance owner)' },
+                }, { fetch: runtime.fetch, now: runtime.now ?? Date.now });
+            }
+            catch (error) {
+                if (error instanceof CreateInstanceError && error.kind !== 'rejected') {
+                    await record(PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY, 'unknown');
+                }
+                throw error;
+            }
+            const { instanceId, pat } = created;
+            // Recorded before the PAT leaves this function, like the admin key below.
+            await record(PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY, instanceId);
+            const attempts = runtime.deliverAttempts ?? 3;
+            for (let attempt = 1;; attempt += 1) {
+                try {
+                    await deps.deliverSecret(PLATFORM_AUTH_INSTANCE_BEARER_REF, pat);
+                    break;
+                }
+                catch (error) {
+                    if (attempt >= attempts) {
+                        const reason = redactSecrets(error instanceof Error ? error.message : String(error), [
+                            pat,
+                        ]);
+                        throw new Error(`platform:auth: the owner token of ZITADEL instance ${instanceId} was created but could not be delivered after ${attempts} tries (${reason}); ${instanceWithoutBearerMessage(hostname, `instance ${instanceId}`)}`);
+                    }
+                    await sleep(1000 * attempt);
+                }
+            }
+            // The marker stays recorded but is ignored once this is set.
+            await record(PLATFORM_AUTH_INSTANCE_SCOPE_KEY, instanceId);
+        }
         // Minting needs a running instance behind the tunnel: the first apply
         // gets here right after the deploy.
         await waitForInstance(`https://${hostname}/.well-known/openid-configuration`, runtime, sleep);
@@ -2360,7 +2462,7 @@ function isCanonicalOrigin(value) {
  * malformed entry never reaches a property read on a non-object and throws a
  * `TypeError` instead of this named error.
  */
-function validateBrokerAppEntry(entry, brokerAppId, index) {
+export function validateBrokerAppEntry(entry, brokerAppId, index) {
     const label = `platform:archon: broker app ${brokerAppId}'s AUTH_APPS[${index}]`;
     if (typeof entry !== 'object' || entry === null || Array.isArray(entry)) {
         throw new Error(`${label} is not an object`);
@@ -3480,7 +3582,7 @@ async function findOrCreatePlatformApp(coolify, name, environment, create) {
  * `CoolifyPort.listEnvs` doc comment already warns a live value does not
  * always match its declared `string` type.
  */
-async function missingBrokerInputs(coolify, brokerAppId) {
+export async function missingBrokerInputs(coolify, brokerAppId) {
     const envs = await coolify.listEnvs(brokerAppId);
     const value = (key) => envs.find((e) => e.key === key && !e.is_preview && e.value != null && e.value !== '')?.value;
     const has = (key) => value(key) !== undefined;
@@ -3560,7 +3662,7 @@ function hashAuthApps(raw) {
  * left unchanged), so neither caller needs a second read to agree with
  * itself about what is now true of the app.
  */
-function brokerDeployNeeded(missingInputs, brokerAppId, authAppsRaw, recordedDeploy) {
+export function brokerDeployNeeded(missingInputs, brokerAppId, authAppsRaw, recordedDeploy) {
     if (missingInputs.length > 0)
         return false;
     if (recordedDeploy === null)
@@ -3575,7 +3677,7 @@ function brokerDeployNeeded(missingInputs, brokerAppId, authAppsRaw, recordedDep
  * caller's own `record()`), replacing the plain broker-id string round 2-4
  * wrote there. Paired with {@link brokerDeployNeeded}, which reads it back.
  */
-function brokerDeployRecordValue(brokerAppId, authAppsRaw) {
+export function brokerDeployRecordValue(brokerAppId, authAppsRaw) {
     const record = { brokerAppId, authAppsHash: hashAuthApps(authAppsRaw) };
     return JSON.stringify(record);
 }
@@ -3642,10 +3744,6 @@ async function ensureMasterkey(coolify, appId, databaseInUse) {
     }
     return randomBytes(24).toString('base64');
 }
-/** `docs/deploy.md`: "ZITADEL_ADMIN_PASSWORD: 8+, upper, lower, digit, symbol". */
-function generateAdminPassword() {
-    return `${randomBytes(18).toString('base64')}Aa1!`;
-}
 /**
  * #171: how the host signs a files management request with the key this step
  * delivers under {@link FILES_MANAGEMENT_KEY_REF} (files'
@@ -4388,17 +4486,384 @@ async function waitForIntegrationsMigration(coolify, appId, runtime, sleep) {
 const PLATFORM_BACKUP_S3_STORAGE_NAME = 'platform-backups';
 /** The uuid `provisionPlatformBackup` records once {@link PLATFORM_BACKUP_S3_STORAGE_NAME} is registered on Coolify, so a later apply never re-registers it. */
 const PLATFORM_BACKUP_S3_STORAGE_SCOPE_KEY = 'coolify_platform_backup_s3_storage';
+/**
+ * #257 fix round: a sha256 of the exact `endpoint|bucket|key|secret` the
+ * storage was last pushed with -- never the values themselves, the same
+ * "hash only, never the secret" rule {@link hashAuthApps} already follows
+ * in this file -- so a rotation is detected without ever writing a
+ * credential to applied state.
+ */
+const PLATFORM_BACKUP_S3_STORAGE_FINGERPRINT_SCOPE_KEY = 'coolify_platform_backup_s3_storage_fingerprint';
+/**
+ * #257 fix round 6, point 1: the uuid a freshly created
+ * {@link PLATFORM_BACKUP_S3_STORAGE_NAME} is recorded under the moment
+ * `createS3Storage` returns, persisted at once (the same "one mint" shape
+ * CLAUDE.md's platform-steps rules require of any id a step cannot safely
+ * re-mint), and cleared back to the empty string only once
+ * {@link PLATFORM_BACKUP_S3_STORAGE_SCOPE_KEY} itself is updated to match --
+ * the same "empty string means nothing pending" convention
+ * `LOGIN_HOST_PENDING_REMOVAL_SCOPE_KEY` (`login-host.ts`) already uses.
+ * Without this, a `repointPlatformBackupSchedules` failure partway through
+ * the service list left the new storage created but unrecorded; the next
+ * apply, finding the old recorded uuid still dead, created *another* one,
+ * stranding the first holding this run's R2 credentials with nothing ever
+ * pointed at it. `ensurePlatformBackupStorage` checks this before creating,
+ * so a retry reuses the same storage instead.
+ */
+const PLATFORM_BACKUP_S3_STORAGE_PENDING_SCOPE_KEY = 'coolify_platform_backup_s3_storage_pending';
 /** Coolify's own name for "once a day" -- `createDatabaseBackup`'s `frequency`. */
 const PLATFORM_BACKUP_DEFAULT_FREQUENCY = 'daily';
+/**
+ * Trims a trailing slash and lowercases the host, so a Coolify-normalised
+ * endpoint (or one a person typed with a trailing slash) is never read as a
+ * rotation. #257 fix round, point 3: the prior strict string compare treated
+ * `https://R2.example.com/` and `https://r2.example.com` as different
+ * buckets. Falls back to a plain trim-and-lowercase for a value `URL` cannot
+ * parse, rather than throwing on a malformed endpoint here.
+ */
+function normalizePlatformBackupEndpoint(endpoint) {
+    const trimmed = endpoint.trim().replace(/\/+$/, '');
+    try {
+        const url = new URL(trimmed);
+        return `${url.protocol}//${url.host.toLowerCase()}${url.pathname.replace(/\/+$/, '')}${url.search}`;
+    }
+    catch {
+        return trimmed.toLowerCase();
+    }
+}
+/**
+ * #257 fix round, point 1: sha256 of the R2 config this step would push to
+ * Coolify, normalised the same way the live/desired endpoint compare is, so
+ * a merely differently-formatted endpoint never reads as a rotated
+ * credential. Compared against the fingerprint recorded on the last apply
+ * that actually pushed this config -- never the key or secret themselves.
+ * #257 fix round 3, point 4: `region` is in the hash too -- a region-only
+ * change (R2 is `auto` today, but a host could point this at a different
+ * S3-compatible provider later) must push an update the same as a changed
+ * bucket or endpoint would, not go unnoticed because only the other four
+ * fields were compared.
+ */
+function platformBackupStorageFingerprint(r2) {
+    return createHash('sha256')
+        .update([
+        normalizePlatformBackupEndpoint(r2.endpoint),
+        r2.bucket,
+        r2.region ?? 'auto',
+        r2.accessKeyId,
+        r2.secretAccessKey,
+    ].join('|'))
+        .digest('hex');
+}
+/**
+ * #257 fix round, point 1 and 2: pushes the configured R2 bucket/endpoint/
+ * key/secret onto an existing Coolify S3 storage via `PATCH /s3-storages/
+ * {uuid}` (Coolify's own update-by-uuid route; `CoolifyPort` has no typed
+ * method for it, so this goes through `raw` the way `coolifyApp`'s
+ * stop/start calls already do for a route the port has not wrapped). The
+ * secret is sent to Coolify and nowhere else -- it never reaches `raw`'s
+ * own error path below, a log, or applied state.
+ */
+async function updatePlatformBackupS3Storage(coolify, storageUuid, r2) {
+    await coolify.raw('PATCH', `/s3-storages/${storageUuid}`, {
+        endpoint: r2.endpoint,
+        bucket: r2.bucket,
+        region: r2.region ?? 'auto',
+        key: r2.accessKeyId,
+        secret: r2.secretAccessKey,
+    });
+}
+/**
+ * #257 fix round 3, point 3, revised in fix round 4 (points 1 and 2): when
+ * {@link ensurePlatformBackupStorage} moves the shared storage to a new uuid
+ * -- a brand new registration (the recorded one was deleted by hand) or an
+ * existing one adopted under a different uuid by name -- every service's
+ * already-scheduled `save_s3` backup still carries the old uuid in
+ * `s3_storage_uuid`. Coolify dumps a database on schedule regardless, but the
+ * S3 copy step fails every night pointed at a storage that is gone or no
+ * longer the one configured. This repoints each live schedule to the new
+ * storage, the same "own live-vs-recorded compare per service" pattern
+ * {@link ProvidersPlatformCoolifyReader.listResources} already uses to find a
+ * service's own schedule in the first place.
+ *
+ * Fix round 4, point 2: a schedule is repointed only when it currently points
+ * at `oldStorageUuid` -- the caller's own recorded uuid, never "whatever the
+ * first enabled `save_s3` schedule happens to be." A hand-made schedule
+ * pointed at an unrelated storage is left alone.
+ *
+ * Fix round 4, point 1: this throws on the first failure -- a list read or a
+ * PATCH -- rather than swallowing it per service and logging. Both call
+ * sites in {@link ensurePlatformBackupStorage} record the new uuid and
+ * fingerprint only once this resolves, so a repoint failure here leaves the
+ * old uuid recorded and the next plan sees the storage as still drifted (the
+ * newly created entry is found again, by uuid or by name, and every schedule
+ * still pointed at the old uuid is retried) rather than silently leaving a
+ * schedule on a storage that no longer backs it up. A service with no
+ * recorded database is skipped: there is nothing to repoint.
+ *
+ * Fix round 5, point 3: every `save_s3` schedule on the database is
+ * checked, not just the first one Coolify happens to list -- a database
+ * could carry more than one. And a schedule whose `s3_storage_uuid` is
+ * missing is an unknown state, not a silent "doesn't match": a prior round
+ * read a missing field the same as "points elsewhere" and skipped it, which
+ * would leave a schedule nobody can tell is stale. Throwing here fails this
+ * repoint the same way a list read or PATCH failure already does, which is
+ * what keeps the storage reachable as drifted (see {@link
+ * platformBackupStorageDrifted}) rather than quietly wrong.
+ *
+ * Fix round 6, point 3: before throwing on a missing `s3_storage_uuid`, this
+ * reads the schedule's own detail route, `GET /databases/{uuid}/backups/
+ * {backup_uuid}` -- the same resource `CoolifyPort.listDatabaseBackups`'s own
+ * doc comment already flags as unconfirmed on the *list* response, and the
+ * read half of the PATCH this file already sends to the identical path. Not
+ * independently confirmed against a live Coolify instance; if the detail
+ * route also omits the field, this still throws, now naming both endpoints
+ * checked.
+ *
+ * Fix round 7, point 3: `backup.enabled` is no longer part of the skip
+ * check -- a disabled `save_s3` schedule is repointed exactly like an
+ * enabled one. Left on a dead storage, it breaks the day someone re-enables
+ * it, and "disabled" says nothing about whether it still carries the right
+ * `s3_storage_uuid`.
+ */
+async function repointPlatformBackupSchedules(coolify, recorded, oldStorageUuid, newStorageUuid) {
+    for (const service of PLATFORM_BACKUP_SERVICE_KEYS) {
+        const databaseId = recorded[PLATFORM_BACKUP_DATABASE_SCOPE_KEYS[service]];
+        if (!databaseId)
+            continue;
+        const backups = await coolify.listDatabaseBackups(databaseId);
+        for (const backup of backups) {
+            // #257 fix round 7, point 3: a disabled schedule is repointed too --
+            // left on a dead storage, it breaks the day someone re-enables it, and
+            // nothing about "disabled" means "will never run again."
+            if (!backup.save_s3)
+                continue;
+            let s3StorageUuid = backup.s3_storage_uuid;
+            if (s3StorageUuid === undefined) {
+                const detail = (await coolify.raw('GET', `/databases/${databaseId}/backups/${backup.uuid}`));
+                s3StorageUuid = detail?.s3_storage_uuid;
+            }
+            if (s3StorageUuid === undefined) {
+                throw new Error(`platform:backup-storage: ${service}'s backup ${backup.uuid} is enabled with save_s3 but has no s3_storage_uuid on either the list or the detail endpoint -- cannot tell whether it needs repointing`);
+            }
+            if (s3StorageUuid !== oldStorageUuid)
+                continue;
+            await coolify.raw('PATCH', `/databases/${databaseId}/backups/${backup.uuid}`, {
+                s3_storage_uuid: newStorageUuid,
+            });
+        }
+    }
+}
+/**
+ * #257 fix round: self-heals the shared R2 storage entry rather than
+ * trusting a recorded uuid forever, the same "stale id never blocks
+ * planning for good" rule {@link recordedZone} already follows elsewhere in
+ * this file. A recorded uuid Coolify no longer has (deleted by hand, or
+ * never recorded at all) falls through to a create, exactly as a
+ * first-ever apply does. When a uuid *was* recorded before and a new one had
+ * to be created, every service's own schedule is repointed from the old
+ * recorded uuid to the new one before either is recorded (fix round 4, point
+ * 1: `repointPlatformBackupSchedules` now throws on any failure, so a failed
+ * repoint leaves the old uuid recorded and the drift reachable again next
+ * plan -- see {@link platformBackupStorageDrifted} -- rather than recording a
+ * move that left a schedule behind). Once the uuid is settled, the entry's
+ * fingerprint decides whether anything needs pushing: unchanged, nothing
+ * happens; changed (a rotated key, secret, bucket or endpoint alike),
+ * `updatePlatformBackupS3Storage` pushes the current config onto it rather
+ * than refusing or duplicating it -- `listS3Storages` never returns a
+ * storage's secret, so the fingerprint, not a field-by-field compare, is
+ * what catches a key/secret rotation that left the bucket and endpoint
+ * untouched.
+ *
+ * #257 fix round 5, point 2: earlier rounds also looked up a storage by
+ * *name* (`PLATFORM_BACKUP_S3_STORAGE_NAME`) when the recorded uuid wasn't
+ * live, and pushed this run's R2 credentials onto whatever it found. That
+ * adopts and overwrites any storage that merely happens to share the name --
+ * not necessarily one this package ever created -- which is indistinguishable
+ * here from handing a stranger's bucket our key and secret. So this function
+ * only ever mutates a storage whose uuid is the one already recorded: when
+ * that uuid isn't live, it always creates a fresh one rather than taking an
+ * existing same-named entry over. (`platformBackupStorageDrifted`'s own
+ * by-name lookup is unaffected -- it only ever reads, to decide whether a
+ * plan is needed, never writes.)
+ *
+ * #257 fix round 6, point 1: when the recorded uuid isn't live, a pending
+ * uuid recorded under {@link PLATFORM_BACKUP_S3_STORAGE_PENDING_SCOPE_KEY} is
+ * reused if it is still live, rather than creating a second storage. Before
+ * this, a `repointPlatformBackupSchedules` failure left the just-created
+ * storage unrecorded anywhere, so the next apply's "recorded uuid isn't
+ * live" check always created another one -- one orphan per failed retry,
+ * each holding a live copy of the R2 secret pointed at by nothing. The
+ * pending uuid is persisted the moment it exists (create, or an existing
+ * pending storage adopted unchanged) and cleared only once every schedule's
+ * repoint has actually succeeded and the main key has caught up to it.
+ *
+ * #257 fix round 7, point 1: a reused pending storage is re-pushed with the
+ * current `r2` config before anything repoints onto it. Without this, a
+ * pending storage created on a failed attempt keeps whatever credential it
+ * was given then -- if the R2 secret rotates before the retry, the retry
+ * reuses the pending storage unchanged, records a fingerprint that matches
+ * the *new* secret, and Coolify is left holding the old one. Nightly
+ * backups then fail on auth with drift reporting clean.
+ *
+ * #257 fix round 7, point 2: once the recorded uuid is found live, any
+ * leftover pending uuid is cleared even though nothing else needed fixing
+ * this apply. `platformBackupStorageDrifted` treats a non-empty pending key
+ * as "self-heal incomplete" on its own, with no Coolify call -- so a stale
+ * pending uuid from an attempt that is no longer relevant (the recorded
+ * storage came back, or was restored by hand) would otherwise plan this
+ * step forever.
+ */
+async function ensurePlatformBackupStorage(coolify, r2, recorded, context) {
+    const fingerprint = platformBackupStorageFingerprint(r2);
+    const recordedUuid = recorded[PLATFORM_BACKUP_S3_STORAGE_SCOPE_KEY];
+    const pendingUuid = recorded[PLATFORM_BACKUP_S3_STORAGE_PENDING_SCOPE_KEY];
+    const recordedFingerprint = recorded[PLATFORM_BACKUP_S3_STORAGE_FINGERPRINT_SCOPE_KEY];
+    // Persists at once, so a crash between this write and the repoint below
+    // can never strand a created-but-unrecorded storage (this function's own
+    // doc comment, fix round 6 point 1).
+    const recordPending = async (id) => {
+        context.recordScopeId(PLATFORM_BACKUP_S3_STORAGE_PENDING_SCOPE_KEY, id);
+        await context.persist?.();
+    };
+    const storages = await coolify.listS3Storages();
+    const live = recordedUuid ? storages.find((storage) => storage.uuid === recordedUuid) : undefined;
+    if (!live) {
+        const pendingLive = pendingUuid
+            ? storages.find((storage) => storage.uuid === pendingUuid)
+            : undefined;
+        const target = pendingLive ??
+            (await coolify.createS3Storage({
+                name: PLATFORM_BACKUP_S3_STORAGE_NAME,
+                endpoint: r2.endpoint,
+                bucket: r2.bucket,
+                region: r2.region ?? 'auto',
+                key: r2.accessKeyId,
+                secret: r2.secretAccessKey,
+                is_usable: true,
+            }));
+        if (!pendingLive) {
+            await recordPending(target.uuid);
+        }
+        else {
+            // #257 fix round 7, point 1: a pending storage can be carrying a
+            // credential this run already left behind -- it was created (or
+            // adopted) on an earlier, failed attempt, and the R2 secret may have
+            // rotated since. Reusing it without pushing the current config would
+            // record a fingerprint that matches `r2` while Coolify still holds
+            // whatever was configured when the pending storage was first made.
+            await updatePlatformBackupS3Storage(coolify, pendingLive.uuid, r2);
+        }
+        if (recordedUuid && recordedUuid !== target.uuid) {
+            await repointPlatformBackupSchedules(coolify, recorded, recordedUuid, target.uuid);
+        }
+        context.recordScopeId(PLATFORM_BACKUP_S3_STORAGE_SCOPE_KEY, target.uuid);
+        context.recordScopeId(PLATFORM_BACKUP_S3_STORAGE_FINGERPRINT_SCOPE_KEY, fingerprint);
+        context.recordScopeId(PLATFORM_BACKUP_S3_STORAGE_PENDING_SCOPE_KEY, '');
+        return target.uuid;
+    }
+    if (recordedFingerprint !== fingerprint) {
+        await updatePlatformBackupS3Storage(coolify, live.uuid, r2);
+        context.recordScopeId(PLATFORM_BACKUP_S3_STORAGE_FINGERPRINT_SCOPE_KEY, fingerprint);
+    }
+    // #257 fix round 7, point 2: the recorded storage is live, so the
+    // self-heal this run needed (if any) is done. A pending uuid left over
+    // from an earlier, now-irrelevant attempt (the recorded uuid came back
+    // live again, or was fixed by hand) must not keep reporting drift forever
+    // -- `platformBackupStorageDrifted` treats any non-empty pending key as
+    // "incomplete self-heal" with no Coolify call, so a stale one would plan
+    // this step on every apply from here on.
+    if (pendingUuid) {
+        context.recordScopeId(PLATFORM_BACKUP_S3_STORAGE_PENDING_SCOPE_KEY, '');
+    }
+    return live.uuid;
+}
+/**
+ * #257 fix round 2: a read-only echo of {@link ensurePlatformBackupStorage}'s
+ * own live-vs-recorded compare, for {@link FoundryPlatformStepExecutor.driftedSteps}
+ * to call every plan -- never pushes anything to Coolify, just answers
+ * whether the next apply would. `planBackups` (`platform-diff.ts`) needs
+ * this because once every service's own `coolify:backup:<service>` step is
+ * live, none of them is replanned again (the same "no park branch, claimed
+ * for good" rule every other provision-only platform step here follows), so
+ * nothing would otherwise ever call {@link ensurePlatformBackupStorage}
+ * again to notice a rotated credential. True whenever the recorded uuid is
+ * missing from Coolify's live list (it would re-register) or a live entry's
+ * fingerprint no longer matches the desired one (it would push an update).
+ *
+ * #257 fix round 3, point 2, revised in fix round 4, point 4: Coolify being
+ * unreachable here must not fail the whole platform plan -- the same "a plan
+ * must not fail outright over a transient read" reason
+ * {@link authLoginClientDrifted} already follows -- but a `console.error` a
+ * person never sees hid exactly the kind of persistent failure (credentials
+ * revoked, DNS broken, R2 endpoint unreachable) this check exists to catch.
+ * So a failure here reads as "not drifted" (never plans a step doomed to
+ * fail the same read again) *and* carries a `blocked` reason the caller
+ * surfaces in the plan's own unmanaged list, the same "blocked, not failing,
+ * not silent" shape every other blocked platform step in this file already
+ * has.
+ *
+ * #257 fix round 5, point 1: also drifted when a live entry is found (by
+ * uuid or by the shared name) under a *different* uuid than the one
+ * recorded. This is what actually catches a repoint that failed partway:
+ * {@link ensurePlatformBackupStorage} only records the new uuid once every
+ * schedule's repoint has succeeded, so a failed repoint leaves the old,
+ * now-dead uuid recorded. Before this fix, if the live entry's config
+ * already matched (nothing rotated -- the storage was simply recreated
+ * under a new uuid, found again here by name), the fingerprint compare
+ * alone said "not drifted" and the stranded repoint was never retried.
+ *
+ * #257 fix round 6, point 1: also drifted whenever
+ * {@link PLATFORM_BACKUP_S3_STORAGE_PENDING_SCOPE_KEY} is non-empty, with no
+ * Coolify call needed to know it. A pending uuid means
+ * {@link ensurePlatformBackupStorage} created or adopted a storage but has
+ * not yet finished repointing every schedule onto it, so the self-heal is
+ * incomplete regardless of what the fingerprint compare below would say.
+ */
+async function platformBackupStorageDrifted(coolify, r2, recorded) {
+    const fingerprint = platformBackupStorageFingerprint(r2);
+    const recordedUuid = recorded[PLATFORM_BACKUP_S3_STORAGE_SCOPE_KEY];
+    const recordedFingerprint = recorded[PLATFORM_BACKUP_S3_STORAGE_FINGERPRINT_SCOPE_KEY];
+    if (recorded[PLATFORM_BACKUP_S3_STORAGE_PENDING_SCOPE_KEY])
+        return { drifted: true };
+    try {
+        const storages = await coolify.listS3Storages();
+        const live = (recordedUuid && storages.find((storage) => storage.uuid === recordedUuid)) ||
+            storages.find((storage) => storage.name === PLATFORM_BACKUP_S3_STORAGE_NAME);
+        if (!live)
+            return { drifted: true };
+        if (recordedUuid && live.uuid !== recordedUuid)
+            return { drifted: true };
+        return { drifted: recordedFingerprint !== fingerprint };
+    }
+    catch (e) {
+        return {
+            drifted: false,
+            blocked: `platform:backup-storage is blocked: could not check the R2 backup storage's liveness: ${e instanceof Error ? e.message : String(e)}`,
+        };
+    }
+}
+/**
+ * #257 fix round 2: the one step `planBackups` plans, independent of every
+ * per-service `coolify:backup:<service>` step's own claimed/live status,
+ * purely to keep {@link ensurePlatformBackupStorage}'s self-heal reachable
+ * once every service's own step has gone quiet. Calling it again once a
+ * per-service step has already self-healed this same apply is a safe no-op:
+ * {@link ensurePlatformBackupStorage}'s own fingerprint compare is what
+ * decides whether anything is pushed, not which step called it.
+ */
+async function provisionPlatformBackupStorage(context, deps) {
+    const { coolify, r2 } = deps;
+    const recorded = context.recordedIds();
+    await ensurePlatformBackupStorage(coolify, r2, recorded, context);
+    return undefined;
+}
 /**
  * #257: schedules a nightly R2 backup of one platform database --
  * `planBackups` (`platform-diff.ts`) plans this step once per service, each
  * carrying that service's own database id as `action.desired.databaseId`
  * (recorded by that service's own `coolify:app` step; never minted here).
  * The R2 bucket is registered as a Coolify S3 storage once, by name, and
- * its uuid recorded under {@link PLATFORM_BACKUP_S3_STORAGE_SCOPE_KEY} so
- * every service's own backup step -- and a rerun of this one -- reuses it
- * rather than registering a second storage entry under the same name.
+ * self-healed on every apply by {@link ensurePlatformBackupStorage} --
  * `createDatabaseBackup` itself is the idempotency boundary for the
  * schedule: an existing enabled, `save_s3` schedule is left alone rather
  * than replaced, since Coolify has no "upsert" for this call and a second
@@ -4408,26 +4873,7 @@ async function provisionPlatformBackup(action, context, deps) {
     const { coolify, r2 } = deps;
     const databaseId = action.desired.databaseId;
     const recorded = context.recordedIds();
-    let storageUuid = recorded[PLATFORM_BACKUP_S3_STORAGE_SCOPE_KEY];
-    if (!storageUuid) {
-        const found = (await coolify.listS3Storages()).find((storage) => storage.name === PLATFORM_BACKUP_S3_STORAGE_NAME);
-        if (found) {
-            storageUuid = found.uuid;
-        }
-        else {
-            const created = await coolify.createS3Storage({
-                name: PLATFORM_BACKUP_S3_STORAGE_NAME,
-                endpoint: r2.endpoint,
-                bucket: r2.bucket,
-                region: r2.region ?? 'auto',
-                key: r2.accessKeyId,
-                secret: r2.secretAccessKey,
-                is_usable: true,
-            });
-            storageUuid = created.uuid;
-        }
-        context.recordScopeId(PLATFORM_BACKUP_S3_STORAGE_SCOPE_KEY, storageUuid);
-    }
+    const storageUuid = await ensurePlatformBackupStorage(coolify, r2, recorded, context);
     const existing = (await coolify.listDatabaseBackups(databaseId)).find((backup) => backup.enabled && backup.save_s3);
     if (!existing) {
         await coolify.createDatabaseBackup(databaseId, {