@wtfalch/foundry 0.16.14 → 0.16.16

package/dist/plan/platform-providers.js

@@ -4,9 +4,10 @@ import { applyObjects, openInstance, platformMailSettingsObjects } from '@wtfalc
 import { VERSIONS, floatingTagBlocked } from '../bootstrap/versions.js';
 import { redactSecrets } from '../calls-sfu-deploy-support.js';
 import { ensureCloudflaredConnector } from './cloudflared-connector.js';
+import { waitForDeploymentFinished, waitForHealthyVersion, waitForNewDeployment, } from './deploy-health.js';
 import { ACCESS_POLICY_NAME, getDatabaseStarted, isNotFoundError, isRunningOrStarting, } from './executor.js';
-import { coolifyDomains } from './hostnames.js';
-import { AUTH_ZITADEL_RUN_OPTIONS, JOBS_SCHEDULER_KEY, PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY, PLATFORM_AI_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_AI_DEPLOY_PENDING_SCOPE_KEY, PLATFORM_AI_INTERNAL_URL_SCOPE_KEY, PLATFORM_AI_KEY, PLATFORM_AI_MANAGEMENT_KEY_CONFLICT_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_AI_MIGRATE_BLOCKED_SCOPE_KEY, PLATFORM_AI_ORGANISATION_SCOPE_KEY, PLATFORM_AI_RUNNER_DEPLOYMENT_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_SET_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_BROKER_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_ARCHON_CONNECTOR_APP_SCOPE_KEY, PLATFORM_ARCHON_GATE_CLOSED_SCOPE_KEY, PLATFORM_ARCHON_KEY, PLATFORM_ARCHON_MASTER_CLOUDFLARE_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_URL_REF, PLATFORM_AUTH_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY, PLATFORM_AUTH_BROKER_DEPLOYED_SCOPE_KEY, PLATFORM_AUTH_BROKER_SCOPE_KEY, PLATFORM_AUTH_CONNECTOR_APP_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_KEY, PLATFORM_AUTH_LOGIN_CLIENT_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_LOGIN_CLIENT_USER_SCOPE_KEY, PLATFORM_AUTH_OWNER_INVITE_SCOPE_KEY, PLATFORM_AUTH_SMTP_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_APPLIED_IMAGE_TAG_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_RUN_OPTIONS_SCOPE_KEY, PLATFORM_BACKUP_DATABASE_SCOPE_KEYS, PLATFORM_BACKUP_SERVICE_KEYS, PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_KEY, PLATFORM_FILES_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_FILES_MANAGEMENT_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_ZITADEL_PROJECT_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY, PLATFORM_INTEGRATIONS_KEY_SET_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_ADMIN_KEY_SCOPE_KEY, PLATFORM_MAIL_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_MAIL_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_MAIL_CONNECTOR_APP_SCOPE_KEY, PLATFORM_MAIL_INTERNAL_URL_SCOPE_KEY, PLATFORM_MAIL_KEY, PLATFORM_MAIL_ZITADEL_PROJECT_SCOPE_KEY, parseBrokerDeployRecord, parseLoginClientRecord, platformMailMxHost, } from './platform-diff.js';
+import { coolifyDomains, filesShareHostname } from './hostnames.js';
+import { AUTH_ZITADEL_RUN_OPTIONS, JOBS_SCHEDULER_KEY, PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY, PLATFORM_AI_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_AI_DEPLOY_PENDING_SCOPE_KEY, PLATFORM_AI_INTERNAL_URL_SCOPE_KEY, PLATFORM_AI_KEY, PLATFORM_AI_MANAGEMENT_KEY_CONFLICT_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_AI_MIGRATE_BLOCKED_SCOPE_KEY, PLATFORM_AI_ORGANISATION_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_SET_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_BROKER_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_OPERATOR_SCOPE_KEY, PLATFORM_ARCHON_CONNECTOR_APP_SCOPE_KEY, PLATFORM_ARCHON_GATE_CLOSED_SCOPE_KEY, PLATFORM_ARCHON_INTEGRATIONS_KEY_SYNCED_SCOPE_KEY, PLATFORM_ARCHON_KEY, PLATFORM_ARCHON_MASTER_CLOUDFLARE_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_URL_REF, PLATFORM_AUTH_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY, PLATFORM_AUTH_BROKER_DEPLOYED_SCOPE_KEY, PLATFORM_AUTH_BROKER_SCOPE_KEY, PLATFORM_AUTH_CONNECTOR_APP_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_KEY, PLATFORM_AUTH_LOGIN_CLIENT_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_LOGIN_CLIENT_USER_SCOPE_KEY, PLATFORM_AUTH_OWNER_INVITE_SCOPE_KEY, PLATFORM_AUTH_SMTP_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_APPLIED_IMAGE_TAG_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_RUN_OPTIONS_SCOPE_KEY, PLATFORM_BACKUP_DATABASE_SCOPE_KEYS, PLATFORM_BACKUP_SERVICE_KEYS, PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_KEY, PLATFORM_FILES_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_FILES_MANAGEMENT_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_ZITADEL_PROJECT_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY, PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_ROTATION_PREVIOUS_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_ROTATION_UNDELIVERED_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_SET_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_INTEGRATIONS_RUNNER_KEYRING_DEPLOYED_SCOPE_KEY, PLATFORM_MAIL_ADMIN_KEY_SCOPE_KEY, PLATFORM_MAIL_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_MAIL_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_MAIL_CONNECTOR_APP_SCOPE_KEY, PLATFORM_MAIL_INTERNAL_URL_SCOPE_KEY, PLATFORM_MAIL_KEY, PLATFORM_MAIL_ZITADEL_PROJECT_SCOPE_KEY, archonBootstrapOperator, parseBrokerDeployRecord, parseLoginClientRecord, platformMailMxHost, } from './platform-diff.js';
 import { applyCallsSfuPlatformStep } from './platform-sfu.js';
 import { CreateInstanceError, createZitadelInstance, generateSystemApiKeypair, instanceWithoutBearerMessage, systemApiUsersEnv, } from './zitadel-system-api.js';
 /** The key `runPlatformSteps` records the apex zone's id under (`system:kind:key`). */
@@ -258,7 +259,10 @@ export class FoundryPlatformStepExecutor {
             recordedIds[`coolify:app:${PLATFORM_ARCHON_KEY}`]) {
             const accessDrifted = await archonAccessDrifted(this.archon.cloudflare, platform.archon.hostname, this.archon.accessEmails);
             const brokerDrifted = !accessDrifted && (await archonBrokerRegistrationDrifted(this.archon.coolify, recordedIds));
-            if (accessDrifted || brokerDrifted)
+            const bootstrapOperatorDrifted = !accessDrifted &&
+                !brokerDrifted &&
+                (await archonBootstrapOperatorDrifted(this.archon.coolify, recordedIds[`coolify:app:${PLATFORM_ARCHON_KEY}`], recordedIds));
+            if (accessDrifted || brokerDrifted || bootstrapOperatorDrifted)
                 result.archon = true;
         }
         if (platform.auth.enabled && this.auth) {
@@ -276,7 +280,10 @@ export class FoundryPlatformStepExecutor {
         if (platform.files.enabled && this.files && recordedIds[`coolify:app:${PLATFORM_FILES_KEY}`]) {
             // A Coolify read error must not fail the whole plan: treat it as not drifted.
             try {
-                if (await filesRunnerEnvDrifted(this.files.coolify, recordedIds))
+                if (await filesRunnerEnvDrifted(this.files.coolify, recordedIds, {
+                    publicHosts: this.files.publicHosts,
+                    shareDomains: filesShareDomains(await this.declaredOrgStore.list()),
+                }))
                     result.files = true;
             }
             catch {
@@ -297,6 +304,25 @@ export class FoundryPlatformStepExecutor {
         }
         return result;
     }
+    /**
+     * Gives the files runner a changed share-domain set and nothing else: its
+     * Coolify `domains`, `FILES_PUBLIC_HOSTS` and one redeploy of the runner.
+     * Returns the scope ids to record, or `null` when nothing changed
+     * (or the runner was never applied).
+     */
+    async refreshFilesShare(_platform, recordedIds) {
+        const runnerAppId = recordedIds[`coolify:app:${PLATFORM_FILES_KEY}`];
+        if (!this.files || !runnerAppId)
+            return null;
+        const wanted = filesShareDomains(await this.declaredOrgStore.list());
+        if (!shareDomainsChanged(recordedIds, wanted))
+            return null;
+        const { coolify } = this.files;
+        await coolify.updateApplication(runnerAppId, filesShareDomainsPatch(recordedIds, wanted));
+        await coolify.bulkUpsertEnvs(runnerAppId, filesPublicHostsEnvs(recordedIds, this.files.publicHosts, wanted));
+        await coolify.deploy(runnerAppId);
+        return { [PLATFORM_FILES_SHARE_DOMAINS_SCOPE_KEY]: normalHosts(wanted).join(',') };
+    }
     async execute(action, context) {
         if (action.system === 'coolify' && action.kind === 'app' && action.key === PLATFORM_AUTH_KEY) {
             if (!this.auth) {
@@ -309,7 +335,7 @@ export class FoundryPlatformStepExecutor {
             if (!this.files) {
                 throw new Error('platform:files needs FoundryPlatformStepExecutor to be constructed with its files dependencies (#171)');
             }
-            return provisionPlatformFiles(context, this.files);
+            return provisionPlatformFiles(context, this.files, filesShareDomains(await this.declaredOrgStore.list()));
         }
         if (action.system === 'coolify' &&
             action.kind === 'app' &&
@@ -550,7 +576,11 @@ const AUTH_BROKER_PORT = '3000';
  * falls through to the login app's own ingress rule below (login v2 answers
  * everything else, including the root).
  */
-const AUTH_ZITADEL_PATH_PATTERN = '^/(oidc|oauth|saml|management|admin|system|auth|assets|debug/healthz|ui/console|\\.well-known)(/|$)';
+// `v2`, `v2beta` and `zitadel.<service>` are ZITADEL's v2 REST and connect
+// APIs (providers' `inviteOrgOwner` calls `POST /v2/users`); without them those
+// calls reach the login app and answer 404. The login app itself lives under
+// `/ui/v2/login`, which none of these match.
+export const AUTH_ZITADEL_PATH_PATTERN = '^/(oidc|oauth|saml|management|admin|system|auth|assets|debug/healthz|ui/console|\\.well-known|v2|v2beta|zitadel\\.[A-Za-z0-9.]+)(/|$)';
 /**
  * `@wtfalch/auth`'s own `brokerUrl` default, `<issuer>/api`
  * (`packages/auth/src/config.ts`) -- `deploy/proxy/Caddyfile`'s own
@@ -1057,6 +1087,15 @@ async function provisionPlatformAuth(context, deps, owners = []) {
         },
         { key: 'OTEL_NODE_RESOURCE_DETECTORS', value: 'none' },
         { key: 'API_CACHE_CONFIG', value: '{"defaultMinutes":2,"longMinutes":5}' },
+        // #405: the login app's own requestId-free fallback (`resolveRedirectUri`
+        // in `wtfalch/auth`'s `login/src/lib/client.ts`) is what a user without an
+        // OIDC request in flight lands on after `/signedin` -- exactly the owner
+        // invite's own path (set a password, then nowhere to go). Only set once
+        // Archon is declared, and only ever to Archon: the platform's one landing
+        // app for anyone who reaches the login app outside an app's own sign-in.
+        ...(platform.archon.enabled
+            ? [{ key: 'DEFAULT_REDIRECT_URI', value: `https://${platform.archon.hostname}` }]
+            : []),
     ];
     // #210: read back now, not inside the deploy-decision block below -- this
     // push has to land before `missingBrokerInputs` is read a few lines down,
@@ -1423,8 +1462,12 @@ async function provisionPlatformAuth(context, deps, owners = []) {
             });
             await record(PLATFORM_AUTH_SMTP_SCOPE_KEY, smtp.id);
         }
-        catch {
-            throw new Error(`platform:auth: could not configure ZITADEL's SMTP provider for ${hostname}; the cause is left out because it can carry the Lettermint key`);
+        catch (error) {
+            // The cause, with the Lettermint key redacted, so a failure says why.
+            const cause = redactSecrets(error instanceof Error ? error.message : String(error), [
+                mailToken,
+            ]);
+            throw new Error(`platform:auth: could not configure ZITADEL's SMTP provider for ${hostname}: ${cause}`);
         }
         if (!context.recordedIds()[PLATFORM_AUTH_OWNER_INVITE_SCOPE_KEY]) {
             const owner = owners[0];
@@ -1438,14 +1481,22 @@ async function provisionPlatformAuth(context, deps, owners = []) {
                 await record(PLATFORM_ZITADEL_ORG_SCOPE_KEY, ownerOrgId);
                 let invitedUserId;
                 try {
+                    // ZITADEL's default invite link is `/ui/v2/login/verify`, which 404s on the login app;
+                    // it serves `/verify` at the root.
                     ({ userId: invitedUserId } = await mailZitadel.inviteOrgOwner(ownerOrgId, {
                         givenName: owner.givenName,
                         familyName: owner.familyName,
                         email: owner.email,
-                    }));
+                    }, `https://${hostname}/verify?userId={{.UserID}}&code={{.Code}}&organization={{.OrgID}}&invite=true`));
                 }
-                catch {
-                    throw new Error(`platform:auth: could not invite the platform owner into ZITADEL org ${ownerOrgId}; the cause is left out because it can carry the owner's email`);
+                catch (error) {
+                    // The cause, with the owner's email and the Lettermint key redacted, so a failure says why.
+                    const cause = redactSecrets(error instanceof Error ? error.message : String(error), [
+                        owner.email,
+                        owner.email.toLowerCase(),
+                        mailToken,
+                    ]);
+                    throw new Error(`platform:auth: could not invite the platform owner into ZITADEL org ${ownerOrgId}: ${cause}`);
                 }
                 await record(PLATFORM_AUTH_OWNER_INVITE_SCOPE_KEY, invitedUserId);
             }
@@ -1777,7 +1828,8 @@ async function provisionPlatformMail(context, deps) {
     const declaredChanged = recorded[PLATFORM_MAIL_DEPLOYED_HOSTNAME_SCOPE_KEY] !== hostname ||
         recorded[PLATFORM_MAIL_DEPLOYED_GIT_REF_SCOPE_KEY] !== imageTag;
     // A new storage binds into the container only at its next start.
-    if (app.justCreated || !app.running || envsChanged || declaredChanged || storagesAdded) {
+    const mailDeployedNow = app.justCreated || !app.running || envsChanged || declaredChanged || storagesAdded;
+    if (mailDeployedNow) {
         // A container still up from before would answer the readiness probe with
         // the old deployment's health. Stop it and see it stopped first, so the
         // wait below only ever reads the new deployment.
@@ -1785,7 +1837,16 @@ async function provisionPlatformMail(context, deps) {
             await deps.coolify.raw('POST', `/applications/${appId}/stop`);
             await waitForMailStopped(deps.coolify, appId, runtime, sleep);
         }
-        await deps.coolify.deploy(appId);
+        // #252: phase 1 only -- Coolify's own deployment record reaching
+        // `'finished'`. Phase 2 (the readiness probe, `ensureMailReady` below)
+        // runs right after the tunnel and DNS wiring that follows, which is what
+        // `hostname` needs before it answers anything at all; running it here
+        // instead would just poll a hostname that cannot resolve yet on a first
+        // apply. Either way it runs before any applied ref is recorded.
+        // The old container was stopped above (or never ran), so nothing stays live.
+        await waitForDeploymentFinished(deps.coolify, appId, runtime, 'platform:mail', app.justCreated || !app.running
+            ? `mail stays at ${recorded[PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY] ?? 'no recorded ref'}`
+            : 'mail is stopped until the next apply');
         await record(PLATFORM_MAIL_DEPLOYED_SCOPE_KEY, appId);
         // Right after the deploy, so a later failure (tunnel, DNS, readiness,
         // settings, mint) does not make the next run stop and redeploy the app.
@@ -1859,6 +1920,31 @@ async function provisionPlatformMail(context, deps) {
             proxied: false,
         });
     }
+    // #252: phase 2, now that the tunnel and DNS above actually route
+    // `hostname`. `'ready'`, not `'version'`: Stalwart's `/healthz/ready`
+    // carries no version field. Run once per apply, whichever of the three
+    // things below needs it first -- a redeploy just above, the OIDC-audience
+    // re-apply right after, or the admin-key mint further down, none of which
+    // can reach an instance that is not up yet either. Combined with phase 1
+    // above having already confirmed Coolify's own deployment record
+    // finished, this is what actually closes the loophole named in #252 (an
+    // old container staying healthy while the deployment itself fails) -- not
+    // this probe alone, which a container that merely kept answering could
+    // still pass.
+    let mailReadyConfirmed = false;
+    const ensureMailReady = async () => {
+        if (mailReadyConfirmed)
+            return;
+        await waitForHealthyVersion(appId, { kind: 'ready', url: `https://${hostname}${STALWART_READY_PATH}` }, runtime, sleep, 'platform:mail', `mail stays at ${recorded[PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY] ?? 'no recorded ref'}`);
+        mailReadyConfirmed = true;
+    };
+    // Also probe when the applied ref lags the target: the deployed refs are
+    // recorded before this probe, so a probe that timed out on the last run must
+    // run again, not be skipped because the deploy itself is already recorded.
+    const appliedLagsTarget = recorded[PLATFORM_MAIL_APPLIED_HOSTNAME_SCOPE_KEY] !== hostname ||
+        recorded[PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY] !== imageTag;
+    if (mailDeployedNow || appliedLagsTarget)
+        await ensureMailReady();
     // ---- the mail ZITADEL project, in the platform's sign-in -------------------
     // #346: found or created on every run, like the files/Archon steps' own
     // projects -- its id is the OIDC audience Stalwart's directory checks an
@@ -1884,7 +1970,7 @@ async function provisionPlatformMail(context, deps) {
         // run to retry. A project deleted out of band is noticed only here, on
         // the next plan/apply -- the same way the files/Archon steps' own
         // projects are.
-        await waitForMailInstance(`https://${hostname}${STALWART_READY_PATH}`, runtime, sleep);
+        await ensureMailReady();
         const recoveryEnv = (await deps.coolify.listEnvs(appId)).find((e) => e.key === 'STALWART_RECOVERY_ADMIN' && !e.is_preview);
         if (!recoveryEnv) {
             throw new Error('platform:mail: STALWART_RECOVERY_ADMIN is not set on the app; cannot authenticate to re-apply the OIDC audience');
@@ -1924,7 +2010,7 @@ async function provisionPlatformMail(context, deps) {
         if (lost) {
             throw new Error(`platform:mail: an admin API key ("${lost}") was minted but never delivered; revoke it in Stalwart and remove the recorded key ${PLATFORM_MAIL_ADMIN_UNDELIVERED_SCOPE_KEY} to mint a new one`);
         }
-        await waitForMailInstance(`https://${hostname}${STALWART_READY_PATH}`, runtime, sleep);
+        await ensureMailReady();
         const recoveryEnv = (await deps.coolify.listEnvs(appId)).find((e) => e.key === 'STALWART_RECOVERY_ADMIN' && !e.is_preview);
         if (!recoveryEnv) {
             throw new Error('platform:mail: STALWART_RECOVERY_ADMIN is not set on the app; cannot authenticate to mint the admin key');
@@ -2304,9 +2390,6 @@ export const PLATFORM_ARCHON_CLIENT_ID_SCOPE_KEY = 'zitadel_platform_archon_clie
  * needed so the two apps agree the moment the broker can run.
  *
  * Known gaps, left for review:
- * - `AUTHZ_BOOTSTRAP_OPERATOR`: the first operator's ZITADEL subject id. A
- *   person has to exist in the new instance first; not something foundry
- *   mints.
  * - Foundry's own database URL. Archon runs foundry in its panel, but foundry
  *   does not know its own connection string at plan/apply time: that is host
  *   config, not something `DeclaredPlatform` or held keys carry.
@@ -2333,6 +2416,8 @@ async function provisionPlatformArchon(context, deps) {
     const recorded = context.recordedIds();
     const issuer = `https://${platform.auth.hostname}`;
     const appliedGitRef = recorded[PLATFORM_ARCHON_APPLIED_GIT_REF_SCOPE_KEY];
+    const runtime = deps.runtime ?? {};
+    const sleep = runtime.sleep ?? defaultSleep;
     if (deps.accessEmails.length === 0) {
         throw new Error('platform:archon: no Access emails given; refusing to put Archon behind an Access app nobody can pass');
     }
@@ -2523,6 +2608,24 @@ async function provisionPlatformArchon(context, deps) {
         throw new Error(`platform:archon: app ${appId} lost its AUTH_APP_KEY between two reads`);
     }
     envs.push({ key: 'AUTH_APP_KEY', value: appKey });
+    // The owner `platform:auth` invited is Archon's first operator.
+    const bootstrapOperator = archonBootstrapOperator(recorded);
+    if (bootstrapOperator)
+        envs.push({ key: 'AUTHZ_BOOTSTRAP_OPERATOR', value: bootstrapOperator });
+    // #406: the invited platform owner's ZITADEL subject id, once
+    // `platform:auth` has recorded one -- the guard at the top of this
+    // function already requires `platform:auth` to have been applied before
+    // this step is reached, and `provisionPlatformAuth` records the owner
+    // invite (or, with no owner declared, the literal string 'none') as part
+    // of that same step, so by the time this runs the value here is whatever
+    // is actually true. `bootstrapOperator` (archon's `src/lib/authz/
+    // bootstrap.ts`) compares this verbatim against a signed-in user's
+    // subject id, so 'none' is skipped rather than written -- it would simply
+    // never match, but it is not a real id and should not look like one.
+    const ownerInviteUserId = recorded[PLATFORM_AUTH_OWNER_INVITE_SCOPE_KEY];
+    if (ownerInviteUserId && ownerInviteUserId !== 'none') {
+        envs.push({ key: 'AUTHZ_BOOTSTRAP_OPERATOR', value: ownerInviteUserId });
+    }
     // A value with a line break (the PEM management keys) must be flagged:
     // unflagged, Coolify writes it into `.env` unquoted and the container never
     // starts.
@@ -2631,12 +2734,32 @@ async function provisionPlatformArchon(context, deps) {
         await record(PLATFORM_ARCHON_ACCESS_SCOPE_KEY, accessAppId);
     }
     // ---- deploy ---------------------------------------------------------------
-    if (app.justCreated ||
+    // #252: phase 1 of the shared deploy-health helper only -- Coolify's own
+    // deployment record reaching `'finished'`, which is enough on its own to
+    // catch the bug this closes (the old container stays healthy regardless
+    // of whether the deployment record itself fails). Phase 2, the health
+    // check, has to wait until after the tunnel and DNS below are wired: on a
+    // first apply `hostname` answers nothing until then, and splitting the two
+    // phases (`waitForDeploymentFinished` then `waitForHealthyVersion`,
+    // instead of one `waitForNewDeployment` call) is exactly for a step shaped
+    // like this one.
+    // #398: the integrations key id this run wrote into Archon's env. Archon
+    // counts as synced to it only after a deployment finishes and the health
+    // check passes, so a still-unsynced id forces both even when nothing else
+    // changed (a run that died between the env write and the deploy).
+    const integrationsKeyId = inputs.envs.find((e) => e.key === 'INTEGRATIONS_MANAGEMENT_KEY_ID')?.value;
+    // Not forced on a platform recorded before #398, which has no recorded id.
+    const keyUnsynced = Boolean(integrationsKeyId) &&
+        Boolean(recorded[PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY]) &&
+        recorded[PLATFORM_ARCHON_INTEGRATIONS_KEY_SYNCED_SCOPE_KEY] !== integrationsKeyId;
+    const archonDeployedNow = keyUnsynced ||
+        app.justCreated ||
         !app.running ||
         changed ||
         appliedGitRef !== gitRef ||
-        recorded[PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY] !== hostname) {
-        await deps.coolify.deploy(appId);
+        recorded[PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY] !== hostname;
+    if (archonDeployedNow) {
+        await waitForDeploymentFinished(deps.coolify, appId, runtime, 'platform:archon', `archon stays at ${appliedGitRef ?? 'no recorded ref'} on ${recorded[PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY] ?? 'no recorded hostname'}`);
     }
     // ---- tunnel and DNS ---------------------------------------------------------
     const existingTunnel = await deps.cloudflare.findTunnelByName(PLATFORM_ARCHON_TUNNEL_NAME);
@@ -2690,6 +2813,24 @@ async function provisionPlatformArchon(context, deps) {
             proxied: true,
         });
     }
+    // #252: phase 2, now that the tunnel and DNS above actually route
+    // `hostname`. `'ready'`, not `'version'`: archon's `/api/health`
+    // (`@wtfalch/reporting`'s `healthHandler`, the same route `executor.ts`'s
+    // `healthUrls` already assumes every hosted app serves) carries no
+    // version field this package knows to check, so a plain 2xx is the mode
+    // available -- combined with phase 1 above having already confirmed
+    // Coolify's own deployment record finished, which is what actually closes
+    // the loophole named in #252 (an old container staying healthy while the
+    // deployment itself fails).
+    if (archonDeployedNow) {
+        await waitForHealthyVersion(appId, { kind: 'ready', url: `https://${hostname}/api/health` }, runtime, sleep, 'platform:archon', `archon stays at ${appliedGitRef ?? 'no recorded ref'} on ${recorded[PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY] ?? 'no recorded hostname'}`);
+    }
+    // #398: the deployment finished and answers health, so Archon now runs with
+    // the key id its env carries. `provisionPlatformIntegrations` drops the
+    // previous public key only on this marker.
+    if (keyUnsynced && integrationsKeyId) {
+        await record(PLATFORM_ARCHON_INTEGRATIONS_KEY_SYNCED_SCOPE_KEY, integrationsKeyId);
+    }
     // Last, after every write the applied values depend on, so a run that failed
     // halfway plans again.
     await record(PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY, hostname);
@@ -2699,6 +2840,7 @@ async function provisionPlatformArchon(context, deps) {
     // (whose AUTH_APPS never carried this registration) from one still holding
     // it.
     await record(PLATFORM_ARCHON_APPLIED_BROKER_SCOPE_KEY, brokerAppId);
+    await record(PLATFORM_ARCHON_APPLIED_OPERATOR_SCOPE_KEY, bootstrapOperator ?? 'none');
     return { resourceId: appId };
 }
 // #211 round 6: mirrors `wtfalch/auth/broker/src/apps.ts`'s own `isValidFrom`
@@ -3074,6 +3216,32 @@ async function archonBrokerRegistrationDrifted(coolify, recordedIds) {
         return false;
     }
 }
+/**
+ * #406: whether the platform owner invite recorded under
+ * {@link PLATFORM_AUTH_OWNER_INVITE_SCOPE_KEY} is a real user id (not
+ * absent, not the literal `'none'` {@link provisionPlatformAuth} records
+ * when no owner is declared) that Archon's live env does not yet carry as
+ * `AUTHZ_BOOTSTRAP_OPERATOR`. `provisionPlatformAuth`'s own invite can
+ * record after Archon's `coolify:app` id already has -- the owner invite
+ * happens inside `platform:auth` only once its Lettermint token is held,
+ * which Archon's own apply does not wait on -- so without this, an owner
+ * invited after Archon's first deploy would need a redeploy by hand, the
+ * way companyincluded.com's did. A failed read is not drift, the same rule
+ * {@link archonAccessDrifted} and {@link archonBrokerRegistrationDrifted}
+ * follow: the next plan reads again.
+ */
+async function archonBootstrapOperatorDrifted(coolify, appId, recordedIds) {
+    const invitedUserId = recordedIds[PLATFORM_AUTH_OWNER_INVITE_SCOPE_KEY];
+    if (!appId || !invitedUserId || invitedUserId === 'none')
+        return false;
+    try {
+        const live = await currentEnvValue(coolify, appId, 'AUTHZ_BOOTSTRAP_OPERATOR');
+        return live !== invitedUserId;
+    }
+    catch {
+        return false;
+    }
+}
 /**
  * What Archon reads from each enabled service, gathered before anything is
  * built. Either everything is there and the env values come back, or the first
@@ -3091,6 +3259,23 @@ async function collectArchonServiceInputs(platform, deps, recorded) {
     if (services.length > 0 && !deps.heldKeys) {
         return block(`the host built the platform executor without a held-key reader, and ${services.join(', ')} ${services.length === 1 ? 'is' : 'are'} enabled`);
     }
+    const rotationUndelivered = recorded[PLATFORM_INTEGRATIONS_KEY_ROTATION_UNDELIVERED_SCOPE_KEY];
+    if (services.includes('integrations') &&
+        rotationUndelivered &&
+        rotationUndelivered !== recorded[PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY]) {
+        // #398: the held key may already be the new pair while the recorded id is
+        // still the old one; writing both would sign with a key id the runner
+        // does not match.
+        return block(`the integrations rotation key ${rotationUndelivered} is minted but not confirmed delivered and pinned`);
+    }
+    const activeKeyId = recorded[PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY];
+    if (services.includes('integrations') &&
+        activeKeyId &&
+        recorded[PLATFORM_INTEGRATIONS_RUNNER_KEYRING_DEPLOYED_SCOPE_KEY] !== activeKeyId) {
+        // #398: the runner must run with the new public key before Archon signs
+        // with the new private key.
+        return block(`the integrations runner is not yet deployed with management key ${activeKeyId}`);
+    }
     for (const service of services) {
         const needs = service === 'jobs'
             ? {
@@ -3154,7 +3339,10 @@ async function collectArchonServiceInputs(platform, deps, recorded) {
                 value: INTEGRATIONS_MANAGEMENT_ISSUER,
             }, {
                 key: 'INTEGRATIONS_MANAGEMENT_KEY_ID',
-                value: INTEGRATIONS_MANAGEMENT_KEY_ID,
+                // #398: the active id, which a rotation moves past `foundry-1` --
+                // falls back to the fixed constant for a platform that applied
+                // integrations before this scope key existed.
+                value: recorded[PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY] ?? INTEGRATIONS_MANAGEMENT_KEY_ID,
             }, { key: 'INTEGRATIONS_MANAGEMENT_AUDIENCE', value: url });
         }
         else {
@@ -3703,9 +3891,15 @@ async function provisionPlatformAi(context, deps) {
             await coolify.raw('POST', `/applications/${migrate.appId}/stop`);
             await waitForStopped(coolify, migrate.appId, runtime, sleep);
         }
-        await coolify.deploy(migrate.appId);
+        const migrateDeployed = await coolify.deploy(migrate.appId);
         try {
             await waitForAiMigration(coolify, migrate.appId, runtime, sleep);
+            // #399: the migrate job can read healthy well before Coolify's own
+            // deployment does -- it is still waiting out the image's HEALTHCHECK
+            // start period -- so stopping it below waits for Coolify's own record
+            // too, or it can find the container already gone and mark a deployment
+            // that actually migrated as `failed`.
+            await waitForMigrateDeployment(coolify, migrate.appId, queuedDeploymentUuid(migrateDeployed) ?? 'untracked', 'platform:ai', 'ai-migrate', runtime.now ?? Date.now, runtime.migrateTimeoutMs ?? 600_000, runtime.migrateIntervalMs ?? 5_000, sleep);
         }
         catch (error) {
             if (error instanceof MigrationNotHealthyError) {
@@ -3718,49 +3912,44 @@ async function provisionPlatformAi(context, deps) {
         }
         await coolify.raw('POST', `/applications/${migrate.appId}/stop`);
     }
-    // `deploy` only queues a build that takes minutes, so the id of the host's
-    // deployment is kept for the mint below to wait on.
-    const runnerDeployment = deployRunner ? await coolify.deploy(runner.appId) : undefined;
+    // #252: confirmed before anything below -- the applied ref recorded just
+    // after, and the mint further down -- depends on the new version actually
+    // being live. Never read the old container's healthy status as the new
+    // deployment's (`CLAUDE.md`'s platform-step rule 5): `'deployment-only'`,
+    // because the runner's own doc comment above says why there is no url to
+    // check instead ("Nothing public is wired: no Coolify `domains`, tunnel,
+    // DNS record or Access app"). This replaces the old deploy-then-defer-the-
+    // wait-until-the-mint-needs-it shape (#375) with the one every other
+    // platform step uses: the wait happens once, right here, so a failed
+    // deployment is caught before either thing downstream trusts it.
     if (deployRunner) {
-        await record(PLATFORM_AI_RUNNER_DEPLOYMENT_SCOPE_KEY, queuedDeploymentUuid(runnerDeployment) ?? 'untracked');
+        await waitForNewDeployment(coolify, runner.appId, { kind: 'deployment-only' }, runtime, sleep, 'platform:ai', `the runner stays at ${recorded[PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY] ?? 'no recorded ref'}`);
+    }
+    if (deployWorker) {
+        await waitForNewDeployment(coolify, worker.appId, { kind: 'deployment-only' }, runtime, sleep, 'platform:ai', `the worker stays at ${recorded[PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY] ?? 'no recorded ref'}`);
     }
-    if (deployWorker)
-        await coolify.deploy(worker.appId);
     await record(PLATFORM_AI_DEPLOY_PENDING_SCOPE_KEY, '');
-    // The deploys are done, so record what they deployed now. A later failure
-    // at the mint or delivery must not make the next apply see a changed
-    // hostname or gitRef and redeploy the live host again. The planner still
-    // plans the step until the resource id is recorded, once the key is delivered.
+    // The deploys are confirmed, so record what they deployed now. A later
+    // failure at the mint or delivery must not make the next apply see a
+    // changed hostname or gitRef and redeploy the live host again. The
+    // planner still plans the step until the resource id is recorded, once
+    // the key is delivered.
     await record(PLATFORM_AI_APPLIED_HOSTNAME_SCOPE_KEY, hostname);
     await record(PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY, gitRef);
     const internalUrl = `http://${PLATFORM_AI_RUNNER.internalName}:${PLATFORM_AI_PORT}`;
     await record(PLATFORM_AI_INTERNAL_URL_SCOPE_KEY, internalUrl);
     // ---- Archon's AI management key, once ------------------------------------
+    // #252: nothing left to wait on here -- the deployment above is already
+    // confirmed finished by the time this runs. The host's own readiness
+    // beyond that (a 503 for a moment after `AI_WORKER_TIMEOUT_MS` warms up)
+    // is unchanged from before this change: a single mint attempt, not
+    // retried here.
     const now = context.recordedIds();
     if (!now[PLATFORM_AI_MANAGEMENT_KEY_SCOPE_KEY]) {
         const lost = now[PLATFORM_AI_MANAGEMENT_KEY_UNDELIVERED_SCOPE_KEY];
         if (lost) {
             throw new Error(`platform:ai: AI management key ${lost} was minted but never delivered; revoke it and remove the recorded key ${PLATFORM_AI_MANAGEMENT_KEY_UNDELIVERED_SCOPE_KEY} to mint a new one`);
         }
-        // The host answers 503 (or nothing) until the deployment that carries its
-        // management settings has finished.
-        const queued = context.recordedIds()[PLATFORM_AI_RUNNER_DEPLOYMENT_SCOPE_KEY];
-        if (queued) {
-            try {
-                await waitForAiDeployment(coolify, runner.appId, queued, runtime, sleep);
-            }
-            catch (error) {
-                // Only a build still running is worth waiting on again. A deployment
-                // that failed, was cancelled or cannot be read is not: mark the host
-                // pending so the next apply queues a new one instead of polling it.
-                if (!(error instanceof AiDeploymentTimeoutError)) {
-                    await record(PLATFORM_AI_DEPLOY_PENDING_SCOPE_KEY, '1');
-                    await record(PLATFORM_AI_RUNNER_DEPLOYMENT_SCOPE_KEY, '');
-                }
-                throw error;
-            }
-            await record(PLATFORM_AI_RUNNER_DEPLOYMENT_SCOPE_KEY, '');
-        }
         // #201: a test injects its own minter; without one, build the real one
         // over ai's own POST /v1/platform/keys, signed with the key pinned above.
         const minter = deps.managementKeyMinter ??
@@ -3900,22 +4089,23 @@ function queuedDeploymentUuid(result) {
     return typeof uuid === 'string' && uuid ? uuid : undefined;
 }
 /**
- * Waits until the host's queued deployment has `finished`; throws on
- * `failed`/`cancelled-by-user` or once `migrateTimeoutMs` has passed. With no
- * deployment id (`untracked`) the only signal left is the app's own status,
- * which an old container can already satisfy, so that is best effort.
+ * Waits until `deploymentUuid`'s Coolify deployment has `finished`; throws on
+ * `failed`/`cancelled-by-user` (via `onEnded`) or once `timeoutMs` has passed
+ * (via `onTimeout`). With no deployment id (`untracked`) the only signal left
+ * is the app's own status, which an old container can already satisfy, so
+ * that case is best effort and returns as soon as the app itself reads
+ * healthy. Shared by every caller that must not act on an app's status
+ * until Coolify's own deployment record agrees it is done (#399: Coolify's
+ * own `HEALTHCHECK` start period can outlive the app already reading
+ * `running:healthy`; acting then finds the container already gone, and
+ * Coolify marks its own deployment `failed` -- "Removing the new version" --
+ * though whatever the app was doing had already finished).
  */
-/** The host's deployment was still running when the wait gave up. */
-class AiDeploymentTimeoutError extends Error {
-}
-async function waitForAiDeployment(coolify, appId, deploymentUuid, runtime, sleep) {
-    const now = runtime.now ?? Date.now;
-    const timeout = runtime.migrateTimeoutMs ?? 600_000;
-    const interval = runtime.migrateIntervalMs ?? 5_000;
-    const deadline = now() + timeout;
+async function waitForCoolifyDeployment(coolify, appId, deploymentUuid, now, timeoutMs, intervalMs, sleep, onEnded, onTimeout) {
+    const deadline = now() + timeoutMs;
     let status;
     for (;;) {
-        await sleep(interval);
+        await sleep(intervalMs);
         if (deploymentUuid === 'untracked') {
             status = (await coolify.getApplication(appId)).status;
             if (status?.startsWith('running:healthy'))
@@ -3926,15 +4116,25 @@ async function waitForAiDeployment(coolify, appId, deploymentUuid, runtime, slee
             status = typeof found?.status === 'string' ? found.status : undefined;
             if (status === 'finished')
                 return;
-            if (status === 'failed' || status === 'cancelled-by-user') {
-                throw new Error(`platform:ai: the host's deployment for ${appId} ended ${status}; the AI management key was not minted. Check its build log in Coolify, then apply again`);
-            }
-        }
-        if (now() >= deadline) {
-            throw new AiDeploymentTimeoutError(`platform:ai: the host's deployment for ${appId} did not finish within ${Math.round(timeout / 1000)}s (last status ${status ?? 'unknown'}); the AI management key was not minted. Apply again`);
+            if (status === 'failed' || status === 'cancelled-by-user')
+                throw onEnded(status);
         }
+        if (now() >= deadline)
+            throw onTimeout(status ?? 'unknown');
     }
 }
+/**
+ * Waits until a migrate job's own queued deployment has `finished`, so stopping a migrate job never races Coolify's own
+ * `HEALTHCHECK` start period (#399: every platform migrate job idles rather
+ * than exits once its migration succeeds, so it reports `running:healthy`
+ * well before Coolify's own deployment does, and stopping it on that signal
+ * alone can make Coolify mark a deployment that actually migrated `failed`).
+ * `label` and `jobName` build an error naming the step and the job, never
+ * just "the deployment".
+ */
+async function waitForMigrateDeployment(coolify, appId, deploymentUuid, label, jobName, now, timeoutMs, intervalMs, sleep) {
+    await waitForCoolifyDeployment(coolify, appId, deploymentUuid, now, timeoutMs, intervalMs, sleep, (status) => new Error(`${label}: ${jobName} (${appId})'s deployment ended ${status}; check its build log in Coolify, then apply again`), (status) => new Error(`${label}: ${jobName} (${appId})'s deployment did not finish within ${Math.round(timeoutMs / 1000)}s (last status ${status}); apply again`));
+}
 /**
  * Why a mint failed, safe to show: the status ai answered (the minter's own
  * fixed message), or a network error's `name` and `cause.code`. Never a
@@ -4390,7 +4590,9 @@ const PLATFORM_FILES_RUNNER_SCOPE_KEY = 'coolify_platform_files_runner_app';
  *   Coolify network, so the host must run there to use it for management
  *   calls.
  */
-async function provisionPlatformFiles(context, deps) {
+async function provisionPlatformFiles(context, deps, 
+/** {@link filesShareDomains} of every declared org: each is a runner domain and a public host. */
+shareDomains) {
     const { platform } = context;
     const gitRef = platform.files.gitRef;
     const recorded = context.recordedIds();
@@ -4492,7 +4694,11 @@ async function provisionPlatformFiles(context, deps) {
         dockerfile_target_build: 'runner',
         custom_internal_name: FILES_RUNNER_INTERNAL_NAME,
         git_branch: gitRef,
+        // Coolify wants full URLs. Written only when the set changed, so a runner
+        // nobody gave a share host keeps no `domains`.
+        ...filesShareDomainsPatch(recorded, shareDomains),
     });
+    context.recordScopeId(PLATFORM_FILES_SHARE_DOMAINS_SCOPE_KEY, normalHosts(shareDomains).join(','));
     // ---- database env ----------------------------------------------------------
     // The migrate app sets `files_rt`'s password from DATABASE_URL on every
     // boot (files' `scripts/db-runtime-role.sh`), and the runner connects with
@@ -4543,11 +4749,7 @@ async function provisionPlatformFiles(context, deps) {
                 value: deps.clamd.port,
             });
     }
-    if (deps.publicHosts)
-        runnerOptionalEnvs.push({
-            key: 'FILES_PUBLIC_HOSTS',
-            value: deps.publicHosts,
-        });
+    runnerOptionalEnvs.push(...filesPublicHostsEnvs(recorded, deps.publicHosts, shareDomains));
     if (deps.jobsSigningSecrets)
         runnerOptionalEnvs.push({
             key: 'JOBS_SIGNING_SECRETS',
@@ -4628,10 +4830,22 @@ async function provisionPlatformFiles(context, deps) {
     // deploy. Stopped first, so the wait below cannot pass on the old
     // container before the new migration has run.
     await stopIfRunning(deps.coolify, migrateApp.appId);
-    await deps.coolify.deploy(migrateApp.appId);
+    const migrateDeployed = await deps.coolify.deploy(migrateApp.appId);
     await waitForMigration(deps.coolify, migrateApp.appId, runtime, sleep);
+    // #399: files-migrate idles rather than exits once it has migrated, so it
+    // reads healthy well before Coolify's own deployment does; stopping it on
+    // that signal alone can find the container already gone and make Coolify
+    // mark a deployment that actually migrated as `failed`.
+    await waitForMigrateDeployment(deps.coolify, migrateApp.appId, queuedDeploymentUuid(migrateDeployed) ?? 'untracked', 'platform:files', 'files-migrate', runtime.now ?? Date.now, runtime.migrateTimeoutMs ?? 900_000, runtime.migrateIntervalMs ?? 10_000, sleep);
     await deps.coolify.raw('POST', `/applications/${migrateApp.appId}/stop`);
-    await deps.coolify.deploy(runnerApp.appId);
+    // #252: `'deployment-only'` -- the runner's own doc comment above says why
+    // ("The runner is created without `domains`... Nothing public is wired"):
+    // there is no url this executor, running outside files' own Docker
+    // network, could ever reach, so `coolify.deployAndWait` reaching Coolify's
+    // own deployment record is the whole check. Still enough to catch the bug
+    // this closes: the applied ref recorded below while the *old* container
+    // (never read here) stayed healthy through a failed deploy.
+    await waitForNewDeployment(deps.coolify, runnerApp.appId, { kind: 'deployment-only' }, runtime, sleep, 'platform:files', `the runner stays at ${recorded[PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY] ?? 'no recorded ref'}`);
     await record(PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, `http://${FILES_RUNNER_INTERNAL_NAME}:${FILES_PORT}`);
     // Last: a run that failed anywhere above is planned again.
     context.recordScopeId(PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, platform.files.hostname);
@@ -4708,11 +4922,21 @@ function filesDepsInvalid(deps) {
  * nothing ever re-checked whether its env could actually pass a health
  * check.
  */
-async function filesRunnerEnvDrifted(coolify, recordedIds) {
+async function filesRunnerEnvDrifted(coolify, recordedIds, hosts = {}) {
     const runnerAppId = recordedIds[`coolify:app:${PLATFORM_FILES_KEY}`];
     if (!runnerAppId)
         return false;
-    const present = new Set((await coolify.listEnvs(runnerAppId)).filter((e) => !e.is_preview && e.value).map((e) => e.key));
+    // The share set the runner was last given differs from the one now wanted.
+    if (shareDomainsChanged(recordedIds, hosts.shareDomains))
+        return true;
+    const live = (await coolify.listEnvs(runnerAppId)).filter((e) => !e.is_preview);
+    const envs = live.filter((e) => e.value);
+    const present = new Set(envs.map((e) => e.key));
+    // An unset or blank live value equals an empty wanted one.
+    const wantedHosts = filesPublicHostsValue(hosts.publicHosts, hosts.shareDomains);
+    if ((envs.find((e) => e.key === 'FILES_PUBLIC_HOSTS')?.value ?? '') !== wantedHosts) {
+        return true;
+    }
     const required = [
         'R2_BUCKET',
         'R2_ENDPOINT',
@@ -4876,11 +5100,19 @@ export const INTEGRATIONS_MANAGEMENT_KEY_ID = 'foundry-1';
  * 3. The migrate app first, once per `gitRef`: stop it if running, deploy it,
  *    wait until it reports healthy, stop it, and record the ref it migrated.
  *    Only then are the runner and worker deployed.
- * 4. The management key, once: an Ed25519 pair minted here, its id recorded
- *    before delivery, its private half handed to
- *    {@link PlatformIntegrationsDeps.deliverSecret} and its public half pinned
- *    on the runner as `MANAGEMENT_PUBLIC_KEYS`, with `MANAGEMENT_ISSUER`. The
- *    guard is recorded after both.
+ * 4. The management key, once, and rotated on request (#398): an Ed25519
+ *    pair minted here, its id recorded before delivery, its private half
+ *    handed to {@link PlatformIntegrationsDeps.deliverSecret} and its public
+ *    half pinned on the runner as `MANAGEMENT_PUBLIC_KEYS`, with
+ *    `MANAGEMENT_ISSUER`. The guard is recorded after both.
+ *    A rotation, requested by recording
+ *    {@link PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY}, mints a second
+ *    pair under a new key id the same way, pinned *beside* the first rather
+ *    than replacing it; the first is dropped only once
+ *    {@link PLATFORM_ARCHON_INTEGRATIONS_KEY_SYNCED_SCOPE_KEY} shows Archon
+ *    holding the new id. Archon reads the active id back from
+ *    {@link PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY}, so a rotation reaches it
+ *    without this step touching Archon directly.
  * 5. The applied `hostname` and `gitRef` are recorded last. A later change
  *    plans the step again (`planIntegrations`), which patches `git_branch`
  *    and the runner's `domains`, reruns the migration for a new ref, and
@@ -5053,21 +5285,28 @@ async function provisionPlatformIntegrations(action, context, deps) {
         // container's healthy status is never read as the new deployment's.
         if (migrateApp.running)
             await coolify.raw('POST', `/applications/${migrateApp.appId}/stop`);
-        await coolify.deploy(migrateApp.appId);
+        const migrateDeployed = await coolify.deploy(migrateApp.appId);
         await waitForIntegrationsMigration(coolify, migrateApp.appId, runtime, sleep);
+        // #399: integrations-migrate idles rather than exits once it has
+        // migrated (the Dockerfile's own comment, below), so it reads healthy
+        // well before Coolify's own deployment does; stopping it on that signal
+        // alone can find the container already gone and make Coolify mark a
+        // deployment that actually migrated as `failed`.
+        await waitForMigrateDeployment(coolify, migrateApp.appId, queuedDeploymentUuid(migrateDeployed) ?? 'untracked', 'platform:integrations', 'integrations-migrate', runtime.now ?? Date.now, runtime.migrateTimeoutMs ?? 20 * 60_000, runtime.migratePollMs ?? 10_000, sleep);
         // It idles after migrating rather than exiting (the Dockerfile's own
         // comment), so it is stopped here; a person starts it again for
         // maintenance commands.
         await coolify.raw('POST', `/applications/${migrateApp.appId}/stop`);
         await record(INTEGRATIONS_MIGRATED_SCOPE_KEY, gitRef);
     }
-    // ---- the management key, once --------------------------------------------------
+    // ---- the management key, once, and rotated on request (#398) -------------
     // The pair's id is recorded and persisted before the private half is
     // delivered. The guard is recorded only after delivery and the write of the
     // public half. A run that dies in between leaves the id without the guard:
     // the planner then reports the key and mints nothing, so a second pair can
     // never replace one the host may already hold.
     let keyJustSet = false;
+    let rotated = false;
     if (!context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_SET_SCOPE_KEY]) {
         if (context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY]) {
             throw new Error(`platform:integrations: management key ${INTEGRATIONS_MANAGEMENT_KEY_ID} was minted but not confirmed delivered and pinned; not minting another. Check the held key and the runner's MANAGEMENT_PUBLIC_KEYS, then remove the recorded key ${PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY} to mint a new one`);
@@ -5077,20 +5316,7 @@ async function provisionPlatformIntegrations(action, context, deps) {
             privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
         });
         await record(PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY, INTEGRATIONS_MANAGEMENT_KEY_ID);
-        const attempts = runtime.deliverAttempts ?? 3;
-        for (let attempt = 1;; attempt += 1) {
-            try {
-                await deps.deliverSecret(INTEGRATIONS_MANAGEMENT_KEY_REF, privateKey);
-                break;
-            }
-            catch {
-                // Retried; the error is dropped in case it echoes the key.
-                if (attempt >= attempts) {
-                    throw new Error(`platform:integrations: management key ${INTEGRATIONS_MANAGEMENT_KEY_ID} was minted but could not be delivered after ${attempts} attempts`);
-                }
-                await sleep(1000 * attempt);
-            }
-        }
+        await deliverManagementKey(deps, runtime, sleep, privateKey, INTEGRATIONS_MANAGEMENT_KEY_ID);
         await coolify.bulkUpsertEnvs(runnerApp.appId, [
             { key: 'MANAGEMENT_ISSUER', value: INTEGRATIONS_MANAGEMENT_ISSUER },
             {
@@ -5098,28 +5324,190 @@ async function provisionPlatformIntegrations(action, context, deps) {
                 value: JSON.stringify({ [INTEGRATIONS_MANAGEMENT_KEY_ID]: publicKey }),
             },
         ]);
+        await record(PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY, INTEGRATIONS_MANAGEMENT_KEY_ID);
         await record(PLATFORM_INTEGRATIONS_KEY_SET_SCOPE_KEY, runnerApp.appId);
         keyJustSet = true;
     }
+    else if (context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY] &&
+        !context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ROTATION_PREVIOUS_SCOPE_KEY]) {
+        // A rotation, requested by the recorded scope id
+        // `PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY`.
+        // Mints a new pair under the next key id and pins it *beside* the
+        // current one -- never replacing it outright, so a request signed with
+        // the current key during the overlap window still verifies.
+        // `rotationUndelivered` is never cleared on success (the once-only
+        // mint's own `UNDELIVERED` key never is either) -- a completed rotation
+        // leaves it equal to the now-active id, which is what lets the *next*
+        // rotation request read "no attempt in flight" without this step ever
+        // clearing it. It differs from the active id only while a mint is
+        // unconfirmed: minted and recorded, but not yet delivered and pinned.
+        const previousKeyId = context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY] ??
+            INTEGRATIONS_MANAGEMENT_KEY_ID;
+        const rotationUndelivered = context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ROTATION_UNDELIVERED_SCOPE_KEY];
+        if (rotationUndelivered && rotationUndelivered !== previousKeyId) {
+            throw new Error(`platform:integrations: rotation key ${rotationUndelivered} was minted but not confirmed delivered and pinned; not minting another. Check the held key and the runner's MANAGEMENT_PUBLIC_KEYS, then remove the recorded key ${PLATFORM_INTEGRATIONS_KEY_ROTATION_UNDELIVERED_SCOPE_KEY} and set ${PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY} again to mint a new one`);
+        }
+        const keyring = await currentManagementPublicKeys(coolify, runnerApp.appId);
+        const nextKeyId = nextManagementKeyId(previousKeyId, keyring);
+        const { publicKey, privateKey } = generateKeyPairSync('ed25519', {
+            publicKeyEncoding: { type: 'spki', format: 'pem' },
+            privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
+        });
+        await record(PLATFORM_INTEGRATIONS_KEY_ROTATION_UNDELIVERED_SCOPE_KEY, nextKeyId);
+        await deliverManagementKey(deps, runtime, sleep, privateKey, nextKeyId);
+        keyring[nextKeyId] = publicKey;
+        await coolify.bulkUpsertEnvs(runnerApp.appId, [
+            { key: 'MANAGEMENT_PUBLIC_KEYS', value: JSON.stringify(keyring) },
+        ]);
+        // The previous id and the new active id go down in one persist: a crash
+        // can never leave the new id active with the previous one unrecorded, or
+        // the previous one recorded while still active.
+        context.recordScopeId(PLATFORM_INTEGRATIONS_KEY_ROTATION_PREVIOUS_SCOPE_KEY, previousKeyId);
+        context.recordScopeId(PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY, nextKeyId);
+        await context.persist?.();
+        // One request, one rotation: cleared only now the new key is delivered
+        // and pinned. A failure before this leaves the request set and the
+        // undelivered id ahead of the active id, which the planner reports as
+        // blocked and the next run refuses to mint past.
+        await record(PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY, '');
+        rotated = true;
+        keyJustSet = true;
+    }
+    // A request that was not honoured this run (no key minted before it, or an
+    // earlier rotation still waiting to drop its previous key) is cleared, not
+    // kept to fire later; the plan item says so.
+    if (!rotated && context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY]) {
+        await record(PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY, '');
+    }
+    // ---- drop the previous key once Archon is confirmed on the new one (#398) -
+    // Dropping it any earlier would refuse a request signed by an Archon
+    // container that is still running the previous key while its own redeploy
+    // catches up.
+    const rotationPrevious = context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ROTATION_PREVIOUS_SCOPE_KEY];
+    const activeKeyId = context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY];
+    if (rotationPrevious && rotationPrevious === activeKeyId) {
+        // Corrupt record: the previous id is the active one. Never delete it.
+        await record(PLATFORM_INTEGRATIONS_KEY_ROTATION_PREVIOUS_SCOPE_KEY, '');
+    }
+    else if (rotationPrevious &&
+        // The last delivered mint (the held key) is the active id.
+        context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ROTATION_UNDELIVERED_SCOPE_KEY] ===
+            activeKeyId &&
+        // No Archon to wait for when it is not enabled: nothing else holds the key.
+        (!context.platform.archon.enabled ||
+            context.recordedIds()[PLATFORM_ARCHON_INTEGRATIONS_KEY_SYNCED_SCOPE_KEY] === activeKeyId)) {
+        const keyring = await currentManagementPublicKeys(coolify, runnerApp.appId);
+        // The active key must be pinned before any other is removed. The planner
+        // cannot see the live env, so a missing active key is an error here, not
+        // a silent no-op on every apply.
+        if (!activeKeyId || !keyring[activeKeyId]) {
+            throw new Error(`platform:integrations: the runner's MANAGEMENT_PUBLIC_KEYS does not hold the active key ${activeKeyId ?? ''}; not removing ${rotationPrevious}. Pin the active public key on the runner, then apply again`);
+        }
+        delete keyring[rotationPrevious];
+        await coolify.bulkUpsertEnvs(runnerApp.appId, [
+            { key: 'MANAGEMENT_PUBLIC_KEYS', value: JSON.stringify(keyring) },
+        ]);
+        // One persist: the previous id is gone and the runner is marked as not
+        // yet deployed with this keyring, so a failed deploy below is retried by
+        // the next plan (`runnerKeyStale`) until the live runner stops accepting
+        // the old key.
+        context.recordScopeId(PLATFORM_INTEGRATIONS_KEY_ROTATION_PREVIOUS_SCOPE_KEY, '');
+        context.recordScopeId(PLATFORM_INTEGRATIONS_RUNNER_KEYRING_DEPLOYED_SCOPE_KEY, '');
+        await context.persist?.();
+        keyJustSet = true;
+    }
+    const activeKeyForRunner = context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY];
+    const runnerKeyStale = Boolean(activeKeyForRunner) &&
+        context.recordedIds()[PLATFORM_INTEGRATIONS_RUNNER_KEYRING_DEPLOYED_SCOPE_KEY] !==
+            activeKeyForRunner;
     // ---- then the runner and the worker ------------------------------------------------
     // Decided from the live state: an env change only reaches a container on its
     // next deploy, and a stopped runner is deployed so an earlier failed deploy
     // is retried. The worker exits after one pass by design, so "not running"
     // says nothing about it and it is deployed only when created or changed.
+    // #252: `'deployment-only'` -- like `coolify:jobs`, the runner is reached
+    // through its Coolify `domains` binding, not a tunnel/DNS record this
+    // executor set up itself, and no Cloudflare wiring is created for
+    // `hostname` (this function's own doc comment, "Known gaps"). With no
+    // reachability confirmed from here, `coolify.deployAndWait` reaching
+    // Coolify's own deployment record is the whole check -- still enough to
+    // catch the bug this closes: the applied ref recorded below while the
+    // *old* container (never read here) stayed healthy through a failed
+    // deploy.
     if (runnerApp.justCreated ||
         !runnerApp.running ||
         runnerEnvsChanged ||
         keyJustSet ||
+        runnerKeyStale ||
         configChanged) {
-        await coolify.deploy(runnerApp.appId);
+        await waitForNewDeployment(coolify, runnerApp.appId, { kind: 'deployment-only' }, runtime, sleep, 'platform:integrations', `the runner stays at ${recorded[PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY] ?? 'no recorded ref'}`);
+    }
+    // #398: the runner is up with the keyring that holds the active id; only
+    // now may Archon move to it.
+    if (runnerKeyStale && activeKeyForRunner) {
+        await record(PLATFORM_INTEGRATIONS_RUNNER_KEYRING_DEPLOYED_SCOPE_KEY, activeKeyForRunner);
     }
     if (workerApp.justCreated || workerEnvsChanged || configChanged) {
-        await coolify.deploy(workerApp.appId);
+        await waitForNewDeployment(coolify, workerApp.appId, { kind: 'deployment-only' }, runtime, sleep, 'platform:integrations', `the worker stays at ${recorded[PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY] ?? 'no recorded ref'}`);
     }
     await record(PLATFORM_INTEGRATIONS_APPLIED_HOSTNAME_SCOPE_KEY, hostname);
     await record(PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY, gitRef);
     return { resourceId: runnerApp.appId };
 }
+/**
+ * #398: `deliverSecret` with the once-only mint's own retry discipline,
+ * shared with the rotation path so both mint the same way. The error never
+ * carries the key, only its id.
+ */
+async function deliverManagementKey(deps, runtime, sleep, privateKey, keyId) {
+    const attempts = runtime.deliverAttempts ?? 3;
+    for (let attempt = 1;; attempt += 1) {
+        try {
+            await deps.deliverSecret(INTEGRATIONS_MANAGEMENT_KEY_REF, privateKey);
+            return;
+        }
+        catch {
+            // Retried; the error is dropped in case it echoes the key.
+            if (attempt >= attempts) {
+                throw new Error(`platform:integrations: management key ${keyId} was minted but could not be delivered after ${attempts} attempts`);
+            }
+            await sleep(1000 * attempt);
+        }
+    }
+}
+/**
+ * #398: the runner's live `MANAGEMENT_PUBLIC_KEYS`, parsed -- `{}` when the
+ * env is absent (no key minted yet). The rotation path reads this rather
+ * than keeping its own copy, so it edits whatever the runner actually holds,
+ * including a key id a person pinned by hand.
+ */
+async function currentManagementPublicKeys(coolify, runnerAppId) {
+    const raw = (await coolify.listEnvs(runnerAppId)).find((e) => e.key === 'MANAGEMENT_PUBLIC_KEYS' && !e.is_preview)?.value;
+    if (!raw)
+        return {};
+    try {
+        return JSON.parse(raw);
+    }
+    catch {
+        throw new Error("platform:integrations: the runner's MANAGEMENT_PUBLIC_KEYS is not valid JSON; not touching the keyring");
+    }
+}
+/**
+ * #398: the key id a rotation mints next -- `foundry-1` to `foundry-2`, and
+ * so on, by incrementing the trailing number. A current id with no trailing
+ * number (hand-pinned, never by this step) gets `-2` appended rather than
+ * refusing: the host's keyring only requires `^[A-Za-z0-9_-]{1,100}$`, and
+ * this never collides with a number the current id does not already carry.
+ */
+function nextManagementKeyId(currentKeyId, keyring) {
+    const match = /^(.*-)(\d+)$/.exec(currentKeyId);
+    const prefix = match ? match[1] : `${currentKeyId}-`;
+    let n = match ? Number(match[2]) + 1 : 2;
+    // Never overwrite an id already pinned on the runner.
+    while (keyring[`${prefix}${n}`])
+        n += 1;
+    return `${prefix}${n}`;
+}
 /**
  * The runtime `DATABASE_URL` already on the migrate app, or a new one:
  * {@link INTEGRATIONS_RUNTIME_ROLE} with a password minted once from a
@@ -5759,3 +6147,50 @@ const ARCHON_SERVICE_INPUTS = {
         refs: [INTEGRATIONS_MANAGEMENT_KEY_REF],
     },
 };
+/** Scope id (no `:`): the share domains the runner's Coolify `domains` was last set to, comma-joined. */
+export const PLATFORM_FILES_SHARE_DOMAINS_SCOPE_KEY = 'platform_files_share_domains';
+/** Lowercased, trimmed, deduplicated and sorted, so the same set always gives the same value. */
+function normalHosts(hosts) {
+    return [...new Set((hosts ?? []).map((h) => h.trim().toLowerCase()).filter(Boolean))].sort();
+}
+/** The runner's Coolify `domains`, written only when the share set changed (a runner nobody gave a share host keeps none). */
+function filesShareDomainsPatch(recorded, shareDomains) {
+    return shareDomainsChanged(recorded, shareDomains)
+        ? { domains: coolifyDomains(normalHosts(shareDomains)) }
+        : {};
+}
+/**
+ * The runner's `FILES_PUBLIC_HOSTS` env. Always written when an earlier apply
+ * recorded a share set: files reads an empty value as no public host
+ * (`filesPublicHosts()`), so an emptied set blanks the old value (the port has
+ * no env delete). A runner never given a host keeps no such env.
+ */
+function filesPublicHostsEnvs(recorded, publicHosts, shareDomains) {
+    const value = filesPublicHostsValue(publicHosts, shareDomains);
+    return value || recorded[PLATFORM_FILES_SHARE_DOMAINS_SCOPE_KEY]
+        ? [{ key: 'FILES_PUBLIC_HOSTS', value }]
+        : [];
+}
+function shareDomainsChanged(recorded, shareDomains) {
+    return ((recorded[PLATFORM_FILES_SHARE_DOMAINS_SCOPE_KEY] ?? '') !== normalHosts(shareDomains).join(','));
+}
+/**
+ * The runner's `FILES_PUBLIC_HOSTS`: the host-supplied `publicHosts` first, in
+ * their given order, then the share domains sorted, with no host twice.
+ * Empty when there is neither.
+ */
+export function filesPublicHostsValue(publicHosts, shareDomains) {
+    const given = (publicHosts ?? '')
+        .split(',')
+        .map((h) => h.trim())
+        .filter(Boolean);
+    return [...new Set([...given, ...normalHosts(shareDomains)])].join(',');
+}
+/**
+ * The share domain of every org that has a domain and an entry declaring
+ * `uses.files.shareHost`: `share.<domain>`. The platform files step derives it
+ * from the declared orgs, for its apply and its drift check alike.
+ */
+export function filesShareDomains(orgs) {
+    return normalHosts(orgs.map((org) => filesShareHostname(org) ?? ''));
+}