@wtfalch/foundry 0.16.16 → 0.17.1

package/dist/plan/plan-apply.js

@@ -1,16 +1,38 @@
-import { createHash } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 import { VERSIONS, floatingTagBlocked } from '../bootstrap/versions.js';
 import { validateHosting } from '../org.js';
 import { PLATFORM_SERVICE_KEYS } from '../platform.js';
+import { AlreadyRunningError, StepFailedError } from '../runner/errors.js';
+import { StaleLeaseError, } from '../store.js';
+import { AdoptRefusedError, adoptCandidates, adoptedKeysUnreadable, adoptedScopeKey, parseAdoptSnapshot, planAdoptFrom, } from './adopt.js';
 import { NO_AGORA_HOST_VALUES, readAgoraHost, } from './agora.js';
 import { AI_KEY_UNDELIVERED_SCOPE_KEY, JOBS_REGISTRATION_UNDELIVERED_SCOPE_KEY, computePlanDetail, toPlan, } from './diff.js';
+import { edgeWorkerTraceScopeKeys, liveEdgeWorkers } from './edge-worker.js';
 import { entryAiUndeliveredScopeKey } from './entry-ai-key.js';
 import { FILES_ROOT_UNDELIVERED_SCOPE_KEY, entryFilesUndeliveredScopeKey, } from './entry-files-key.js';
+import { entryNamesRetiredScopeKey, entryNamesUndeliveredScopeKey } from './entry-names-key.js';
 import { readLivePlatformResources, readLiveResources, } from './live.js';
 import { lockedHostnames } from './locked-hostnames.js';
-import { PLATFORM_ARCHON_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_KEY, planPlatform as computePlatformPlanDetail, toPlatformPlan, } from './platform-diff.js';
+import { PLATFORM_ARCHON_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_KEY, appStepKey, planPlatform as computePlatformPlanDetail, toPlatformPlan, } from './platform-diff.js';
 import { DEFAULT_ORG_PLATFORM_READINESS, ORG_PLATFORM_KEY_REFS, agoraHostValuesFromPlatform, heldPlatformKeyRefs, orgPlatformReadinessFrom, } from './platform-org.js';
 import { PLATFORM_AUTH_BROKER_MAIL_KEY_REF } from './platform-providers.js';
+/** #441: the lease scope that serializes every mutation of one company, and the one for the shared platform. */
+const companyScope = (orgId) => `company:${orgId}`;
+const PLATFORM_SCOPE = 'platform';
+/** A lease nobody renews: long enough for one apply, short enough that a dead worker's scope frees itself. The fence stops a slow worker that outlives it. */
+const DEFAULT_LEASE_TTL_MS = 30 * 60 * 1000;
+/** #441: another apply, destroy, adopt or release holds the lease for this company or the platform. Nothing ran; try again when it finishes. */
+export class MutationInProgressError extends Error {
+    scope;
+    heldUntil;
+    /** @param heldUntil when the holder's lease runs out if it is never released. */
+    constructor(scope, heldUntil) {
+        super(`another change to ${scope} is in progress (lease until ${heldUntil.toISOString()}); try again when it finishes`);
+        this.scope = scope;
+        this.heldUntil = heldUntil;
+        this.name = 'MutationInProgressError';
+    }
+}
 export class DeclaredOrgNotFoundError extends Error {
     constructor(orgId) {
         super(`no declared org ${orgId}`);
@@ -77,6 +99,9 @@ const HOSTNAME_STEPS = new Set([
     'cloudflare:tunnel',
     'cloudflare:access',
     'jobs:registration',
+    // #318 (F8): the route is itself hostname-shaped, validated against the
+    // org's own domain by `validateEntryEdge`.
+    'cloudflare:worker',
 ]);
 /**
  * The #19 implementation of {@link PlanApply}: computes a plan against live
@@ -96,11 +121,15 @@ export class FoundryPlanApply {
     jobsScheduler;
     loginHostBindings;
     broker;
+    edgeWorkers;
     hostFiles;
     hostAi;
+    hostNames;
     now;
+    leaseTtlMs;
     appliedVersions;
     platformDeps;
+    runner;
     constructor(options) {
         this.store = options.store;
         this.readers = options.readers;
@@ -113,17 +142,202 @@ export class FoundryPlanApply {
         this.jobsScheduler = options.jobsScheduler ?? false;
         this.loginHostBindings = options.loginHostBindings ?? false;
         this.broker = options.broker ?? false;
+        this.edgeWorkers = options.edgeWorkers ?? false;
         this.hostFiles = options.filesOrganisation ?? false;
         this.hostAi = options.aiOrganisation ?? false;
+        this.hostNames = options.names ?? false;
         this.now = options.now ?? (() => new Date());
+        this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
         this.appliedVersions = options.appliedVersions;
         this.platformDeps = options.platform;
+        this.runner = options.runner;
     }
     async plan(id) {
         const { plan } = await this.computeFreshPlan(id);
         await this.audit.recordPlan({ actor: this.actor, orgId: id, plan, at: this.now() });
         return plan;
     }
+    /**
+     * Archon#271: what adopting a live app and database into entry
+     * `input.entryKey` would do. A read, like {@link plan}: it reads the project
+     * through the reader and changes nothing. A mismatch is reported in the
+     * result, with no steps. Needs the apply permission, since it reads a
+     * project the caller names.
+     */
+    async planAdopt(id, input) {
+        if (!(await this.authz.canApply(this.actor, id)))
+            throw new NotAuthorizedError(this.actor, id);
+        const { adoptPlan } = await this.computeAdopt(id, input);
+        await this.audit.recordPlan({
+            actor: this.actor,
+            orgId: id,
+            plan: adoptPlan.plan,
+            at: this.now(),
+        });
+        return adoptPlan;
+    }
+    /**
+     * Archon#271: the live apps and databases in `projectUuid` this org has not
+     * recorded, for Changes' "Live, not theirs". A read, but it needs the apply
+     * permission: it reads a project the caller names.
+     */
+    async adoptCandidates(id, entryKey, projectUuid) {
+        if (!(await this.authz.canApply(this.actor, id)))
+            throw new NotAuthorizedError(this.actor, id);
+        const org = await this.store.get(id);
+        if (!org)
+            throw new DeclaredOrgNotFoundError(id);
+        const recorded = (await this.store.getAppliedState(id))?.serviceIds ?? {};
+        const project = projectUuid ?? recorded.coolify_project;
+        const live = project ? ((await this.readers.coolify.listProject?.(project)) ?? []) : [];
+        return adoptCandidates(org, entryKey, recorded, live);
+    }
+    /**
+     * Archon#271: records an app and database that already run in Coolify as
+     * entry `input.entryKey`'s resources, so the next plan finds them and plans
+     * no create. Verifies first and refuses on a mismatch or an entry that is
+     * already applied or held by another org. Same permission as {@link apply}.
+     * It never calls an executor: nothing in Coolify changes. Nothing redeploys
+     * here, but the next normal plan may propose updates to the adopted app
+     * that do (see `adopt.ts`); show that plan before applying it. Two
+     * concurrent adopts of one resource are not locked against each other.
+     */
+    async adopt(id, input) {
+        const refuse = async (reason) => {
+            await this.audit.recordAdoption({
+                actor: this.actor,
+                orgId: id,
+                action: 'adopt',
+                entryKey: input.entryKey,
+                outcome: 'refused',
+                reason,
+                at: this.now(),
+            });
+        };
+        if (!(await this.authz.canApply(this.actor, id))) {
+            await refuse('not authorized');
+            throw new NotAuthorizedError(this.actor, id);
+        }
+        return this.withMutationLease(companyScope(id), refuse, (fence) => this.withRunnerSerialization(id, refuse, async () => {
+            const { adoptPlan, previous } = await this.computeAdopt(id, input);
+            const { snapshot } = adoptPlan;
+            if (adoptPlan.mismatches.length > 0 || !snapshot) {
+                await refuse(adoptPlan.mismatches.join('; '));
+                throw new AdoptRefusedError(adoptPlan.mismatches);
+            }
+            const org = await this.store.get(id);
+            if (!org)
+                throw new DeclaredOrgNotFoundError(id);
+            const serviceIds = {
+                ...(previous?.serviceIds ?? {}),
+                coolify_project: snapshot.projectUuid,
+                [`coolify:app:${appStepKey(org, input.entryKey)}`]: snapshot.application.uuid,
+                [`coolify:database:${id}`]: snapshot.database.uuid,
+                [adoptedScopeKey(input.entryKey)]: JSON.stringify(snapshot),
+            };
+            const applied = {
+                orgId: id,
+                serviceIds,
+                checkpoints: previous?.checkpoints ?? [],
+                failedFields: previous?.failedFields ?? [],
+                lastApplied: previous?.lastApplied ?? null,
+            };
+            await this.store.putAppliedState(applied, fence);
+            await this.audit.recordAdoption({
+                actor: this.actor,
+                orgId: id,
+                action: 'adopt',
+                entryKey: input.entryKey,
+                outcome: 'adopted',
+                at: this.now(),
+            });
+            return applied;
+        }));
+    }
+    /**
+     * Archon#271: the undo of {@link adopt}. Drops the entry's recorded Coolify
+     * ids and its `adopted.<entryKey>` snapshot (and `coolify_project` when
+     * adopt recorded it and no other Coolify id remains). It leaves the
+     * declaration alone: the entry goes back to declared and not applied, so the
+     * next plan creates it, exactly as before adopt. The host then adopts it
+     * again or removes the entry from the declaration. Deletes and stops nothing
+     * in Coolify. Only an adopted entry can be released. Same permission as
+     * {@link apply}. Every path writes an audit row.
+     */
+    async release(id, entryKey) {
+        const record = (outcome, reason) => this.audit.recordAdoption({
+            actor: this.actor,
+            orgId: id,
+            action: 'release',
+            entryKey,
+            outcome,
+            ...(reason === undefined ? {} : { reason }),
+            at: this.now(),
+        });
+        if (!(await this.authz.canApply(this.actor, id))) {
+            await record('refused', 'not authorized');
+            throw new NotAuthorizedError(this.actor, id);
+        }
+        return this.withMutationLease(companyScope(id), (reason) => record('refused', reason), (fence) => this.withRunnerSerialization(id, (reason) => record('refused', reason), async () => {
+            const org = await this.store.get(id);
+            if (!org) {
+                await record('refused', `no declared org ${id}`);
+                throw new DeclaredOrgNotFoundError(id);
+            }
+            const previous = await this.store.getAppliedState(id);
+            const snapshot = parseAdoptSnapshot(previous?.serviceIds[adoptedScopeKey(entryKey)]);
+            if (!previous || !snapshot) {
+                await record('refused', `entry ${entryKey} was not adopted`);
+                throw new AdoptRefusedError([`entry ${entryKey} was not adopted`]);
+            }
+            const dropped = new Set([
+                adoptedScopeKey(entryKey),
+                `coolify:app:${appStepKey(org, entryKey)}`,
+                `coolify:database:${id}`,
+            ]);
+            const kept = Object.entries(previous.serviceIds).filter(([k]) => !dropped.has(k));
+            const otherCoolify = kept.some(([k]) => k.startsWith('coolify:'));
+            const serviceIds = Object.fromEntries(kept.filter(([k]) => !(k === 'coolify_project' && snapshot.projectRecorded && !otherCoolify)));
+            const released = { ...previous, serviceIds };
+            try {
+                await this.store.putAppliedState(released, fence);
+            }
+            catch (e) {
+                if (e instanceof StaleLeaseError)
+                    throw e;
+                await record('refused', e instanceof Error ? e.message : String(e));
+                throw e;
+            }
+            await record('released');
+            return released;
+        }));
+    }
+    async computeAdopt(id, input) {
+        const org = await this.store.get(id);
+        if (!org)
+            throw new DeclaredOrgNotFoundError(id);
+        const previous = await this.store.getAppliedState(id);
+        const recorded = previous?.serviceIds ?? {};
+        const project = input.projectUuid ?? recorded.coolify_project;
+        const reader = this.readers.coolify;
+        const live = project && reader.listProject ? await reader.listProject(project) : null;
+        const claimedElsewhere = [];
+        for (const [what, uuid] of [
+            ['application', input.applicationUuid],
+            ['database', input.databaseUuid],
+            ['project', project],
+        ]) {
+            if (!uuid)
+                continue;
+            const owner = await this.store.findOtherOrgRecordingCoolifyId(uuid, id);
+            if (owner)
+                claimedElsewhere.push(`${what} ${uuid} is already recorded by org ${owner}`);
+        }
+        return {
+            adoptPlan: planAdoptFrom(org, recorded, input, live, this.now(), claimedElsewhere),
+            previous,
+        };
+    }
     async apply(plan, confirmedOrgName) {
         const orgId = plan.orgId;
         const authorized = await this.authz.canApply(this.actor, orgId);
@@ -131,18 +345,21 @@ export class FoundryPlanApply {
             await this.refuse(plan, 'not authorized');
             throw new NotAuthorizedError(this.actor, orgId);
         }
-        const { org, plan: freshPlan, detail } = await this.computeFreshPlan(orgId);
-        if (!plansMatch(plan, freshPlan)) {
-            await this.refuse(plan, 'stale: live state has moved since this plan was computed');
-            throw new StalePlanError(orgId);
-        }
-        if (planNeedsConfirmation(plan.steps) && confirmedOrgName.trim() !== org.displayName.trim()) {
-            await this.refuse(plan, "destructive or migrating step without the org's name typed correctly");
-            throw new ConfirmationRequiredError(orgId);
-        }
-        const applied = await this.runSteps(org, plan, detail);
-        await this.refreshPlatformFilesShare();
-        return applied;
+        return this.withMutationLease(companyScope(orgId), (reason) => this.refuse(plan, reason), (fence) => this.withRunnerSerialization(orgId, (reason) => this.refuse(plan, reason), async () => {
+            const { org, plan: freshPlan, detail } = await this.computeFreshPlan(orgId);
+            if (!plansMatch(plan, freshPlan)) {
+                await this.refuse(plan, 'stale: live state has moved since this plan was computed');
+                throw new StalePlanError(orgId);
+            }
+            if (planNeedsConfirmation(plan.steps) &&
+                confirmedOrgName.trim() !== org.displayName.trim()) {
+                await this.refuse(plan, "destructive or migrating step without the org's name typed correctly");
+                throw new ConfirmationRequiredError(orgId);
+            }
+            const applied = await this.runSteps(org, plan, detail, fence);
+            await this.refreshPlatformFilesShare();
+            return applied;
+        }));
     }
     /**
      * After an org apply: gives the platform files runner a changed share-domain
@@ -150,9 +367,10 @@ export class FoundryPlanApply {
      * declared orgs) without a platform pass. Touches only the share hosts and
      * redeploys the runner; the platform files step itself never runs here.
      * Skips quietly for an actor who may not apply the platform, and when the
-     * platform files step was never applied. There is no platform lock, so the
-     * service ids are re-read just before the write and only the share-domains
-     * key is recorded. A failure is rethrown naming that the org part succeeded.
+     * platform files step was never applied. It holds the platform lease for
+     * the refresh and fences its write with it; while another platform change
+     * holds the lease it skips (this class has no log), and the next platform
+     * apply does the refresh. Only the share-domains key is recorded. A failure is rethrown naming that the org part succeeded.
      */
     async refreshPlatformFilesShare() {
         const deps = this.platformDeps;
@@ -160,6 +378,9 @@ export class FoundryPlanApply {
             return;
         if ((await this.authz.canApplyPlatform?.(this.actor)) !== true)
             return;
+        const lease = await this.store.acquireMutationLease(PLATFORM_SCOPE, randomUUID(), this.leaseTtlMs);
+        if (!('token' in lease))
+            return;
         try {
             const platform = await deps.store.get();
             if (!platform?.files.enabled)
@@ -171,11 +392,14 @@ export class FoundryPlanApply {
             const scopeIds = await deps.executor.refreshFilesShare(platform, recorded);
             if (scopeIds === null)
                 return;
-            await deps.store.putServiceIds({ ...(await deps.store.getServiceIds()), ...scopeIds });
+            await deps.store.putServiceIds({ ...(await deps.store.getServiceIds()), ...scopeIds }, lease);
         }
         catch (e) {
             throw new Error(`org applied; platform files share refresh failed: ${e instanceof Error ? e.message : String(e)}`);
         }
+        finally {
+            await this.store.releaseMutationLease(lease).catch(() => undefined);
+        }
     }
     /**
      * #167: {@link plan}, for the shared platform (#166) rather than an org --
@@ -216,16 +440,18 @@ export class FoundryPlanApply {
             await this.refusePlatform(plan, declared.ownerOrgId, 'not authorized');
             throw new NotAuthorizedError(this.actor, plan.orgId);
         }
-        const { platform, plan: freshPlan, detail } = await this.computeFreshPlatformPlan();
-        if (!plansMatch(plan, freshPlan)) {
-            await this.refusePlatform(plan, platform.ownerOrgId, 'stale: live state has moved since this plan was computed');
-            throw new PlatformStalePlanError();
-        }
-        if (planNeedsConfirmation(plan.steps) && confirmedName.trim() !== 'platform') {
-            await this.refusePlatform(plan, platform.ownerOrgId, "destructive or migrating step without 'platform' typed correctly");
-            throw new PlatformConfirmationRequiredError();
-        }
-        return this.runPlatformSteps(platform, plan, detail, executor);
+        return this.withMutationLease(PLATFORM_SCOPE, (reason) => this.refusePlatform(plan, declared.ownerOrgId, reason), async (fence) => {
+            const { platform, plan: freshPlan, detail } = await this.computeFreshPlatformPlan();
+            if (!plansMatch(plan, freshPlan)) {
+                await this.refusePlatform(plan, platform.ownerOrgId, 'stale: live state has moved since this plan was computed');
+                throw new PlatformStalePlanError();
+            }
+            if (planNeedsConfirmation(plan.steps) && confirmedName.trim() !== 'platform') {
+                await this.refusePlatform(plan, platform.ownerOrgId, "destructive or migrating step without 'platform' typed correctly");
+                throw new PlatformConfirmationRequiredError();
+            }
+            return this.runPlatformSteps(platform, plan, detail, executor, fence);
+        });
     }
     /**
      * Deletes every resource foundry recorded for `id`, in reverse creation
@@ -238,59 +464,270 @@ export class FoundryPlanApply {
      * declared org in place with whatever remains recorded in its applied
      * state, so a retried `destroy` only tears down what's left.
      */
-    async destroy(id, confirmedOrgName) {
+    async destroy(id, confirmedOrgName, options) {
         const authorized = await this.authz.canDestroy(this.actor, id);
         if (!authorized) {
             await this.refuseDestroy(id, 'not authorized');
             throw new NotAuthorizedError(this.actor, id);
         }
+        return this.withMutationLease(companyScope(id), (reason) => this.refuseDestroy(id, reason), (fence) => this.withRunnerSerialization(id, (reason) => this.refuseDestroy(id, reason), async () => {
+            const org = await this.store.get(id);
+            if (!org)
+                throw new DeclaredOrgNotFoundError(id);
+            if (!org.test) {
+                await this.refuseDestroy(id, 'org is not marked test; only a test org can be destroyed');
+                throw new NotATestOrgError(id);
+            }
+            if (confirmedOrgName.trim() !== org.displayName.trim()) {
+                await this.refuseDestroy(id, "org's name not typed correctly");
+                throw new ConfirmationRequiredError(id);
+            }
+            const state = await this.store.getAppliedState(id);
+            const serviceIds = { ...(state?.serviceIds ?? {}) };
+            let targets;
+            try {
+                targets = destroyTargets(serviceIds, id);
+            }
+            catch (e) {
+                await this.refuseDestroy(id, e instanceof Error ? e.message : String(e));
+                throw e;
+            }
+            const descriptions = destroyStepDescriptions(targets);
+            const steps = descriptions.map((step) => ({
+                step,
+                status: 'planned',
+            }));
+            const persist = () => this.store.putAppliedState({
+                orgId: id,
+                serviceIds,
+                checkpoints: state?.checkpoints ?? [],
+                failedFields: state?.failedFields ?? [],
+                lastApplied: state?.lastApplied ?? null,
+                destroy: { steps: steps.map((s) => ({ ...s })) },
+            }, fence);
+            // A throwing listener must never change the record, the audit row or the error.
+            const emit = (index, status, error) => {
+                const step = descriptions[index];
+                try {
+                    options?.onProgress?.({
+                        step,
+                        index: index + 1,
+                        total: steps.length,
+                        status,
+                        error,
+                    });
+                }
+                catch {
+                    // ignored on purpose
+                }
+            };
+            let current = 0;
+            // True once the step's `started` record is saved, i.e. the step is about to run.
+            let running = false;
+            try {
+                for (; current < steps.length; current++) {
+                    running = false;
+                    await this.renewLease(fence);
+                    steps[current] = { step: descriptions[current], status: 'started' };
+                    await persist();
+                    running = true;
+                    emit(current, 'started');
+                    const target = targets[current];
+                    if (target) {
+                        // A provider 404 counts as deleted -- `StepDestroyer.destroy`'s own
+                        // doc comment -- so nothing here needs to special-case "already
+                        // gone" itself.
+                        await this.destroyer.destroy(target, { org, recordedIds: { ...serviceIds } });
+                        delete serviceIds[destroyServiceIdKey(target)];
+                        if (target.system === 'cloudflare' && target.kind === 'worker') {
+                            // #318: its partial-deploy traces too, or a retry after a later
+                            // step fails would read this Worker as live again.
+                            const entryKey = target.key.slice(id.length + 1);
+                            for (const scopeKey of edgeWorkerTraceScopeKeys(entryKey)) {
+                                delete serviceIds[scopeKey];
+                            }
+                        }
+                        steps[current] = {
+                            step: descriptions[current],
+                            status: 'finished',
+                            completedAt: this.now().toISOString(),
+                        };
+                        await persist();
+                    }
+                    else {
+                        // The last step: the org and its applied state (and this record) go together.
+                        await this.store.delete(id, fence);
+                    }
+                    emit(current, 'finished');
+                }
+            }
+            catch (e) {
+                // A lost lease is audited once, by `withMutationLease`.
+                if (e instanceof StaleLeaseError)
+                    throw e;
+                const reason = e instanceof Error ? e.message : String(e);
+                if (running) {
+                    steps[current] = {
+                        step: descriptions[current],
+                        status: 'failed',
+                        error: reason,
+                    };
+                    emit(current, 'failed', reason);
+                }
+                else {
+                    // The step never ran (its `started` write failed): leave it planned.
+                    steps[current] = { step: descriptions[current], status: 'planned' };
+                }
+                // Best effort: the failure the caller sees is the step's, not this write's.
+                await persist().catch(() => undefined);
+                await this.audit.recordDestroy({
+                    actor: this.actor,
+                    orgId: id,
+                    outcome: 'failed',
+                    reason,
+                    at: this.now(),
+                });
+                throw e;
+            }
+            await this.audit.recordDestroy({
+                actor: this.actor,
+                orgId: id,
+                outcome: 'destroyed',
+                at: this.now(),
+            });
+        }));
+    }
+    /** The steps `destroy` runs for `id`, in order. Refuses a non-test org like `destroy` does; writes no audit row since it changes nothing. */
+    async planDestroy(id) {
+        if (!(await this.authz.canDestroy(this.actor, id))) {
+            throw new NotAuthorizedError(this.actor, id);
+        }
         const org = await this.store.get(id);
         if (!org)
             throw new DeclaredOrgNotFoundError(id);
-        if (!org.test) {
-            await this.refuseDestroy(id, 'org is not marked test; only a test org can be destroyed');
+        if (!org.test)
             throw new NotATestOrgError(id);
-        }
-        if (confirmedOrgName.trim() !== org.displayName.trim()) {
-            await this.refuseDestroy(id, "org's name not typed correctly");
-            throw new ConfirmationRequiredError(id);
-        }
         const state = await this.store.getAppliedState(id);
         const serviceIds = { ...(state?.serviceIds ?? {}) };
-        const targets = destroyTargets(serviceIds);
-        try {
-            for (const target of targets) {
-                // A provider 404 counts as deleted -- `StepDestroyer.destroy`'s own
-                // doc comment -- so nothing here needs to special-case "already
-                // gone" itself.
-                await this.destroyer.destroy(target, { org, recordedIds: { ...serviceIds } });
-                delete serviceIds[destroyServiceIdKey(target)];
-                await this.store.putAppliedState({
-                    orgId: id,
-                    serviceIds,
-                    checkpoints: state?.checkpoints ?? [],
-                    failedFields: state?.failedFields ?? [],
-                    lastApplied: state?.lastApplied ?? null,
-                });
+        return {
+            steps: destroyStepDescriptions(destroyTargets(serviceIds, id)),
+            kept: keptDescriptions(serviceIds),
+        };
+    }
+    /**
+     * #441 (part b): records `fn`'s run in the durable runner, when one is
+     * configured, and calls it. Must run with the mutation lease already
+     * held -- every one of this method's four callers (`apply`, `destroy`,
+     * `adopt`, `release`) calls it from inside its own `withMutationLease`
+     * callback, never around it. The reason is `JobRunner.runSteps` itself:
+     * it checkpoints a run's one step (the literal id `'mutate'`, the same
+     * for all four callers so they share one row per org) by a plain read of
+     * the row followed by a later blind write, with nothing to stop two
+     * concurrent calls for the same `runId` from both reading "not yet
+     * checkpointed" and both calling `fn` -- the runner sequences and
+     * checkpoints (its own doc comment), it does not itself exclude a second
+     * caller the way the lease's atomic acquire does. Holding the lease first
+     * is what makes it safe to reuse an existing, not-yet-succeeded run
+     * ({@link JobRunner.currentRun}) rather than refuse it outright: by the
+     * time this method runs, this caller is already the lease's sole holder,
+     * so any row `currentRun` finds can only be a previous, now-abandoned
+     * attempt (the lease's own atomic acquire already refused a second live
+     * caller before either of them got here) -- reusing it is the resume #9
+     * calls "a human decides when to call it again", not a race.
+     *
+     * Two *first* attempts for the same org can still race the insert itself
+     * when neither has an existing row yet -- `JobRunner.enqueueApply`'s own
+     * unique constraint decides it at the database level, a real race rather
+     * than one decided by this process's await order, and the loser's
+     * {@link AlreadyRunningError} is refused and audited through `refuse`,
+     * the same as a lost lease. In practice this needs two lease holders at
+     * once to reach it, which the lease itself already prevents; it is this
+     * method's defence against a bug in the lease's own acquire path, not
+     * the primary lock.
+     *
+     * `fn`'s result is threaded out through a closure, since
+     * {@link RunnerStep.run} returns nothing.
+     */
+    async withRunnerSerialization(orgId, refuse, fn) {
+        if (!this.runner)
+            return fn();
+        let result;
+        const steps = [
+            {
+                id: 'mutate',
+                run: async () => {
+                    result = await fn();
+                },
+            },
+        ];
+        const current = await this.runner.currentRun(orgId);
+        let runId;
+        if (current) {
+            runId = current.id;
+        }
+        else {
+            try {
+                ({ runId } = await this.runner.enqueueApply(orgId, steps));
+            }
+            catch (e) {
+                if (e instanceof AlreadyRunningError) {
+                    await refuse(e.message);
+                    throw e;
+                }
+                throw e;
             }
-            await this.store.delete(id);
+        }
+        try {
+            await this.runner.runSteps(runId, steps);
         }
         catch (e) {
-            await this.audit.recordDestroy({
-                actor: this.actor,
-                orgId: id,
-                outcome: 'failed',
-                reason: e instanceof Error ? e.message : String(e),
-                at: this.now(),
-            });
+            // `StepFailedError` wraps whatever `fn` threw only to say which run
+            // and step failed; every caller of `apply`/`destroy`/`adopt`/
+            // `release` expects that original error (`StalePlanError`, a
+            // provider error, whatever `fn` itself throws), not the runner's
+            // wrapper around it. `refuse` already ran inside `fn` (every
+            // `withMutationLease` callback audits its own refusal), so this does
+            // not call it again.
+            if (e instanceof StepFailedError && e.cause !== undefined)
+                throw e.cause;
             throw e;
         }
-        await this.audit.recordDestroy({
-            actor: this.actor,
-            orgId: id,
-            outcome: 'destroyed',
-            at: this.now(),
-        });
+        return result;
+    }
+    /**
+     * #441: runs `fn` while holding the one mutation lease for `scope`, so two
+     * `FoundryPlanApply` instances on one store never run provider steps for the
+     * same company or platform at once. A second caller is refused with
+     * {@link MutationInProgressError}; `fn` gets the lease as the fence its
+     * store writes carry, so a worker that lost the lease cannot overwrite the
+     * newer holder's state. Both refusals are audited through `refuse`.
+     */
+    async withMutationLease(scope, refuse, fn) {
+        const lease = await this.store.acquireMutationLease(scope, randomUUID(), this.leaseTtlMs);
+        if (!('token' in lease)) {
+            const held = new MutationInProgressError(scope, lease.heldUntil);
+            await refuse(held.message);
+            throw held;
+        }
+        try {
+            return await fn(lease);
+        }
+        catch (e) {
+            // The paths inside `fn` rethrow a lost lease without auditing it, so this is its one audit row.
+            if (e instanceof StaleLeaseError)
+                await refuse('mutation lease lost to a newer holder');
+            throw e;
+        }
+        finally {
+            // Best effort: an unreleased lease frees itself when it expires.
+            await this.store.releaseMutationLease(lease).catch(() => undefined);
+        }
+    }
+    /** #441: before each provider step, extends the lease; a holder that lost it stops here, before the step runs. */
+    async renewLease(fence) {
+        if (!(await this.store.renewMutationLease(fence, this.leaseTtlMs))) {
+            throw new StaleLeaseError(fence.scope);
+        }
     }
     async refuse(plan, reason) {
         await this.audit.recordApply({
@@ -322,7 +759,7 @@ export class FoundryPlanApply {
             at: this.now(),
         });
     }
-    async runSteps(org, plan, detail) {
+    async runSteps(org, plan, detail, fence) {
         const orgId = plan.orgId;
         const previous = await this.store.getAppliedState(orgId);
         const serviceIds = { ...(previous?.serviceIds ?? {}) };
@@ -343,11 +780,13 @@ export class FoundryPlanApply {
                     checkpoints: [...checkpoints],
                     failedFields: previous?.failedFields ?? [],
                     lastApplied: previous?.lastApplied ?? null,
-                });
+                    destroy: previous?.destroy,
+                }, fence);
             },
         };
         try {
             for (const planned of detail.plannedSteps) {
+                await this.renewLease(fence);
                 const result = await this.executor.execute(planned.action, context);
                 // `system:kind:key`, not just `kind`: two steps can share a `kind`
                 // (an org's several declared ZITADEL apps, #71) and would otherwise
@@ -365,13 +804,20 @@ export class FoundryPlanApply {
             // #9 (resolved): "it stops at the checkpoint... there is no automatic
             // rollback." Whatever completed stays recorded; the failure is
             // rethrown for a human to fix and resume.
-            await this.store.putAppliedState({
+            // A lost lease is audited once, by `withMutationLease`.
+            if (e instanceof StaleLeaseError)
+                throw e;
+            // The failure the caller sees is the step's, not a refused write's.
+            await this.store
+                .putAppliedState({
                 orgId,
                 serviceIds,
                 checkpoints,
                 failedFields: failedFields(org, detail, checkpoints.length, serviceIds),
                 lastApplied: previous?.lastApplied ?? null,
-            });
+                destroy: previous?.destroy,
+            }, fence)
+                .catch(() => undefined);
             await this.audit.recordApply({
                 actor: this.actor,
                 orgId,
@@ -389,8 +835,9 @@ export class FoundryPlanApply {
             checkpoints,
             failedFields: failedFields(org, detail, detail.plannedSteps.length, serviceIds),
             lastApplied,
+            destroy: previous?.destroy,
         };
-        await this.store.putAppliedState(applied);
+        await this.store.putAppliedState(applied, fence);
         // #253: the single-snapshot write just above replaces whatever
         // `lastApplied` this org had; this is the point in today's code where a
         // version is recorded as applied, so it's also where the appended
@@ -425,6 +872,7 @@ export class FoundryPlanApply {
         const detail = computePlanDetail(org, live, applied?.serviceIds ?? {}, agoraHost, this.jobsScheduler, platformReadiness, {
             loginHostBindings: this.loginHostBindings,
             broker: this.broker,
+            edgeWorkers: this.edgeWorkers,
             otherSlugs,
             now: () => this.now().getTime(),
             archonUrl,
@@ -479,7 +927,7 @@ export class FoundryPlanApply {
             refs.add(ORG_PLATFORM_KEY_REFS.ai);
         if (this.jobsScheduler)
             refs.add(ORG_PLATFORM_KEY_REFS.jobs);
-        const readiness = orgPlatformReadinessFrom(platform, recorded, refs, !heldKeys);
+        const readiness = orgPlatformReadinessFrom(platform, recorded, refs, !heldKeys, this.hostNames);
         // Entry env from the platform's mail key or a shared secret (#T8).
         let mailKeyHeld = false;
         try {
@@ -588,7 +1036,7 @@ export class FoundryPlanApply {
      * `serviceIds`), so a later plan finds a resource by its recorded id even
      * when a provider's list has not caught up with a create.
      */
-    async runPlatformSteps(platform, plan, detail, executor) {
+    async runPlatformSteps(platform, plan, detail, executor, fence) {
         const { store } = this.requirePlatformDeps();
         const serviceIds = { ...(await store.getServiceIds()) };
         const context = {
@@ -597,10 +1045,11 @@ export class FoundryPlanApply {
             recordScopeId: (key, id) => {
                 serviceIds[key] = id;
             },
-            persist: () => store.putServiceIds(serviceIds),
+            persist: () => store.putServiceIds(serviceIds, fence),
         };
         try {
             for (const planned of detail.plannedSteps) {
+                await this.renewLease(fence);
                 const result = await executor.execute(planned.action, context);
                 if (result?.resourceId) {
                     const { system, kind, key } = planned.action;
@@ -609,8 +1058,12 @@ export class FoundryPlanApply {
             }
         }
         catch (e) {
-            // Whatever completed stays recorded, as an org's apply does.
-            await store.putServiceIds(serviceIds);
+            // Whatever completed stays recorded, as an org's apply does. A lost
+            // lease is audited once, by `withMutationLease`; any other failure
+            // stays the step's, not a refused write's.
+            if (e instanceof StaleLeaseError)
+                throw e;
+            await store.putServiceIds(serviceIds, fence).catch(() => undefined);
             await this.audit.recordApply({
                 actor: this.actor,
                 orgId: platform.ownerOrgId,
@@ -622,7 +1075,7 @@ export class FoundryPlanApply {
             });
             throw e;
         }
-        await store.putServiceIds(serviceIds);
+        await store.putServiceIds(serviceIds, fence);
         await this.audit.recordApply({
             actor: this.actor,
             orgId: platform.ownerOrgId,
@@ -677,7 +1130,7 @@ const ZITADEL_ORG_SCOPE_KEY = 'zitadel_org';
  * belongs to is already left in place -- deleting only the rule and not the
  * zone is a decision left for whoever wires teardown in, not made here.
  */
-function destroyTargets(serviceIds) {
+function destroyTargets(serviceIds, orgId) {
     // Reverse-teardown order: everything under one of these prefixes is
     // collected in this order, then every ZITADEL app (there can be several,
     // #71), then the two scope ids last.
@@ -685,6 +1138,8 @@ function destroyTargets(serviceIds) {
         'zitadel:login-host:',
         'cloudflare:tunnel:',
         'cloudflare:access:',
+        // #318: an entry's edge Worker and its route.
+        'cloudflare:worker:',
         'coolify:database:',
         'coolify:app:',
         'zitadel:customers:',
@@ -692,20 +1147,80 @@ function destroyTargets(serviceIds) {
         'ai:key:',
         // F6b: an entry's files key, `<orgId>:<entry>`; the org's own key has no `:` and is left alone.
         'files:key:',
+        // F7 (#317): an entry's names key, `<orgId>:<entry>` -- names has no org-root key.
+        'names:key:',
     ];
     const targets = [];
+    const adopted = adoptedResourceIds(serviceIds);
     for (const prefix of ORDERED_PREFIXES) {
+        if (prefix === 'cloudflare:worker:') {
+            // #318 re-review: not the main ids alone -- `liveEdgeWorkers` also
+            // finds an upload-only or route-only partial deploy, and leaves out an
+            // entry whose Worker is already retired (nothing live, so a host with
+            // no WorkersPort is not blocked from destroying the org by it).
+            for (const [entryKey, scriptName] of liveEdgeWorkers(orgId, serviceIds)) {
+                targets.push({
+                    system: 'cloudflare',
+                    kind: 'worker',
+                    key: `${orgId}:${entryKey}`,
+                    id: scriptName,
+                });
+            }
+            continue;
+        }
         for (const [resourceKey, id] of Object.entries(serviceIds)) {
             if (!resourceKey.startsWith(prefix))
                 continue;
+            // Archon#271: foundry did not create an adopted app or database, so it never deletes it.
+            if (adopted.has(resourceKey))
+                continue;
             if (prefix === 'ai:key:' && !resourceKey.slice(prefix.length).includes(':'))
                 continue;
             if (prefix === 'files:key:' && !resourceKey.slice(prefix.length).includes(':'))
                 continue;
+            // A key a retire step already revoked before org destroy ever ran
+            // (#317 review): `runSteps` never clears this main id on a step that
+            // returns no resourceId, so without this check destroy would target
+            // an already-revoked key and get stuck on names' 404 forever.
+            if (prefix === 'names:key:') {
+                // `resourceKey.slice(prefix.length)` is `<orgId>:<entryKey>`, but
+                // `entryNamesRetiredScopeKey` takes the bare entry key alone (as
+                // every caller in entry-names-key.ts/executor.ts gives it) --
+                // entry keys hold no `:` of their own, so the first one found is
+                // exactly the org id/entry key boundary.
+                const fullKey = resourceKey.slice(prefix.length);
+                const entryKey = fullKey.slice(fullKey.indexOf(':') + 1);
+                if (serviceIds[entryNamesRetiredScopeKey(entryKey)] === id)
+                    continue;
+            }
             const [system, kind] = prefix.split(':');
             targets.push({ system, kind, key: resourceKey.slice(prefix.length), id });
         }
     }
+    // F7 (#317): a key minted but never delivered never gets the main
+    // `names:key:<org>:<entry>` id recorded (the mint step throws before
+    // `runSteps` sees a `resourceId`), so the loop above never finds it to
+    // revoke on destroy. Skipped if a later mint already succeeded and holds
+    // its own, still-live main id -- entry keys hold no `:`, so matching the
+    // suffix is unambiguous. A main id that equals the entry's retired marker
+    // does not count as "later": mint A, deliver, retire (main id A, retired
+    // marker A); re-declare, mint B, fail delivery three times (undelivered
+    // B, main id stays A). Without this check, A's stale-but-truthy main id
+    // hid B from destroy, leaking it the same way the gone-scan in
+    // `entry-names-key.ts` once did (#317 re-review).
+    const undeliveredNamesPrefix = 'names_key_undelivered_';
+    for (const [resourceKey, id] of Object.entries(serviceIds)) {
+        if (!id || !resourceKey.startsWith(undeliveredNamesPrefix))
+            continue;
+        const entryKey = resourceKey.slice(undeliveredNamesPrefix.length);
+        const hasLaterMainId = Object.entries(serviceIds).some(([k, v]) => v &&
+            k.startsWith('names:key:') &&
+            k.endsWith(`:${entryKey}`) &&
+            serviceIds[entryNamesRetiredScopeKey(entryKey)] !== v);
+        if (hasLaterMainId)
+            continue;
+        targets.push({ system: 'names', kind: 'key', key: entryKey, id });
+    }
     for (const [resourceKey, id] of Object.entries(serviceIds)) {
         if (!resourceKey.startsWith('zitadel:app:'))
             continue;
@@ -726,6 +1241,38 @@ function destroyTargets(serviceIds) {
     }
     return targets;
 }
+/** One step per target, then the declared org's own deletion, which has no target. */
+/** The `serviceIds` keys of the coolify app and database every `adopted.<entry>` record names. */
+function adoptedResourceIds(serviceIds) {
+    // Fail closed: an adopted record that cannot be read could name a resource destroy must keep.
+    const unreadable = adoptedKeysUnreadable(serviceIds);
+    if (unreadable.length > 0) {
+        throw new Error(`destroy refused: the adopted record ${unreadable.join(', ')} cannot be read, so foundry cannot tell which resources it must keep`);
+    }
+    const uuids = new Set();
+    for (const [key, raw] of Object.entries(serviceIds)) {
+        if (!key.startsWith('adopted.'))
+            continue;
+        const snapshot = parseAdoptSnapshot(raw);
+        if (snapshot) {
+            uuids.add(snapshot.application.uuid);
+            uuids.add(snapshot.database.uuid);
+        }
+    }
+    return new Set(Object.entries(serviceIds)
+        .filter(([k, id]) => (k.startsWith('coolify:app:') || k.startsWith('coolify:database:')) && uuids.has(id))
+        .map(([k]) => k));
+}
+/** Lines for adopted resources `destroy` keeps, so the operator sees them beside the deletes. */
+function keptDescriptions(serviceIds) {
+    return [...adoptedResourceIds(serviceIds)].map((k) => `keep ${k.split(':')[0]} ${k.split(':')[1]} ${k.split(':').slice(2).join(':')} (adopted, not deleted)`);
+}
+function destroyStepDescriptions(targets) {
+    return [
+        ...targets.map((t) => `delete ${t.system} ${t.kind} ${t.key}`),
+        'delete the declared org',
+    ];
+}
 /** The `AppliedState.serviceIds` key a completed {@link DestroyTarget} should be removed from. */
 function destroyServiceIdKey(target) {
     if (target.kind === 'project' || target.kind === 'org')
@@ -746,7 +1293,6 @@ const STEP_FIELDS = {
     'coolify:chat': 'chat.usesDomain',
     'coolify:calls': 'calls.enabled',
     'coolify:calls-service': 'calls.enabled',
-    'files:storage': 'storage',
     'ai:budget': 'ai',
     'ai:key': 'ai',
     'cms:site': 'site',
@@ -756,6 +1302,7 @@ const STEP_FIELDS = {
     'cloudflare:zone': 'hosting.usesDomain',
     'cloudflare:access': 'hosting.accessEmails',
     'cloudflare:ratelimit': 'hosting.usesDomain',
+    'cloudflare:worker': 'hosting.entries.edge',
 };
 function stepField(action) {
     // An entry's ai key (F6a) is keyed `<orgId>:<entry>`; the org's own is `<orgId>`.
@@ -765,6 +1312,9 @@ function stepField(action) {
     // The org's own files key (F6b) is keyed `<orgId>`, an entry's `<orgId>:<entry>`.
     if (action.system === 'files' && action.kind === 'key')
         return 'hosting.entries.uses.files';
+    // An entry's names key (F7): always `<orgId>:<entry>`, names has no org-root key.
+    if (action.system === 'names' && action.kind === 'key')
+        return 'hosting.entries.uses.names';
     // An org declares several issuer apps, each keyed by its name (#71).
     if (action.system === 'zitadel' && action.kind === 'app') {
         return `hosting.issuerApps.${action.key}`;
@@ -813,6 +1363,17 @@ function failedFields(org, detail, completed, serviceIds) {
             }
         }
     }
+    // The same for an entry's names key (F7): minted, never delivered is no
+    // longer blocked forever on its own (the next apply revokes it and
+    // mints again, #317 re-review) -- but while that retry step is itself
+    // not planned this run (no names service wired, say), the leftover
+    // undelivered id must still read as unresolved rather than the org
+    // converging silently around it.
+    for (const entry of org.hosting.entries) {
+        if (entry.uses?.names && serviceIds[entryNamesUndeliveredScopeKey(entry.key)]) {
+            failed.add('hosting.entries.uses.names');
+        }
+    }
     // The same for a jobs registration whose credentials never reached the host.
     if (serviceIds[JOBS_REGISTRATION_UNDELIVERED_SCOPE_KEY] &&
         !serviceIds[`jobs:registration:${org.id}`]) {