@wtfalch/foundry 0.16.16 → 0.17.0

package/dist/plan/plan-apply.js

@@ -1,16 +1,36 @@
-import { createHash } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 import { VERSIONS, floatingTagBlocked } from '../bootstrap/versions.js';
 import { validateHosting } from '../org.js';
 import { PLATFORM_SERVICE_KEYS } from '../platform.js';
+import { AlreadyRunningError, StepFailedError } from '../runner/errors.js';
+import { StaleLeaseError, } from '../store.js';
+import { AdoptRefusedError, adoptCandidates, adoptedKeysUnreadable, adoptedScopeKey, parseAdoptSnapshot, planAdoptFrom, } from './adopt.js';
 import { NO_AGORA_HOST_VALUES, readAgoraHost, } from './agora.js';
 import { AI_KEY_UNDELIVERED_SCOPE_KEY, JOBS_REGISTRATION_UNDELIVERED_SCOPE_KEY, computePlanDetail, toPlan, } from './diff.js';
 import { entryAiUndeliveredScopeKey } from './entry-ai-key.js';
 import { FILES_ROOT_UNDELIVERED_SCOPE_KEY, entryFilesUndeliveredScopeKey, } from './entry-files-key.js';
 import { readLivePlatformResources, readLiveResources, } from './live.js';
 import { lockedHostnames } from './locked-hostnames.js';
-import { PLATFORM_ARCHON_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_KEY, planPlatform as computePlatformPlanDetail, toPlatformPlan, } from './platform-diff.js';
+import { PLATFORM_ARCHON_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_KEY, appStepKey, planPlatform as computePlatformPlanDetail, toPlatformPlan, } from './platform-diff.js';
 import { DEFAULT_ORG_PLATFORM_READINESS, ORG_PLATFORM_KEY_REFS, agoraHostValuesFromPlatform, heldPlatformKeyRefs, orgPlatformReadinessFrom, } from './platform-org.js';
 import { PLATFORM_AUTH_BROKER_MAIL_KEY_REF } from './platform-providers.js';
+/** #441: the lease scope that serializes every mutation of one company, and the one for the shared platform. */
+const companyScope = (orgId) => `company:${orgId}`;
+const PLATFORM_SCOPE = 'platform';
+/** A lease nobody renews: long enough for one apply, short enough that a dead worker's scope frees itself. The fence stops a slow worker that outlives it. */
+const DEFAULT_LEASE_TTL_MS = 30 * 60 * 1000;
+/** #441: another apply, destroy, adopt or release holds the lease for this company or the platform. Nothing ran; try again when it finishes. */
+export class MutationInProgressError extends Error {
+    scope;
+    heldUntil;
+    /** @param heldUntil when the holder's lease runs out if it is never released. */
+    constructor(scope, heldUntil) {
+        super(`another change to ${scope} is in progress (lease until ${heldUntil.toISOString()}); try again when it finishes`);
+        this.scope = scope;
+        this.heldUntil = heldUntil;
+        this.name = 'MutationInProgressError';
+    }
+}
 export class DeclaredOrgNotFoundError extends Error {
     constructor(orgId) {
         super(`no declared org ${orgId}`);
@@ -99,8 +119,10 @@ export class FoundryPlanApply {
     hostFiles;
     hostAi;
     now;
+    leaseTtlMs;
     appliedVersions;
     platformDeps;
+    runner;
     constructor(options) {
         this.store = options.store;
         this.readers = options.readers;
@@ -116,14 +138,197 @@ export class FoundryPlanApply {
         this.hostFiles = options.filesOrganisation ?? false;
         this.hostAi = options.aiOrganisation ?? false;
         this.now = options.now ?? (() => new Date());
+        this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
         this.appliedVersions = options.appliedVersions;
         this.platformDeps = options.platform;
+        this.runner = options.runner;
     }
     async plan(id) {
         const { plan } = await this.computeFreshPlan(id);
         await this.audit.recordPlan({ actor: this.actor, orgId: id, plan, at: this.now() });
         return plan;
     }
+    /**
+     * Archon#271: what adopting a live app and database into entry
+     * `input.entryKey` would do. A read, like {@link plan}: it reads the project
+     * through the reader and changes nothing. A mismatch is reported in the
+     * result, with no steps. Needs the apply permission, since it reads a
+     * project the caller names.
+     */
+    async planAdopt(id, input) {
+        if (!(await this.authz.canApply(this.actor, id)))
+            throw new NotAuthorizedError(this.actor, id);
+        const { adoptPlan } = await this.computeAdopt(id, input);
+        await this.audit.recordPlan({
+            actor: this.actor,
+            orgId: id,
+            plan: adoptPlan.plan,
+            at: this.now(),
+        });
+        return adoptPlan;
+    }
+    /**
+     * Archon#271: the live apps and databases in `projectUuid` this org has not
+     * recorded, for Changes' "Live, not theirs". A read, but it needs the apply
+     * permission: it reads a project the caller names.
+     */
+    async adoptCandidates(id, entryKey, projectUuid) {
+        if (!(await this.authz.canApply(this.actor, id)))
+            throw new NotAuthorizedError(this.actor, id);
+        const org = await this.store.get(id);
+        if (!org)
+            throw new DeclaredOrgNotFoundError(id);
+        const recorded = (await this.store.getAppliedState(id))?.serviceIds ?? {};
+        const project = projectUuid ?? recorded.coolify_project;
+        const live = project ? ((await this.readers.coolify.listProject?.(project)) ?? []) : [];
+        return adoptCandidates(org, entryKey, recorded, live);
+    }
+    /**
+     * Archon#271: records an app and database that already run in Coolify as
+     * entry `input.entryKey`'s resources, so the next plan finds them and plans
+     * no create. Verifies first and refuses on a mismatch or an entry that is
+     * already applied or held by another org. Same permission as {@link apply}.
+     * It never calls an executor: nothing in Coolify changes. Nothing redeploys
+     * here, but the next normal plan may propose updates to the adopted app
+     * that do (see `adopt.ts`); show that plan before applying it. Two
+     * concurrent adopts of one resource are not locked against each other.
+     */
+    async adopt(id, input) {
+        const refuse = async (reason) => {
+            await this.audit.recordAdoption({
+                actor: this.actor,
+                orgId: id,
+                action: 'adopt',
+                entryKey: input.entryKey,
+                outcome: 'refused',
+                reason,
+                at: this.now(),
+            });
+        };
+        if (!(await this.authz.canApply(this.actor, id))) {
+            await refuse('not authorized');
+            throw new NotAuthorizedError(this.actor, id);
+        }
+        return this.withMutationLease(companyScope(id), refuse, (fence) => this.withRunnerSerialization(id, refuse, async () => {
+            const { adoptPlan, previous } = await this.computeAdopt(id, input);
+            const { snapshot } = adoptPlan;
+            if (adoptPlan.mismatches.length > 0 || !snapshot) {
+                await refuse(adoptPlan.mismatches.join('; '));
+                throw new AdoptRefusedError(adoptPlan.mismatches);
+            }
+            const org = await this.store.get(id);
+            if (!org)
+                throw new DeclaredOrgNotFoundError(id);
+            const serviceIds = {
+                ...(previous?.serviceIds ?? {}),
+                coolify_project: snapshot.projectUuid,
+                [`coolify:app:${appStepKey(org, input.entryKey)}`]: snapshot.application.uuid,
+                [`coolify:database:${id}`]: snapshot.database.uuid,
+                [adoptedScopeKey(input.entryKey)]: JSON.stringify(snapshot),
+            };
+            const applied = {
+                orgId: id,
+                serviceIds,
+                checkpoints: previous?.checkpoints ?? [],
+                failedFields: previous?.failedFields ?? [],
+                lastApplied: previous?.lastApplied ?? null,
+            };
+            await this.store.putAppliedState(applied, fence);
+            await this.audit.recordAdoption({
+                actor: this.actor,
+                orgId: id,
+                action: 'adopt',
+                entryKey: input.entryKey,
+                outcome: 'adopted',
+                at: this.now(),
+            });
+            return applied;
+        }));
+    }
+    /**
+     * Archon#271: the undo of {@link adopt}. Drops the entry's recorded Coolify
+     * ids and its `adopted.<entryKey>` snapshot (and `coolify_project` when
+     * adopt recorded it and no other Coolify id remains). It leaves the
+     * declaration alone: the entry goes back to declared and not applied, so the
+     * next plan creates it, exactly as before adopt. The host then adopts it
+     * again or removes the entry from the declaration. Deletes and stops nothing
+     * in Coolify. Only an adopted entry can be released. Same permission as
+     * {@link apply}. Every path writes an audit row.
+     */
+    async release(id, entryKey) {
+        const record = (outcome, reason) => this.audit.recordAdoption({
+            actor: this.actor,
+            orgId: id,
+            action: 'release',
+            entryKey,
+            outcome,
+            ...(reason === undefined ? {} : { reason }),
+            at: this.now(),
+        });
+        if (!(await this.authz.canApply(this.actor, id))) {
+            await record('refused', 'not authorized');
+            throw new NotAuthorizedError(this.actor, id);
+        }
+        return this.withMutationLease(companyScope(id), (reason) => record('refused', reason), (fence) => this.withRunnerSerialization(id, (reason) => record('refused', reason), async () => {
+            const org = await this.store.get(id);
+            if (!org) {
+                await record('refused', `no declared org ${id}`);
+                throw new DeclaredOrgNotFoundError(id);
+            }
+            const previous = await this.store.getAppliedState(id);
+            const snapshot = parseAdoptSnapshot(previous?.serviceIds[adoptedScopeKey(entryKey)]);
+            if (!previous || !snapshot) {
+                await record('refused', `entry ${entryKey} was not adopted`);
+                throw new AdoptRefusedError([`entry ${entryKey} was not adopted`]);
+            }
+            const dropped = new Set([
+                adoptedScopeKey(entryKey),
+                `coolify:app:${appStepKey(org, entryKey)}`,
+                `coolify:database:${id}`,
+            ]);
+            const kept = Object.entries(previous.serviceIds).filter(([k]) => !dropped.has(k));
+            const otherCoolify = kept.some(([k]) => k.startsWith('coolify:'));
+            const serviceIds = Object.fromEntries(kept.filter(([k]) => !(k === 'coolify_project' && snapshot.projectRecorded && !otherCoolify)));
+            const released = { ...previous, serviceIds };
+            try {
+                await this.store.putAppliedState(released, fence);
+            }
+            catch (e) {
+                if (e instanceof StaleLeaseError)
+                    throw e;
+                await record('refused', e instanceof Error ? e.message : String(e));
+                throw e;
+            }
+            await record('released');
+            return released;
+        }));
+    }
+    async computeAdopt(id, input) {
+        const org = await this.store.get(id);
+        if (!org)
+            throw new DeclaredOrgNotFoundError(id);
+        const previous = await this.store.getAppliedState(id);
+        const recorded = previous?.serviceIds ?? {};
+        const project = input.projectUuid ?? recorded.coolify_project;
+        const reader = this.readers.coolify;
+        const live = project && reader.listProject ? await reader.listProject(project) : null;
+        const claimedElsewhere = [];
+        for (const [what, uuid] of [
+            ['application', input.applicationUuid],
+            ['database', input.databaseUuid],
+            ['project', project],
+        ]) {
+            if (!uuid)
+                continue;
+            const owner = await this.store.findOtherOrgRecordingCoolifyId(uuid, id);
+            if (owner)
+                claimedElsewhere.push(`${what} ${uuid} is already recorded by org ${owner}`);
+        }
+        return {
+            adoptPlan: planAdoptFrom(org, recorded, input, live, this.now(), claimedElsewhere),
+            previous,
+        };
+    }
     async apply(plan, confirmedOrgName) {
         const orgId = plan.orgId;
         const authorized = await this.authz.canApply(this.actor, orgId);
@@ -131,18 +336,21 @@ export class FoundryPlanApply {
             await this.refuse(plan, 'not authorized');
             throw new NotAuthorizedError(this.actor, orgId);
         }
-        const { org, plan: freshPlan, detail } = await this.computeFreshPlan(orgId);
-        if (!plansMatch(plan, freshPlan)) {
-            await this.refuse(plan, 'stale: live state has moved since this plan was computed');
-            throw new StalePlanError(orgId);
-        }
-        if (planNeedsConfirmation(plan.steps) && confirmedOrgName.trim() !== org.displayName.trim()) {
-            await this.refuse(plan, "destructive or migrating step without the org's name typed correctly");
-            throw new ConfirmationRequiredError(orgId);
-        }
-        const applied = await this.runSteps(org, plan, detail);
-        await this.refreshPlatformFilesShare();
-        return applied;
+        return this.withMutationLease(companyScope(orgId), (reason) => this.refuse(plan, reason), (fence) => this.withRunnerSerialization(orgId, (reason) => this.refuse(plan, reason), async () => {
+            const { org, plan: freshPlan, detail } = await this.computeFreshPlan(orgId);
+            if (!plansMatch(plan, freshPlan)) {
+                await this.refuse(plan, 'stale: live state has moved since this plan was computed');
+                throw new StalePlanError(orgId);
+            }
+            if (planNeedsConfirmation(plan.steps) &&
+                confirmedOrgName.trim() !== org.displayName.trim()) {
+                await this.refuse(plan, "destructive or migrating step without the org's name typed correctly");
+                throw new ConfirmationRequiredError(orgId);
+            }
+            const applied = await this.runSteps(org, plan, detail, fence);
+            await this.refreshPlatformFilesShare();
+            return applied;
+        }));
     }
     /**
      * After an org apply: gives the platform files runner a changed share-domain
@@ -150,9 +358,10 @@ export class FoundryPlanApply {
      * declared orgs) without a platform pass. Touches only the share hosts and
      * redeploys the runner; the platform files step itself never runs here.
      * Skips quietly for an actor who may not apply the platform, and when the
-     * platform files step was never applied. There is no platform lock, so the
-     * service ids are re-read just before the write and only the share-domains
-     * key is recorded. A failure is rethrown naming that the org part succeeded.
+     * platform files step was never applied. It holds the platform lease for
+     * the refresh and fences its write with it; while another platform change
+     * holds the lease it skips (this class has no log), and the next platform
+     * apply does the refresh. Only the share-domains key is recorded. A failure is rethrown naming that the org part succeeded.
      */
     async refreshPlatformFilesShare() {
         const deps = this.platformDeps;
@@ -160,6 +369,9 @@ export class FoundryPlanApply {
             return;
         if ((await this.authz.canApplyPlatform?.(this.actor)) !== true)
             return;
+        const lease = await this.store.acquireMutationLease(PLATFORM_SCOPE, randomUUID(), this.leaseTtlMs);
+        if (!('token' in lease))
+            return;
         try {
             const platform = await deps.store.get();
             if (!platform?.files.enabled)
@@ -171,11 +383,14 @@ export class FoundryPlanApply {
             const scopeIds = await deps.executor.refreshFilesShare(platform, recorded);
             if (scopeIds === null)
                 return;
-            await deps.store.putServiceIds({ ...(await deps.store.getServiceIds()), ...scopeIds });
+            await deps.store.putServiceIds({ ...(await deps.store.getServiceIds()), ...scopeIds }, lease);
         }
         catch (e) {
             throw new Error(`org applied; platform files share refresh failed: ${e instanceof Error ? e.message : String(e)}`);
         }
+        finally {
+            await this.store.releaseMutationLease(lease).catch(() => undefined);
+        }
     }
     /**
      * #167: {@link plan}, for the shared platform (#166) rather than an org --
@@ -216,16 +431,18 @@ export class FoundryPlanApply {
             await this.refusePlatform(plan, declared.ownerOrgId, 'not authorized');
             throw new NotAuthorizedError(this.actor, plan.orgId);
         }
-        const { platform, plan: freshPlan, detail } = await this.computeFreshPlatformPlan();
-        if (!plansMatch(plan, freshPlan)) {
-            await this.refusePlatform(plan, platform.ownerOrgId, 'stale: live state has moved since this plan was computed');
-            throw new PlatformStalePlanError();
-        }
-        if (planNeedsConfirmation(plan.steps) && confirmedName.trim() !== 'platform') {
-            await this.refusePlatform(plan, platform.ownerOrgId, "destructive or migrating step without 'platform' typed correctly");
-            throw new PlatformConfirmationRequiredError();
-        }
-        return this.runPlatformSteps(platform, plan, detail, executor);
+        return this.withMutationLease(PLATFORM_SCOPE, (reason) => this.refusePlatform(plan, declared.ownerOrgId, reason), async (fence) => {
+            const { platform, plan: freshPlan, detail } = await this.computeFreshPlatformPlan();
+            if (!plansMatch(plan, freshPlan)) {
+                await this.refusePlatform(plan, platform.ownerOrgId, 'stale: live state has moved since this plan was computed');
+                throw new PlatformStalePlanError();
+            }
+            if (planNeedsConfirmation(plan.steps) && confirmedName.trim() !== 'platform') {
+                await this.refusePlatform(plan, platform.ownerOrgId, "destructive or migrating step without 'platform' typed correctly");
+                throw new PlatformConfirmationRequiredError();
+            }
+            return this.runPlatformSteps(platform, plan, detail, executor, fence);
+        });
     }
     /**
      * Deletes every resource foundry recorded for `id`, in reverse creation
@@ -238,59 +455,262 @@ export class FoundryPlanApply {
      * declared org in place with whatever remains recorded in its applied
      * state, so a retried `destroy` only tears down what's left.
      */
-    async destroy(id, confirmedOrgName) {
+    async destroy(id, confirmedOrgName, options) {
         const authorized = await this.authz.canDestroy(this.actor, id);
         if (!authorized) {
             await this.refuseDestroy(id, 'not authorized');
             throw new NotAuthorizedError(this.actor, id);
         }
+        return this.withMutationLease(companyScope(id), (reason) => this.refuseDestroy(id, reason), (fence) => this.withRunnerSerialization(id, (reason) => this.refuseDestroy(id, reason), async () => {
+            const org = await this.store.get(id);
+            if (!org)
+                throw new DeclaredOrgNotFoundError(id);
+            if (!org.test) {
+                await this.refuseDestroy(id, 'org is not marked test; only a test org can be destroyed');
+                throw new NotATestOrgError(id);
+            }
+            if (confirmedOrgName.trim() !== org.displayName.trim()) {
+                await this.refuseDestroy(id, "org's name not typed correctly");
+                throw new ConfirmationRequiredError(id);
+            }
+            const state = await this.store.getAppliedState(id);
+            const serviceIds = { ...(state?.serviceIds ?? {}) };
+            let targets;
+            try {
+                targets = destroyTargets(serviceIds);
+            }
+            catch (e) {
+                await this.refuseDestroy(id, e instanceof Error ? e.message : String(e));
+                throw e;
+            }
+            const descriptions = destroyStepDescriptions(targets);
+            const steps = descriptions.map((step) => ({
+                step,
+                status: 'planned',
+            }));
+            const persist = () => this.store.putAppliedState({
+                orgId: id,
+                serviceIds,
+                checkpoints: state?.checkpoints ?? [],
+                failedFields: state?.failedFields ?? [],
+                lastApplied: state?.lastApplied ?? null,
+                destroy: { steps: steps.map((s) => ({ ...s })) },
+            }, fence);
+            // A throwing listener must never change the record, the audit row or the error.
+            const emit = (index, status, error) => {
+                const step = descriptions[index];
+                try {
+                    options?.onProgress?.({
+                        step,
+                        index: index + 1,
+                        total: steps.length,
+                        status,
+                        error,
+                    });
+                }
+                catch {
+                    // ignored on purpose
+                }
+            };
+            let current = 0;
+            // True once the step's `started` record is saved, i.e. the step is about to run.
+            let running = false;
+            try {
+                for (; current < steps.length; current++) {
+                    running = false;
+                    await this.renewLease(fence);
+                    steps[current] = { step: descriptions[current], status: 'started' };
+                    await persist();
+                    running = true;
+                    emit(current, 'started');
+                    const target = targets[current];
+                    if (target) {
+                        // A provider 404 counts as deleted -- `StepDestroyer.destroy`'s own
+                        // doc comment -- so nothing here needs to special-case "already
+                        // gone" itself.
+                        await this.destroyer.destroy(target, { org, recordedIds: { ...serviceIds } });
+                        delete serviceIds[destroyServiceIdKey(target)];
+                        steps[current] = {
+                            step: descriptions[current],
+                            status: 'finished',
+                            completedAt: this.now().toISOString(),
+                        };
+                        await persist();
+                    }
+                    else {
+                        // The last step: the org and its applied state (and this record) go together.
+                        await this.store.delete(id, fence);
+                    }
+                    emit(current, 'finished');
+                }
+            }
+            catch (e) {
+                // A lost lease is audited once, by `withMutationLease`.
+                if (e instanceof StaleLeaseError)
+                    throw e;
+                const reason = e instanceof Error ? e.message : String(e);
+                if (running) {
+                    steps[current] = {
+                        step: descriptions[current],
+                        status: 'failed',
+                        error: reason,
+                    };
+                    emit(current, 'failed', reason);
+                }
+                else {
+                    // The step never ran (its `started` write failed): leave it planned.
+                    steps[current] = { step: descriptions[current], status: 'planned' };
+                }
+                // Best effort: the failure the caller sees is the step's, not this write's.
+                await persist().catch(() => undefined);
+                await this.audit.recordDestroy({
+                    actor: this.actor,
+                    orgId: id,
+                    outcome: 'failed',
+                    reason,
+                    at: this.now(),
+                });
+                throw e;
+            }
+            await this.audit.recordDestroy({
+                actor: this.actor,
+                orgId: id,
+                outcome: 'destroyed',
+                at: this.now(),
+            });
+        }));
+    }
+    /** The steps `destroy` runs for `id`, in order. Refuses a non-test org like `destroy` does; writes no audit row since it changes nothing. */
+    async planDestroy(id) {
+        if (!(await this.authz.canDestroy(this.actor, id))) {
+            throw new NotAuthorizedError(this.actor, id);
+        }
         const org = await this.store.get(id);
         if (!org)
             throw new DeclaredOrgNotFoundError(id);
-        if (!org.test) {
-            await this.refuseDestroy(id, 'org is not marked test; only a test org can be destroyed');
+        if (!org.test)
             throw new NotATestOrgError(id);
-        }
-        if (confirmedOrgName.trim() !== org.displayName.trim()) {
-            await this.refuseDestroy(id, "org's name not typed correctly");
-            throw new ConfirmationRequiredError(id);
-        }
         const state = await this.store.getAppliedState(id);
         const serviceIds = { ...(state?.serviceIds ?? {}) };
-        const targets = destroyTargets(serviceIds);
-        try {
-            for (const target of targets) {
-                // A provider 404 counts as deleted -- `StepDestroyer.destroy`'s own
-                // doc comment -- so nothing here needs to special-case "already
-                // gone" itself.
-                await this.destroyer.destroy(target, { org, recordedIds: { ...serviceIds } });
-                delete serviceIds[destroyServiceIdKey(target)];
-                await this.store.putAppliedState({
-                    orgId: id,
-                    serviceIds,
-                    checkpoints: state?.checkpoints ?? [],
-                    failedFields: state?.failedFields ?? [],
-                    lastApplied: state?.lastApplied ?? null,
-                });
+        return {
+            steps: destroyStepDescriptions(destroyTargets(serviceIds)),
+            kept: keptDescriptions(serviceIds),
+        };
+    }
+    /**
+     * #441 (part b): records `fn`'s run in the durable runner, when one is
+     * configured, and calls it. Must run with the mutation lease already
+     * held -- every one of this method's four callers (`apply`, `destroy`,
+     * `adopt`, `release`) calls it from inside its own `withMutationLease`
+     * callback, never around it. The reason is `JobRunner.runSteps` itself:
+     * it checkpoints a run's one step (the literal id `'mutate'`, the same
+     * for all four callers so they share one row per org) by a plain read of
+     * the row followed by a later blind write, with nothing to stop two
+     * concurrent calls for the same `runId` from both reading "not yet
+     * checkpointed" and both calling `fn` -- the runner sequences and
+     * checkpoints (its own doc comment), it does not itself exclude a second
+     * caller the way the lease's atomic acquire does. Holding the lease first
+     * is what makes it safe to reuse an existing, not-yet-succeeded run
+     * ({@link JobRunner.currentRun}) rather than refuse it outright: by the
+     * time this method runs, this caller is already the lease's sole holder,
+     * so any row `currentRun` finds can only be a previous, now-abandoned
+     * attempt (the lease's own atomic acquire already refused a second live
+     * caller before either of them got here) -- reusing it is the resume #9
+     * calls "a human decides when to call it again", not a race.
+     *
+     * Two *first* attempts for the same org can still race the insert itself
+     * when neither has an existing row yet -- `JobRunner.enqueueApply`'s own
+     * unique constraint decides it at the database level, a real race rather
+     * than one decided by this process's await order, and the loser's
+     * {@link AlreadyRunningError} is refused and audited through `refuse`,
+     * the same as a lost lease. In practice this needs two lease holders at
+     * once to reach it, which the lease itself already prevents; it is this
+     * method's defence against a bug in the lease's own acquire path, not
+     * the primary lock.
+     *
+     * `fn`'s result is threaded out through a closure, since
+     * {@link RunnerStep.run} returns nothing.
+     */
+    async withRunnerSerialization(orgId, refuse, fn) {
+        if (!this.runner)
+            return fn();
+        let result;
+        const steps = [
+            {
+                id: 'mutate',
+                run: async () => {
+                    result = await fn();
+                },
+            },
+        ];
+        const current = await this.runner.currentRun(orgId);
+        let runId;
+        if (current) {
+            runId = current.id;
+        }
+        else {
+            try {
+                ({ runId } = await this.runner.enqueueApply(orgId, steps));
             }
-            await this.store.delete(id);
+            catch (e) {
+                if (e instanceof AlreadyRunningError) {
+                    await refuse(e.message);
+                    throw e;
+                }
+                throw e;
+            }
+        }
+        try {
+            await this.runner.runSteps(runId, steps);
         }
         catch (e) {
-            await this.audit.recordDestroy({
-                actor: this.actor,
-                orgId: id,
-                outcome: 'failed',
-                reason: e instanceof Error ? e.message : String(e),
-                at: this.now(),
-            });
+            // `StepFailedError` wraps whatever `fn` threw only to say which run
+            // and step failed; every caller of `apply`/`destroy`/`adopt`/
+            // `release` expects that original error (`StalePlanError`, a
+            // provider error, whatever `fn` itself throws), not the runner's
+            // wrapper around it. `refuse` already ran inside `fn` (every
+            // `withMutationLease` callback audits its own refusal), so this does
+            // not call it again.
+            if (e instanceof StepFailedError && e.cause !== undefined)
+                throw e.cause;
             throw e;
         }
-        await this.audit.recordDestroy({
-            actor: this.actor,
-            orgId: id,
-            outcome: 'destroyed',
-            at: this.now(),
-        });
+        return result;
+    }
+    /**
+     * #441: runs `fn` while holding the one mutation lease for `scope`, so two
+     * `FoundryPlanApply` instances on one store never run provider steps for the
+     * same company or platform at once. A second caller is refused with
+     * {@link MutationInProgressError}; `fn` gets the lease as the fence its
+     * store writes carry, so a worker that lost the lease cannot overwrite the
+     * newer holder's state. Both refusals are audited through `refuse`.
+     */
+    async withMutationLease(scope, refuse, fn) {
+        const lease = await this.store.acquireMutationLease(scope, randomUUID(), this.leaseTtlMs);
+        if (!('token' in lease)) {
+            const held = new MutationInProgressError(scope, lease.heldUntil);
+            await refuse(held.message);
+            throw held;
+        }
+        try {
+            return await fn(lease);
+        }
+        catch (e) {
+            // The paths inside `fn` rethrow a lost lease without auditing it, so this is its one audit row.
+            if (e instanceof StaleLeaseError)
+                await refuse('mutation lease lost to a newer holder');
+            throw e;
+        }
+        finally {
+            // Best effort: an unreleased lease frees itself when it expires.
+            await this.store.releaseMutationLease(lease).catch(() => undefined);
+        }
+    }
+    /** #441: before each provider step, extends the lease; a holder that lost it stops here, before the step runs. */
+    async renewLease(fence) {
+        if (!(await this.store.renewMutationLease(fence, this.leaseTtlMs))) {
+            throw new StaleLeaseError(fence.scope);
+        }
     }
     async refuse(plan, reason) {
         await this.audit.recordApply({
@@ -322,7 +742,7 @@ export class FoundryPlanApply {
             at: this.now(),
         });
     }
-    async runSteps(org, plan, detail) {
+    async runSteps(org, plan, detail, fence) {
         const orgId = plan.orgId;
         const previous = await this.store.getAppliedState(orgId);
         const serviceIds = { ...(previous?.serviceIds ?? {}) };
@@ -343,11 +763,13 @@ export class FoundryPlanApply {
                     checkpoints: [...checkpoints],
                     failedFields: previous?.failedFields ?? [],
                     lastApplied: previous?.lastApplied ?? null,
-                });
+                    destroy: previous?.destroy,
+                }, fence);
             },
         };
         try {
             for (const planned of detail.plannedSteps) {
+                await this.renewLease(fence);
                 const result = await this.executor.execute(planned.action, context);
                 // `system:kind:key`, not just `kind`: two steps can share a `kind`
                 // (an org's several declared ZITADEL apps, #71) and would otherwise
@@ -365,13 +787,20 @@ export class FoundryPlanApply {
             // #9 (resolved): "it stops at the checkpoint... there is no automatic
             // rollback." Whatever completed stays recorded; the failure is
             // rethrown for a human to fix and resume.
-            await this.store.putAppliedState({
+            // A lost lease is audited once, by `withMutationLease`.
+            if (e instanceof StaleLeaseError)
+                throw e;
+            // The failure the caller sees is the step's, not a refused write's.
+            await this.store
+                .putAppliedState({
                 orgId,
                 serviceIds,
                 checkpoints,
                 failedFields: failedFields(org, detail, checkpoints.length, serviceIds),
                 lastApplied: previous?.lastApplied ?? null,
-            });
+                destroy: previous?.destroy,
+            }, fence)
+                .catch(() => undefined);
             await this.audit.recordApply({
                 actor: this.actor,
                 orgId,
@@ -389,8 +818,9 @@ export class FoundryPlanApply {
             checkpoints,
             failedFields: failedFields(org, detail, detail.plannedSteps.length, serviceIds),
             lastApplied,
+            destroy: previous?.destroy,
         };
-        await this.store.putAppliedState(applied);
+        await this.store.putAppliedState(applied, fence);
         // #253: the single-snapshot write just above replaces whatever
         // `lastApplied` this org had; this is the point in today's code where a
         // version is recorded as applied, so it's also where the appended
@@ -588,7 +1018,7 @@ export class FoundryPlanApply {
      * `serviceIds`), so a later plan finds a resource by its recorded id even
      * when a provider's list has not caught up with a create.
      */
-    async runPlatformSteps(platform, plan, detail, executor) {
+    async runPlatformSteps(platform, plan, detail, executor, fence) {
         const { store } = this.requirePlatformDeps();
         const serviceIds = { ...(await store.getServiceIds()) };
         const context = {
@@ -597,10 +1027,11 @@ export class FoundryPlanApply {
             recordScopeId: (key, id) => {
                 serviceIds[key] = id;
             },
-            persist: () => store.putServiceIds(serviceIds),
+            persist: () => store.putServiceIds(serviceIds, fence),
         };
         try {
             for (const planned of detail.plannedSteps) {
+                await this.renewLease(fence);
                 const result = await executor.execute(planned.action, context);
                 if (result?.resourceId) {
                     const { system, kind, key } = planned.action;
@@ -609,8 +1040,12 @@ export class FoundryPlanApply {
             }
         }
         catch (e) {
-            // Whatever completed stays recorded, as an org's apply does.
-            await store.putServiceIds(serviceIds);
+            // Whatever completed stays recorded, as an org's apply does. A lost
+            // lease is audited once, by `withMutationLease`; any other failure
+            // stays the step's, not a refused write's.
+            if (e instanceof StaleLeaseError)
+                throw e;
+            await store.putServiceIds(serviceIds, fence).catch(() => undefined);
             await this.audit.recordApply({
                 actor: this.actor,
                 orgId: platform.ownerOrgId,
@@ -622,7 +1057,7 @@ export class FoundryPlanApply {
             });
             throw e;
         }
-        await store.putServiceIds(serviceIds);
+        await store.putServiceIds(serviceIds, fence);
         await this.audit.recordApply({
             actor: this.actor,
             orgId: platform.ownerOrgId,
@@ -694,10 +1129,14 @@ function destroyTargets(serviceIds) {
         'files:key:',
     ];
     const targets = [];
+    const adopted = adoptedResourceIds(serviceIds);
     for (const prefix of ORDERED_PREFIXES) {
         for (const [resourceKey, id] of Object.entries(serviceIds)) {
             if (!resourceKey.startsWith(prefix))
                 continue;
+            // Archon#271: foundry did not create an adopted app or database, so it never deletes it.
+            if (adopted.has(resourceKey))
+                continue;
             if (prefix === 'ai:key:' && !resourceKey.slice(prefix.length).includes(':'))
                 continue;
             if (prefix === 'files:key:' && !resourceKey.slice(prefix.length).includes(':'))
@@ -726,6 +1165,38 @@ function destroyTargets(serviceIds) {
     }
     return targets;
 }
+/** One step per target, then the declared org's own deletion, which has no target. */
+/** The `serviceIds` keys of the coolify app and database every `adopted.<entry>` record names. */
+function adoptedResourceIds(serviceIds) {
+    // Fail closed: an adopted record that cannot be read could name a resource destroy must keep.
+    const unreadable = adoptedKeysUnreadable(serviceIds);
+    if (unreadable.length > 0) {
+        throw new Error(`destroy refused: the adopted record ${unreadable.join(', ')} cannot be read, so foundry cannot tell which resources it must keep`);
+    }
+    const uuids = new Set();
+    for (const [key, raw] of Object.entries(serviceIds)) {
+        if (!key.startsWith('adopted.'))
+            continue;
+        const snapshot = parseAdoptSnapshot(raw);
+        if (snapshot) {
+            uuids.add(snapshot.application.uuid);
+            uuids.add(snapshot.database.uuid);
+        }
+    }
+    return new Set(Object.entries(serviceIds)
+        .filter(([k, id]) => (k.startsWith('coolify:app:') || k.startsWith('coolify:database:')) && uuids.has(id))
+        .map(([k]) => k));
+}
+/** Lines for adopted resources `destroy` keeps, so the operator sees them beside the deletes. */
+function keptDescriptions(serviceIds) {
+    return [...adoptedResourceIds(serviceIds)].map((k) => `keep ${k.split(':')[0]} ${k.split(':')[1]} ${k.split(':').slice(2).join(':')} (adopted, not deleted)`);
+}
+function destroyStepDescriptions(targets) {
+    return [
+        ...targets.map((t) => `delete ${t.system} ${t.kind} ${t.key}`),
+        'delete the declared org',
+    ];
+}
 /** The `AppliedState.serviceIds` key a completed {@link DestroyTarget} should be removed from. */
 function destroyServiceIdKey(target) {
     if (target.kind === 'project' || target.kind === 'org')
@@ -746,7 +1217,6 @@ const STEP_FIELDS = {
     'coolify:chat': 'chat.usesDomain',
     'coolify:calls': 'calls.enabled',
     'coolify:calls-service': 'calls.enabled',
-    'files:storage': 'storage',
     'ai:budget': 'ai',
     'ai:key': 'ai',
     'cms:site': 'site',