@nocobase/app-plugin-releases 0.1.0-beta.2 → 0.1.0-beta.21

package/dist/server/drivers/host/driver.js

@@ -19,7 +19,8 @@ import { createDiagnosticLogger, } from '@nocobase/logging';
 import { stringify as stringifyYaml } from 'yaml';
 import { ACCESS_NAMESPACE } from '../../../shared/access.js';
 import { writeTextAtomic } from '../../services/config-file.js';
-import { expandPublicUrl, } from '../types.js';
+export { UNRESTORED_APP_ERROR } from '../types.js';
+import { expandPublicUrl, UNRESTORED_APP_ERROR, } from '../types.js';
 /** The backend of Apps that run in the Host process. */
 export const IN_PROCESS_BACKEND = 'in-process';
 export const DOCKER_BACKEND = 'docker';
@@ -61,6 +62,14 @@ export function createHostDriver(options) {
     const inProcess = options.hosts[IN_PROCESS_BACKEND];
     if (!inProcess)
         throw new Error('The Host driver needs the in-process Host.');
+    for (const [name, value] of Object.entries({
+        restartMaxWaitMs: inProcess.restartMaxWaitMs,
+        restartDrainMs: inProcess.restartDrainMs,
+    })) {
+        if (value !== undefined &&
+            (!Number.isInteger(value) || value < 0 || value > 2_147_483_647))
+            throw new Error(`Invalid ${name}: expected a non-negative timer interval`);
+    }
     const scopes = new Map();
     /** The latest revision sent per scope; the Host refuses an older one. */
     const revisions = new Map();
@@ -72,7 +81,23 @@ export function createHostDriver(options) {
     let runtimeChurn = 0;
     let inFlight = 0;
     let restartScheduled = null;
+    let rotationPreparing = null;
+    let readyRestoring = null;
     let churnTimer = null;
+    let restartDeadline = null;
+    let checkingChurn = false;
+    let recheckChurn = false;
+    let retryCheckAt = 0;
+    const capacityWaiting = new Set();
+    const batchBusy = () => [...scopes.values()].some((scope) => scope.host === inProcess &&
+        scope.deploymentBatchBusy?.(capacityWaiting));
+    let hostGeneration = 0;
+    let rotationSince = null;
+    let capacityRotation = false;
+    const rotationWaiters = new Set();
+    const heapRatio = inProcess.restartHeapRatio ?? (inProcess.restartAfterChurn ? 0.55 : 0);
+    if (!Number.isFinite(heapRatio) || heapRatio < 0 || heapRatio >= 1)
+        throw new Error('Invalid Host restart heap ratio');
     const hostOf = (backend) => {
         const host = options.hosts[backend];
         if (!host)
@@ -164,18 +189,11 @@ export function createHostDriver(options) {
                 : {}),
         };
     };
-    const track = (scope, spec) => {
-        if (spec.activation === 'onDemand')
-            scope.onDemand.add(spec.appId);
-        else
-            scope.onDemand.delete(spec.appId);
-    };
     /** Sends one environment's complete desired set as a new revision of its scope. */
     const sendSet = async (scope) => {
         const management = await managementOf(scope.host);
         const specs = await scope.desired();
-        for (const spec of specs)
-            track(scope, spec);
+        scope.desiredAppIds = new Set(specs.map((spec) => spec.appId));
         const deployments = specs.map((spec) => toHostSpec(scope, spec));
         const send = async (revision) => {
             revisions.set(scope.environmentId, revision);
@@ -191,8 +209,22 @@ export function createHostDriver(options) {
             await send(first.revision + 1);
     };
     const scheduleRestore = (scope) => {
+        if (scope.restoring)
+            return scope.restoring;
         const current = sendSet(scope)
-            .catch((error) => diagnostic.error(`Failed to restore the Apps of environment ${scope.environmentId}`, error))
+            .then(() => {
+            scope.restoreFailed = false;
+            scope.restoreError = null;
+        })
+            .catch((error) => {
+            scope.restoreFailed = true;
+            scope.restoreError =
+                error instanceof Error ? error.message : String(error);
+            if (scopes.get(scope.environmentId) === scope)
+                watchChurn(true);
+            diagnostic.error(`Failed to restore the Apps of environment ${scope.environmentId}`, error);
+            throw error;
+        })
             .finally(() => {
             if (scope.restoring === current)
                 scope.restoring = null;
@@ -201,28 +233,49 @@ export function createHostDriver(options) {
         return current;
     };
     const restoreHost = async (host) => {
+        const failures = [];
         for (const scope of scopes.values())
             if (scope.host === host)
-                await scheduleRestore(scope);
+                await scheduleRestore(scope).catch((error) => {
+                    failures.push(error);
+                });
+        // A broken environment must not prevent the remaining environments from being restored.
+        if (failures.length)
+            throw failures[0];
     };
     const watchReady = (host) => {
         if (watched.has(host))
             return;
         watched.add(host);
         host.controller.onReady(() => {
-            void restoreHost(host);
+            if (host === inProcess && !restartScheduled) {
+                hostGeneration += 1;
+                churn = 0;
+                runtimeChurn = 0;
+                if (restartDeadline)
+                    clearTimeout(restartDeadline);
+                restartDeadline = null;
+                rotationSince = null;
+                capacityRotation = false;
+            }
+            if (!restartScheduled) {
+                const restoring = restoreHost(host);
+                if (host === inProcess)
+                    readyRestoring = restoring;
+                void restoring
+                    .catch(() => undefined)
+                    .finally(() => {
+                    if (readyRestoring === restoring)
+                        readyRestoring = null;
+                });
+            }
         });
     };
     // --- Recycling the in-process Host (`restartAfterChurn`) ---------------------------------------------------------
-    /** Each of the Host's open scopes with its status. */
-    const statusesOf = async (host) => {
-        const management = await managementOf(host);
-        return await Promise.all([...scopes.values()]
-            .filter((scope) => scope.host === host)
-            .map(async (scope) => [
-            scope,
-            await management.getStatus({ scope: scope.environmentId }),
-        ]));
+    const finishRotation = (ok) => {
+        for (const resolve of rotationWaiters)
+            resolve(ok);
+        rotationWaiters.clear();
     };
     const restartHost = async (reason) => {
         const { controller } = inProcess;
@@ -230,10 +283,28 @@ export function createHostDriver(options) {
             throw new Error('This Host runtime cannot be restarted by the driver.');
         restartScheduled ??= (async () => {
             try {
-                await controller.restart(reason);
+                hostGeneration += 1;
+                if (restartDeadline)
+                    clearTimeout(restartDeadline);
+                restartDeadline = null;
+                await controller.restart(reason, {
+                    shutdownTimeoutMs: inProcess.restartDrainMs ?? 30_000,
+                });
                 churn = 0;
                 runtimeChurn = 0;
+                rotationSince = null;
+                capacityRotation = false;
                 await restoreHost(inProcess);
+                finishRotation(true);
+            }
+            catch (error) {
+                // If restart failed before the child exited, serving traffic can resume. New revisions remain paused.
+                const service = await managementOf(inProcess).catch(() => null);
+                await service
+                    ?.prepareRotation?.({ resume: true })
+                    .catch(() => undefined);
+                finishRotation(false);
+                throw error;
             }
             finally {
                 restartScheduled = null;
@@ -241,61 +312,190 @@ export function createHostDriver(options) {
         })();
         await restartScheduled;
     };
-    /** Whether an on-demand App runs or starts, which a restart would cut off. */
-    const onDemandBusy = (statuses) => statuses.some(([scope, status]) => status.deployments.some((deployment) => {
-        const state = observed(deployment).state;
-        return (scope.onDemand.has(deployment.appId) &&
-            (state === 'running' || state === 'starting'));
-    }));
-    /**
-     * Restarts the in-process Host when the churn reached `restartAfterChurn`, nothing is in flight and no on-demand App
-     * is in use; otherwise a later check tries again.
-     */
     const maybeRestart = async () => {
-        const limit = inProcess.restartAfterChurn;
         const { controller } = inProcess;
-        if (!limit || !controller.restart || restartScheduled)
+        if (restartScheduled || rotationPreparing)
             return;
-        if (controller.getInfo().status !== 'ready')
+        if (controller.getInfo().status !== 'ready' || Date.now() < retryCheckAt)
             return;
-        const statuses = await statusesOf(inProcess).catch(() => null);
-        const counters = statuses?.find(([, status]) => status.counters)?.[1]
-            .counters;
-        if (counters)
-            runtimeChurn = (counters.idleStops ?? 0) + (counters.dormancies ?? 0);
+        if ((!inProcess.restartAfterChurn && !heapRatio && !capacityRotation) ||
+            !controller.restart)
+            return;
+        if (batchBusy() && rotationSince === null && !capacityRotation)
+            return;
+        const generation = hostGeneration;
+        const service = await managementOf(inProcess);
+        // The unfiltered status includes resident, staging and unscoped Apps sharing the process.
+        const status = await service.getStatus();
+        if (restartScheduled || generation !== hostGeneration)
+            return;
+        const counters = status.counters;
+        runtimeChurn =
+            (counters?.idleStops ?? 0) +
+                (counters?.dormancies ?? 0) +
+                (counters?.capacityStops ?? 0) +
+                (counters?.startupTimeouts ?? 0);
         const total = churn + runtimeChurn;
-        if (total < limit || inFlight > 0)
+        const memory = counters?.memory;
+        const threshold = total > 0 &&
+            (capacityRotation ||
+                (inProcess.restartAfterChurn && total >= inProcess.restartAfterChurn) ||
+                (heapRatio > 0 &&
+                    memory &&
+                    memory.heapUsed >= memory.heapLimit * heapRatio));
+        if (!threshold && rotationSince === null)
+            return;
+        rotationSince ??= Date.now();
+        const wait = inProcess.restartMaxWaitMs ?? 5 * 60_000;
+        const expired = Date.now() - rotationSince >= wait;
+        if (!expired && !restartDeadline) {
+            restartDeadline = setTimeout(() => {
+                restartDeadline = null;
+                checkChurn();
+            }, Math.max(1, wait - (Date.now() - rotationSince)));
+            restartDeadline.unref?.();
+        }
+        if (expired)
+            await service.prepareRotation?.({ pauseOnly: true });
+        if (inFlight > 0 ||
+            batchBusy() ||
+            [...scopes.values()].some((scope) => scope.host === inProcess && scope.restoring !== null) ||
+            (counters?.operationsInFlight ?? 0) > 0)
+            return;
+        const busy = status.deployments.some((deployment) => deployment.lifecycle?.state === 'starting' ||
+            (!expired && (deployment.app?.activeRequests ?? 0) > 0) ||
+            (!expired && (deployment.app?.activeConnections ?? 0) > 0));
+        if (busy)
             return;
-        if (statuses && onDemandBusy(statuses))
+        // Before the deadline reserve only an idle child. At the deadline gate ingress first, then give existing
+        // responses a bounded drain. The old proxy target remains available to return 503 + Retry-After while draining.
+        if (!service.prepareRotation)
             return;
-        await restartHost(`${total} applications replaced, removed, stopped or made dormant`);
+        const preparing = service.prepareRotation({
+            ignoreConnections: expired,
+            ...(expired
+                ? { drainTimeoutMs: inProcess.restartDrainMs ?? 30_000 }
+                : {}),
+        });
+        rotationPreparing = preparing;
+        let reserved;
+        try {
+            reserved = await preparing;
+        }
+        catch (error) {
+            await service.prepareRotation({ resume: true }).catch(() => undefined);
+            throw error;
+        }
+        finally {
+            if (rotationPreparing === preparing)
+                rotationPreparing = null;
+        }
+        if (!reserved)
+            return;
+        // New tracked work waits for the handoff. New control-plane dispatches have not entered the Host yet;
+        // the batch was settled before the handoff began. Existing operations remain protected by the child recheck.
+        if (inFlight > 0 || generation !== hostGeneration) {
+            await service.prepareRotation({ resume: true });
+            return;
+        }
+        await restartHost(`${total} applications deployed, replaced, removed, stopped or made dormant; recycling Host memory`);
     };
     const checkChurn = () => {
-        void maybeRestart().catch((error) => diagnostic.error('Failed to restart the App Host', error));
+        if (checkingChurn) {
+            recheckChurn = true;
+            return;
+        }
+        checkingChurn = true;
+        void maybeRestart()
+            .catch((error) => {
+            retryCheckAt = Date.now() + (options.churnCheckIntervalMs ?? 60_000);
+            diagnostic.error('Failed to recycle the App Host', error);
+        })
+            .finally(() => {
+            checkingChurn = false;
+            // Retry failed restores after evaluating rotation, never as a foreground management prerequisite. A pending
+            // restore remains protected work, including the interval spent reading its desired set before Host submission.
+            if (!inFlight &&
+                !batchBusy() &&
+                !rotationPreparing &&
+                !restartScheduled &&
+                inProcess.controller.getInfo().status === 'ready' &&
+                Date.now() >= retryCheckAt)
+                for (const scope of scopes.values())
+                    if (scope.restoreFailed && !scope.restoring)
+                        void scheduleRestore(scope).catch(() => undefined);
+            if (recheckChurn) {
+                recheckChurn = false;
+                queueMicrotask(checkChurn);
+            }
+        });
     };
     const countChurn = (scope) => {
         if (scope.host !== inProcess)
             return;
         churn += 1;
-        checkChurn();
     };
-    const watchChurn = () => {
-        if (churnTimer || !inProcess.restartAfterChurn)
+    const watchChurn = (forRecovery = false) => {
+        if (churnTimer ||
+            (!forRecovery && !inProcess.restartAfterChurn && !heapRatio))
             return;
         churnTimer = setInterval(checkChurn, options.churnCheckIntervalMs ?? 60_000);
         churnTimer.unref?.();
     };
+    /** Several failed admissions share one rotation, never hold inFlight while waiting for it. */
+    const recycleForCapacity = async (signal) => {
+        if (!inProcess.controller.restart || signal.aborted)
+            return false;
+        const status = await (await managementOf(inProcess)).getStatus();
+        const counters = status.counters;
+        runtimeChurn =
+            (counters?.idleStops ?? 0) +
+                (counters?.dormancies ?? 0) +
+                (counters?.capacityStops ?? 0) +
+                (counters?.startupTimeouts ?? 0);
+        if (churn + runtimeChurn <= 0)
+            return false;
+        capacityRotation = true;
+        return new Promise((resolve) => {
+            const done = (ok) => {
+                clearTimeout(timer);
+                rotationWaiters.delete(done);
+                signal.removeEventListener('abort', cancelled);
+                resolve(ok);
+            };
+            const cancelled = () => done(false);
+            signal.addEventListener('abort', cancelled, { once: true });
+            const timer = setTimeout(() => done(false), (inProcess.restartMaxWaitMs ?? 300_000) +
+                2 * (inProcess.restartDrainMs ?? 30_000) +
+                (options.churnCheckIntervalMs ?? 60_000));
+            rotationWaiters.add(done);
+            if (signal.aborted)
+                cancelled();
+            else
+                checkChurn();
+        });
+    };
     const tracked = async (scope, work) => {
         if (scope.host !== inProcess)
             return await work();
+        if (rotationPreparing)
+            await rotationPreparing.catch(() => false);
         if (restartScheduled)
             await restartScheduled.catch(() => undefined);
+        if (readyRestoring)
+            await readyRestoring.catch(() => undefined);
+        // A pending replay may still hold an older desired set. Let it finish before submitting any mutation.
+        // Failure leaves management usable; only the background poll initiates another recovery attempt.
+        if (scope.restoring)
+            await scope.restoring.catch(() => undefined);
         inFlight += 1;
         try {
             return await work();
         }
         finally {
             inFlight -= 1;
+            if (inFlight === 0)
+                queueMicrotask(checkChurn);
         }
     };
     const capabilitiesOf = (backend) => HOST_BACKEND_CAPABILITIES[backend] ??
@@ -343,6 +543,10 @@ export function createHostDriver(options) {
         churn: () => churn + runtimeChurn,
         restartHost: (reason = 'restart requested') => restartHost(reason),
         proxyTarget() {
+            // Only the initial replay after the child becomes ready gates ingress. Failed environments and their
+            // background retries must not take healthy applications on this shared Host offline.
+            if (restartScheduled || readyRestoring)
+                return null;
             const info = inProcess.controller.getInfo();
             return info.status === 'ready' && info.targetUrl
                 ? new URL(info.targetUrl)
@@ -361,11 +565,15 @@ export function createHostDriver(options) {
                 scope: scopeOf(environment, settings),
                 publicUrl: environment.publicUrl,
                 desired: () => context.desired(),
-                onDemand: new Set(),
                 restoring: null,
+                restoreFailed: false,
+                restoreError: null,
+                desiredAppIds: new Set(),
+                deploymentBatchBusy: (excluding) => context.deploymentBatchBusy?.(excluding) ?? false,
             };
             scopes.set(environment.id, scope);
             watchReady(host);
+            // Recovery is retried by the background poll. A broken desired set must not block management or repair.
             const management = () => managementOf(host);
             const statusOf = async (appId) => {
                 const status = await (await management()).getStatus({ scope: environment.id, appIds: [appId] });
@@ -380,7 +588,9 @@ export function createHostDriver(options) {
                         error: null,
                     };
             };
+            const pendingDeployments = new Map();
             return {
+                deploymentSettled: checkChurn,
                 async check() {
                     try {
                         const url = (await host.controller.ensureStarted()).toString();
@@ -398,57 +608,122 @@ export function createHostDriver(options) {
                         };
                     }
                 },
+                async cancel(deploymentId, appId, queuedOnly) {
+                    const pending = pendingDeployments.get(deploymentId);
+                    if (pending && pending.appId === appId && !pending.submitted) {
+                        pending.cancelled = true;
+                        pending.abortController.abort();
+                        return 'cancelled';
+                    }
+                    return (await management()).cancelDeployment(deploymentId, appId, {
+                        scope: scope.scope,
+                        queuedOnly,
+                    });
+                },
+                async progress(deploymentId, appId) {
+                    if (pendingDeployments.get(deploymentId)?.appId === appId &&
+                        capacityWaiting.has(deploymentId))
+                        return { state: 'queued', queueAhead: 0, cancellable: true };
+                    return (await management()).deploymentProgress(deploymentId, appId, {
+                        scope: scope.scope,
+                    });
+                },
                 async apply(spec, onEvent) {
-                    track(scope, spec);
-                    let replacing = false;
+                    const pending = {
+                        appId: spec.appId,
+                        submitted: false,
+                        cancelled: false,
+                        abortController: new AbortController(),
+                    };
+                    pendingDeployments.set(spec.deploymentId, pending);
+                    let sequence = 0;
+                    const forward = (entry) => {
+                        onEvent?.({ ...entry, sequence: ++sequence });
+                    };
                     try {
-                        const status = await tracked(scope, async () => {
+                        const submit = (retryCapacity) => tracked(scope, async () => {
                             const service = await management();
-                            const before = await service
-                                .getStatus({ scope: environment.id, appIds: [spec.appId] })
-                                .catch(() => null);
-                            replacing = Boolean(before?.deployments.some((deployment) => deployment.app?.state === 'active'));
-                            // A deployment always starts the App once, so it proves the release runs.
-                            return await service.applyDeployment({
+                            if (pending.cancelled)
+                                throw Object.assign(new Error('Deployment cancelled.'), {
+                                    code: 'DEPLOYMENT_CANCELLED',
+                                });
+                            pending.submitted = true;
+                            const status = await service.applyDeployment({
                                 ...toHostSpec(scope, spec, 'running', 'eager'),
                                 scope: scope.scope,
-                            }, onEvent ? (entry) => onEvent(entry) : undefined);
+                                ...(retryCapacity ? { retryCapacity: true } : {}),
+                            }, onEvent ? forward : undefined);
+                            if (status.deployments.some((deployment) => deployment.appId === spec.appId &&
+                                deployment.observedState !== 'failed'))
+                                countChurn(scope);
+                            return status;
                         });
-                        if (replacing)
-                            countChurn(scope);
+                        let status = await submit(false);
+                        const capacity = status.deployments.find((deployment) => deployment.appId === spec.appId)?.errorCode === 'APP_CAPACITY_EXCEEDED';
+                        if (capacity && scope.host === inProcess) {
+                            pending.submitted = false;
+                            capacityWaiting.add(spec.deploymentId);
+                            forward({
+                                phase: 'queued',
+                                msg: 'Waiting for Host memory recycling',
+                                queueAhead: 0,
+                                cancellable: true,
+                            });
+                            try {
+                                if (await recycleForCapacity(pending.abortController.signal)) {
+                                    capacityWaiting.delete(spec.deploymentId);
+                                    status = await submit(true);
+                                }
+                            }
+                            finally {
+                                capacityWaiting.delete(spec.deploymentId);
+                            }
+                            if (pending.cancelled)
+                                throw Object.assign(new Error('Deployment cancelled.'), {
+                                    code: 'DEPLOYMENT_CANCELLED',
+                                });
+                        }
                         const deployed = status.deployments.find((deployment) => deployment.appId === spec.appId);
                         if (!deployed)
                             return failed(spec, 'The Host did not report the deployment.');
+                        scope.desiredAppIds.add(spec.appId);
                         return outcomeOf(deployed, spec.deploymentId);
                     }
                     catch (error) {
+                        if (error.code === 'DEPLOYMENT_CANCELLED')
+                            return {
+                                ...failed(spec, 'Deployment cancelled.'),
+                                state: 'cancelled',
+                            };
                         return failed(spec, error instanceof Error ? error.message : String(error));
                     }
+                    finally {
+                        pendingDeployments.delete(spec.deploymentId);
+                    }
                 },
                 async start(spec) {
-                    track(scope, spec);
-                    await (await management()).startDeployment({
+                    await tracked(scope, async () => (await management()).startDeployment({
                         ...toHostSpec(scope, spec, 'running'),
                         scope: scope.scope,
-                    });
+                    }));
                     return await statusOf(spec.appId);
                 },
                 async stop(appId) {
-                    await (await management()).stopDeployment(appId);
+                    await tracked(scope, async () => (await management()).stopDeployment(appId));
                     return await statusOf(appId);
                 },
                 async restart(appId) {
-                    await (await management()).restartApp(appId);
+                    await tracked(scope, async () => (await management()).restartApp(appId));
                     return await statusOf(appId);
                 },
                 async remove(appId, { purgeData }) {
-                    await tracked(scope, async () => (await management()).removeDeployment(appId, { purgeData }));
-                    scope.onDemand.delete(appId);
-                    countChurn(scope);
+                    await tracked(scope, async () => {
+                        await (await management()).removeDeployment(appId, { purgeData });
+                        scope.desiredAppIds.delete(appId);
+                        countChurn(scope);
+                    });
                 },
                 async status(appIds) {
-                    if (scope.restoring)
-                        await Promise.race([scope.restoring, delay(5_000)]);
                     const status = await (await management()).getStatus({
                         scope: environment.id,
                         ...(appIds ? { appIds: [...appIds] } : {}),
@@ -456,6 +731,18 @@ export function createHostDriver(options) {
                     const result = new Map();
                     for (const deployment of status.deployments)
                         result.set(deployment.appId, observed(deployment));
+                    // A failed restore left these Apps off the Host until a background retry succeeds.
+                    if (scope.restoreFailed && !scope.restoring && appIds)
+                        for (const appId of appIds)
+                            if (!result.has(appId))
+                                result.set(appId, {
+                                    state: 'failed',
+                                    version: null,
+                                    deploymentId: null,
+                                    startedAt: null,
+                                    error: UNRESTORED_APP_ERROR,
+                                    unrestored: true,
+                                });
                     if ((scope.restoring || !status.scope) && appIds)
                         for (const appId of appIds)
                             if (!result.has(appId))
@@ -469,11 +756,61 @@ export function createHostDriver(options) {
                     return result;
                 },
                 async restore() {
-                    await host.controller.ensureStarted();
-                    await scheduleRestore(scope);
+                    await tracked(scope, async () => {
+                        await host.controller.ensureStarted();
+                        await scheduleRestore(scope);
+                    });
                 },
                 async reloadConfig(appId, content) {
-                    await (await management()).publishAppConfig(appId, content);
+                    await tracked(scope, async () => (await management()).publishAppConfig(appId, content));
+                },
+                async occupancy() {
+                    if (host !== inProcess)
+                        return null;
+                    // The unfiltered status: every scope sharing the process, and Apps deployed without one.
+                    const status = await (await management()).getStatus();
+                    const failedScopes = [...scopes.values()].filter((other) => other.host === inProcess &&
+                        other.restoreFailed &&
+                        !other.restoring);
+                    const present = new Set(status.deployments.map((deployment) => `${deployment.scopeId ?? ''}\u0000${deployment.appId}`));
+                    // A failed restore leaves an environment's Apps off the Host until a background retry succeeds: they are
+                    // still deployed, and failed, as `status()` reports them. Apps still on the Host keep their own state.
+                    const missing = failedScopes.flatMap((other) => [...other.desiredAppIds]
+                        .filter((appId) => !present.has(`${other.environmentId}\u0000${appId}`))
+                        .map((appId) => ({
+                        state: 'failed',
+                        version: null,
+                        deploymentId: null,
+                        startedAt: null,
+                        error: UNRESTORED_APP_ERROR,
+                        unrestored: true,
+                        appId,
+                        environmentId: other.environmentId,
+                    })));
+                    return {
+                        id: IN_PROCESS_BACKEND,
+                        memory: status.counters?.memory ?? null,
+                        maxActive: status.counters?.maxActiveApps ?? null,
+                        recycling: restartScheduled ||
+                            readyRestoring ||
+                            inProcess.controller.getInfo().status !== 'ready'
+                            ? 'restarting'
+                            : rotationSince !== null || status.counters?.rotationPending
+                                ? 'pending'
+                                : null,
+                        apps: [
+                            ...status.deployments.map((deployment) => ({
+                                ...observed(deployment),
+                                appId: deployment.appId,
+                                environmentId: deployment.scopeId ?? null,
+                            })),
+                            ...missing,
+                        ],
+                        failedEnvironments: failedScopes.map((other) => ({
+                            environmentId: other.environmentId,
+                            error: other.restoreError,
+                        })),
+                    };
                 },
                 async logs(appId, query) {
                     return await (await management()).readAppLogs(appId, query);
@@ -524,6 +861,10 @@ function observed(item) {
                 item.lifecycle?.lastFailure?.error ??
                 null),
         lastAccessedAt: item.lifecycle?.lastAccessedAt ?? null,
+        ...(item.lifecycle?.holds?.length ? { holds: item.lifecycle.holds } : {}),
+        ...(item.lifecycle?.waiting && !active
+            ? { waiting: item.lifecycle.waiting }
+            : {}),
     };
 }
 /** How a deployment ended, as the plugin records it: `running` succeeded, anything else failed. */
@@ -535,7 +876,10 @@ function outcomeOf(item, deploymentId) {
             ...status,
             state: 'failed',
             deploymentId,
-            error: item.error ?? status.error ?? 'The deployment failed.',
+            error: item.errorCode
+                ? `[${item.errorCode}] ${item.error ?? status.error ?? 'The deployment failed.'}`
+                : (item.error ?? status.error ?? 'The deployment failed.'),
+            ...(item.errorCode ? { errorCode: item.errorCode } : {}),
         };
     return { ...status, state: 'running', deploymentId, error: null };
 }
@@ -550,11 +894,14 @@ function operationOutcome(result) {
             lastAccessedAt: result.status?.lifecycle?.lastAccessedAt ?? null,
         };
     return {
-        state: 'failed',
+        state: result.state === 'cancelled' ? 'cancelled' : 'failed',
+        ...(result.status?.errorCode ? { errorCode: result.status.errorCode } : {}),
         version: result.status?.version ?? null,
         deploymentId: result.operationId,
         startedAt: null,
-        error: result.error ??
+        error: (result.state === 'interrupted'
+            ? 'Deployment was interrupted by a restart. Deploy the release again.'
+            : result.error) ??
             (result.state === 'running'
                 ? `The deployment of ${result.appId} has not finished on the Host.`
                 : 'The deployment failed.'),
@@ -597,16 +944,15 @@ function unionCapabilities(list) {
         images: list.some((item) => item.images === true),
     };
 }
-function delay(ms) {
-    return new Promise((resolve) => {
-        setTimeout(resolve, ms).unref?.();
-    });
-}
 /** Writes a managed Host child's configuration file (`AppHostSupervisor` `configPath`) before it starts. */
 export async function writeHostConfig(configPath, config) {
     await writeTextAtomic(configPath, stringifyYaml({
         host: {
             mode: 'managed',
+            ...(config.memoryBudget ? { memoryBudget: config.memoryBudget } : {}),
+            ...(config.maxActiveApps !== undefined
+                ? { maxActiveApps: config.maxActiveApps }
+                : {}),
             ...(config.logging ? { logging: config.logging } : {}),
             server: { host: '127.0.0.1', port: 3000 },
             artifact: config.artifact.driver === 'fs'