@nocobase/app-plugin-releases 0.1.0-beta.2 → 0.1.0-beta.21

package/dist/server/services/releases.js

@@ -13,9 +13,9 @@ import { createDriveManager, } from '@nocobase/drive';
 import { appendJournal, createDiagnosticLogger, pruneJournals, readJournal, reportLoggingFailure, } from '@nocobase/logging';
 import { parse as parseYaml } from 'yaml';
 import { BUSINESS_ACTIONS } from '../../shared/access.js';
-import { RESERVED_VARIABLE_NAMES } from '../../shared/releases.js';
+import { RESERVED_VARIABLE_NAMES, runStateOf, } from '../../shared/releases.js';
 import { SYSTEM_CALLER, actorOf, } from '../access/caller.js';
-import { capabilitiesFor, } from '../drivers/types.js';
+import { capabilitiesFor, UNRESTORED_APP_ERROR, } from '../drivers/types.js';
 import { ReleasesError, forbidden, notFound } from '../errors.js';
 import { RELEASE_VERSION_PATTERN, assertMountableAt, inspectArtifact, receiveArtifact, validateIdempotencyKey, } from './artifact.js';
 import { decodeDate, decodeLabels, decodeOptionalDate, decodeRecord, iso, matchesLabels, nullableNumber, nullableString, } from './codec.js';
@@ -24,10 +24,17 @@ import { maskConfig, restoreConfigSecrets } from './config-secrets.js';
 import { ReleaseArtifactStore, normalizeImage, } from './release-artifacts.js';
 import { decryptText, encryptText } from './secrets.js';
 import { DEFAULT_INITIAL_ADMIN, INITIAL_ADMIN_PATHS, PUBLIC_ORIGIN_PATH, SAMPLE_DATA_PATH, appVariableView, assertVariableName, assertVariableValue, configValueAt, decodeVariablesManifest, generateVariable, hasRealValue, isSecretName, parseVariablesManifest, randomPassword, resolveVariables, } from './variables.js';
-import { APP_ID_PATTERN, assertAppId, assertPagination, normalizeLabels, normalizeName, normalizeRuntimePolicy, } from './validation.js';
+import { APP_ID_PATTERN, assertAppId, assertPagination, normalizeLabels, normalizeName, normalizeRuntimePolicy, releaseArtifactKey, } from './validation.js';
 const DEFAULT_STARTUP_WAIT_MS = 5_000;
+/** `<appId>-<version>.tar.gz`, with anything but letters, digits, `.`, `_` and `-` replaced. */
+export function archiveFileName(appId, version) {
+    const safe = (text) => text.replace(/[^A-Za-z0-9._-]+/gu, '_').replace(/^\.+/u, '') || 'release';
+    return `${safe(appId)}-${safe(version)}.tar.gz`;
+}
 export class ReleasesService {
     options;
+    dispatch = new Map();
+    progress = new Map();
     diagnostic;
     disk;
     artifacts;
@@ -38,6 +45,9 @@ export class ReleasesService {
     runs = new Map();
     restoring = null;
     closed = false;
+    artifactKey(appId, releaseId) {
+        return releaseArtifactKey(appId, releaseId, this.options.artifactPrefix);
+    }
     constructor(options) {
         this.options = options;
         this.diagnostic = createDiagnosticLogger(options.logger);
@@ -72,8 +82,12 @@ export class ReleasesService {
             query = query.where((eb) => searchIds.length > 0
                 ? eb('id', 'in', [...searchIds])
                 : eb('id', 'is', null));
-        if (options.environmentId)
-            query = query.where('environmentId', '=', options.environmentId);
+        const environmentIds = [
+            ...(options.environmentId ? [options.environmentId] : []),
+            ...(options.environmentIds ?? []),
+        ];
+        if (environmentIds.length > 0)
+            query = query.where('environmentId', 'in', [...new Set(environmentIds)]);
         const only = filter.only;
         if (only)
             query = query.where((eb) => only.length > 0 ? eb('id', 'in', [...only]) : eb('id', 'is', null));
@@ -85,13 +99,27 @@ export class ReleasesService {
             ]));
         }
         // Labels are opaque JSON, matched here rather than in SQL.
-        const rows = (await query.execute())
+        let rows = (await query.execute())
             .map(decodeApp)
             .filter((app) => matchesLabels(app.labels, options.labels));
+        // A state filter needs every candidate's runtime before the page is cut: one batched status call per environment,
+        // reused for the page's summaries.
+        let statuses;
+        if (options.runStates?.length) {
+            const wanted = new Set(options.runStates);
+            statuses = await this.runtimeStatuses(rows);
+            const known = statuses;
+            rows = rows.filter((app) => wanted.has(runStateOf(runtimeStatus(known.get(app.environmentId), app.id))));
+        }
         const total = rows.length;
         const page = Math.min(requestedPage, Math.max(1, Math.ceil(total / pageSize)));
         const slice = rows.slice((page - 1) * pageSize, page * pageSize);
-        return { items: await this.summarize(slice), total, page, pageSize };
+        return {
+            items: await this.summarize(slice, statuses),
+            total,
+            page,
+            pageSize,
+        };
     }
     async getApp(caller, appId) {
         const app = await this.requireApp(appId);
@@ -101,10 +129,21 @@ export class ReleasesService {
             if (action !== 'create' &&
                 (await this.options.guard.canApp(caller, action, app)))
                 allowed.push(action);
-        const variables = await this.appVariablesOf(app).catch(() => null);
+        const [variables, shared] = await Promise.all([
+            this.appVariablesOf(app).catch(() => null),
+            this.hostOf(caller, app).catch(() => null),
+        ]);
+        const summary = (await this.summarize([app]))[0];
+        // A driver reporting an environment's restore error on its Apps would reveal the others it names: it is the
+        // environment's diagnosis, for those who may read it (`hostOf`).
+        const runtime = summary.runtime?.error && shared?.hiddenErrors.has(summary.runtime.error)
+            ? { ...summary.runtime, error: UNRESTORED_APP_ERROR }
+            : summary.runtime;
         return {
-            ...(await this.summarize([app]))[0],
+            ...summary,
+            runtime,
             allowed,
+            host: shared?.view ?? null,
             ...(variables
                 ? {
                     variables: {
@@ -115,7 +154,97 @@ export class ReleasesService {
                 : {}),
         };
     }
-    async createApp(caller, input) {
+    /**
+     * The runtime the App shares: one status call to the driver. Its counts and memory are totals of the whole runtime;
+     * no other App is named, and an environment's restore error only reaches those who manage environments.
+     */
+    async hostOf(caller, app) {
+        const session = await this.session(app.environmentId);
+        const occupancy = (await session.occupancy?.()) ?? null;
+        if (!occupancy)
+            return null;
+        return await this.hostView(caller, occupancy);
+    }
+    /**
+     * The runtimes the environments with Apps share, each once however many environments run on it, for the
+     * environments page: one driver status call per such environment, then what `hostOf` reads per runtime. Requires the
+     * `rel.environments` `read` setting; restore errors are withheld as on an App's page.
+     */
+    async listHosts(caller) {
+        this.options.guard.requireSetting(caller, 'rel.environments', 'read');
+        // The sessions restore opened already: an environment without Apps runs nothing on any runtime.
+        const rows = await this.query()
+            .selectFrom('relApps')
+            .select('environmentId')
+            .distinct()
+            .execute();
+        const found = await Promise.all(rows.map(async (row) => {
+            const environmentId = String(row.environmentId);
+            const occupancy = await this.session(environmentId)
+                .then((session) => session.occupancy?.() ?? null)
+                .catch(() => null);
+            return occupancy ? { environmentId, occupancy } : null;
+        }));
+        const hosts = new Map();
+        for (const item of found) {
+            if (!item)
+                continue;
+            const host = hosts.get(item.occupancy.id);
+            if (host)
+                host.environmentIds.push(item.environmentId);
+            else
+                hosts.set(item.occupancy.id, {
+                    occupancy: item.occupancy,
+                    environmentIds: [item.environmentId],
+                });
+        }
+        const views = [];
+        for (const { occupancy, environmentIds } of hosts.values()) {
+            const { view } = await this.hostView(caller, occupancy);
+            const environments = [];
+            for (const id of environmentIds.sort())
+                environments.push({
+                    id,
+                    name: (await this.options.environments.find(id))?.name ?? id,
+                    // Counted as the host's `running`, so the environment's Apps filtered to running or starting are as many.
+                    running: occupancy.apps.filter((item) => item.environmentId === id &&
+                        (item.state === 'running' || item.state === 'starting')).length,
+                });
+            views.push({ ...view, environments });
+        }
+        return views.sort((a, b) => a.id.localeCompare(b.id));
+    }
+    /** What `occupancy` says of the runtime, limited to what the caller may read. */
+    async hostView(caller, occupancy) {
+        const environmentNames = new Map();
+        for (const item of occupancy.failedEnvironments)
+            environmentNames.set(item.environmentId, (await this.options.environments.find(item.environmentId))?.name ??
+                null);
+        // An environment's restore error may name any of its Apps or their paths: only those who manage environments
+        // read it; everyone else learns that the restore failed.
+        const diagnoses = this.options.guard.hasSetting(caller, 'rel.environments', 'manage');
+        const hiddenErrors = new Set(diagnoses
+            ? []
+            : occupancy.failedEnvironments.flatMap((item) => item.error ? [item.error] : []));
+        return {
+            view: {
+                id: occupancy.id,
+                memory: occupancy.memory,
+                recycling: occupancy.recycling,
+                deployed: occupancy.apps.length,
+                // Running or starting Apps hold the runtime; a failed one holds nothing and is not counted.
+                running: occupancy.apps.filter((item) => item.state === 'running' || item.state === 'starting').length,
+                maxActive: occupancy.maxActive,
+                failedEnvironments: occupancy.failedEnvironments.map((item) => ({
+                    id: item.environmentId,
+                    name: environmentNames.get(item.environmentId) ?? null,
+                    error: diagnoses ? item.error : null,
+                })),
+            },
+            hiddenErrors,
+        };
+    }
+    async createApp(caller, input, options = {}) {
         this.options.guard.requireCreate(caller);
         if (!input || typeof input.id !== 'string')
             throw new ReleasesError('An App ID is required.', 'INVALID_APP_ID', 'INVALID_ARGUMENT');
@@ -144,28 +273,32 @@ export class ReleasesService {
                 if (await this.findApp(id))
                     throw new ReleasesError('Application ID is unavailable. Choose a different ID.', 'APP_EXISTS', 'ALREADY_EXISTS');
                 await this.assertEnvironmentCapacity(environment);
-                await this.query()
-                    .insertInto('relApps')
-                    .values({
-                    id,
-                    environmentId: environment.id,
-                    name,
-                    description: typeof input.description === 'string' && input.description.trim()
-                        ? input.description.trim()
-                        : null,
-                    currentDeploymentId: null,
-                    enabled: false,
-                    activation: policy.activation,
-                    idleStopMinutes: policy.idleStopMinutes,
-                    dormantAfterHours: policy.dormantAfterHours,
-                    labels: JSON.stringify(labels),
-                    previewOf,
-                    createdBy: caller.userId,
-                    createdVia: caller.kind,
-                    createdAt: now,
-                    updatedAt: now,
-                })
-                    .execute();
+                await this.options.database.transaction(async (connection) => {
+                    await connection.query
+                        .insertInto('relApps')
+                        .values({
+                        id,
+                        environmentId: environment.id,
+                        name,
+                        description: typeof input.description === 'string' &&
+                            input.description.trim()
+                            ? input.description.trim()
+                            : null,
+                        currentDeploymentId: null,
+                        enabled: false,
+                        activation: policy.activation,
+                        idleStopMinutes: policy.idleStopMinutes,
+                        dormantAfterHours: policy.dormantAfterHours,
+                        labels: JSON.stringify(labels),
+                        previewOf,
+                        createdBy: caller.userId,
+                        createdVia: caller.kind,
+                        createdAt: now,
+                        updatedAt: now,
+                    })
+                        .execute();
+                    await options.initialize?.(connection, id);
+                });
             });
         }
         catch (error) {
@@ -268,6 +401,48 @@ export class ReleasesService {
         const images = await this.artifacts.imagesOf([release.id]);
         return releaseView(release, images.get(release.id) ?? []);
     }
+    /**
+     * Several Apps' summaries in one batched read (one runtime status call per environment), for the assembling
+     * application: no caller is checked. Unknown ids are left out; the rest keep the order asked.
+     */
+    async summarizeApps(appIds) {
+        const ids = [...new Set(appIds)];
+        if (ids.length === 0)
+            return [];
+        const rows = await this.query()
+            .selectFrom('relApps')
+            .selectAll()
+            .where('id', 'in', ids)
+            .execute();
+        const byId = new Map(rows.map(decodeApp).map((app) => [app.id, app]));
+        const apps = ids.flatMap((id) => {
+            const app = byId.get(id);
+            return app ? [app] : [];
+        });
+        return this.summarize(apps);
+    }
+    /**
+     * A release's archive, to download: `view` on the App. An image release has none (`RELEASE_HAS_NO_ARCHIVE`), and an
+     * archive missing from the disk is `RELEASE_ARCHIVE_NOT_FOUND`. The stream is opened last, so a refusal never touches
+     * the disk.
+     */
+    async openArchive(caller, appId, releaseId) {
+        const app = await this.requireApp(appId);
+        await this.options.guard.requireApp(caller, 'view', app);
+        const release = await this.requireRelease(appId, releaseId);
+        const key = release.artifactKey;
+        if (!key)
+            throw new ReleasesError('This release is a set of images and has no archive to download.', 'RELEASE_HAS_NO_ARCHIVE', 'FAILED_PRECONDITION');
+        // A local disk opens its file lazily, so a missing archive would fail only after the response headers went out.
+        if (!(await this.disk.exists(key)))
+            throw notFound('Archive', 'RELEASE_ARCHIVE_NOT_FOUND', releaseId);
+        return {
+            stream: await this.disk.getStream(key),
+            size: release.size,
+            checksum: release.checksum,
+            fileName: archiveFileName(app.id, release.version),
+        };
+    }
     /**
      * Registers an image CI built and pushed, by digest, as a release of `version` (an image release, which Docker
      * environments pull). It needs `upload` on the App, as uploading an archive does. The same digest again answers its
@@ -418,7 +593,7 @@ export class ReleasesService {
                 if (existing)
                     return existing;
                 const id = randomUUID();
-                const artifactKey = `${appId}/${id}.tar.gz`;
+                const artifactKey = this.artifactKey(appId, id);
                 const release = {
                     id,
                     appId,
@@ -438,8 +613,6 @@ export class ReleasesService {
                     createdVia: caller.kind,
                     createdAt: new Date(),
                 };
-                if (input.deploy)
-                    await this.requireNoPendingDeployment(appId);
                 // A release whose required variables are not all set is kept, and not deployed: once they are set, it
                 // deploys by its ID without another upload.
                 const missing = input.deploy
@@ -457,8 +630,6 @@ export class ReleasesService {
                         contentType: 'application/gzip',
                     });
                     await this.options.database.transaction(async (connection) => {
-                        if (deployment)
-                            await this.requireNoPendingDeployment(appId, connection);
                         await connection.query
                             .insertInto('relReleases')
                             .values(encodeRelease(release))
@@ -514,8 +685,16 @@ export class ReleasesService {
                 });
                 if (missing.length > 0)
                     throw variablesMissing(current, release, missing);
-                if (deployment)
+                if (deployment) {
+                    try {
+                        await this.replaceQueued(deployment);
+                    }
+                    catch (error) {
+                        await this.markFailed(deployment, error);
+                        throw error;
+                    }
                     this.schedule(deployment);
+                }
                 return { ...view, reused: false, deploymentId: deployment?.id ?? null };
             });
         }
@@ -574,7 +753,7 @@ export class ReleasesService {
             }
             const id = randomUUID();
             const artifactKey = release.artifactKey
-                ? `${targetAppId}/${id}.tar.gz`
+                ? this.artifactKey(targetAppId, id)
                 : null;
             const copy = {
                 ...release,
@@ -654,6 +833,7 @@ export class ReleasesService {
             throw new ReleasesError('Configuration content is required.', 'INVALID_CONFIG_FILE', 'INVALID_ARGUMENT');
         assertConfigSize(content);
         return await this.withLock(appId, async () => {
+            await this.waitForAppDeployments(appId);
             const app = await this.requireApp(appId);
             await this.options.guard.requireApp(caller, 'configure', app);
             const deployment = await this.currentDeployment(app);
@@ -777,12 +957,12 @@ export class ReleasesService {
             .offset((page - 1) * pageSize)
             .execute();
         return {
-            items: rows.map((row) => deploymentView(decodeDeployment(row), typeof row.releaseVersion === 'string'
+            items: rows.map((row) => this.progressView(deploymentView(decodeDeployment(row), typeof row.releaseVersion === 'string'
                 ? {
                     version: row.releaseVersion,
                     checksum: String(row.releaseChecksum),
                 }
-                : null)),
+                : null))),
             total,
             page,
             pageSize,
@@ -798,7 +978,14 @@ export class ReleasesService {
             await this.options.guard.requireApp(caller, 'deploy', app);
         const deployment = await this.requireDeployment(appId, deploymentId);
         const release = await this.findRelease(appId, deployment.releaseId);
-        return deploymentView(deployment, release ? { version: release.version, checksum: release.checksum } : null);
+        if (['queued', 'deploying', 'cancelling'].includes(deployment.status)) {
+            const progress = await (await this.session(app.environmentId)).progress?.(deployment.id, app.id);
+            if (progress)
+                this.progress.set(deployment.id, progress);
+        }
+        return this.progressView(deploymentView(deployment, release
+            ? { version: release.version, checksum: release.checksum }
+            : null));
     }
     /** Resolves once the deployment has finished, or with its current state at the timeout. */
     async waitForDeployment(deploymentId, timeoutMs = 120_000) {
@@ -812,7 +999,7 @@ export class ReleasesService {
             if (!row)
                 throw notFound('Deployment', 'DEPLOYMENT_NOT_FOUND', deploymentId);
             const deployment = decodeDeployment(row);
-            const finished = deployment.status !== 'queued' && deployment.status !== 'deploying';
+            const finished = !['queued', 'deploying', 'cancelling'].includes(deployment.status);
             if (finished || Date.now() >= deadline) {
                 // A finished deployment's run still announces it (events, the request it settled); wait for that too, so a
                 // waiter never sees the deployment done but its consequences pending.
@@ -867,6 +1054,7 @@ export class ReleasesService {
     // --- Lifecycle ----------------------------------------------------------------------------------------------
     async start(caller, appId) {
         return await this.withLock(appId, async () => {
+            await this.waitForAppDeployments(appId);
             const app = await this.requireApp(appId);
             await this.options.guard.requireApp(caller, 'operate', app);
             const deployment = await this.currentDeployment(app);
@@ -887,6 +1075,7 @@ export class ReleasesService {
     }
     async stop(caller, appId) {
         return await this.withLock(appId, async () => {
+            await this.waitForAppDeployments(appId);
             const app = await this.requireApp(appId);
             await this.options.guard.requireApp(caller, 'operate', app);
             if (!(await this.currentDeployment(app)))
@@ -904,6 +1093,7 @@ export class ReleasesService {
     }
     async restart(caller, appId) {
         return await this.withLock(appId, async () => {
+            await this.waitForAppDeployments(appId);
             const app = await this.requireApp(appId);
             await this.options.guard.requireApp(caller, 'operate', app);
             if (!(await this.currentDeployment(app)))
@@ -927,18 +1117,20 @@ export class ReleasesService {
      * not report every App as stopped or a finished deployment as still running.
      */
     async restore() {
-        await this.clearExpiredInitialAdmins().catch((error) => this.diagnostic.warn('Failed to clear expired first administrators', error));
         const interrupted = (await this.query()
             .selectFrom('relDeployments')
             .selectAll()
-            .where('status', 'in', ['queued', 'deploying'])
+            .where('status', 'in', ['queued', 'deploying', 'cancelling'])
+            .orderBy('createdAt', 'asc')
+            .orderBy('id', 'asc')
             .execute()).map(decodeDeployment);
         // A deployment the target recorded (the App Host's operation log) is finalised from what it reports; every other
         // one was cut short.
         const resumed = new Set();
         const recorded = new Map();
         for (const deployment of interrupted) {
-            if (deployment.status !== 'deploying')
+            if (deployment.status !== 'deploying' &&
+                deployment.status !== 'cancelling')
                 continue;
             const app = await this.findApp(deployment.appId);
             if (!app)
@@ -963,7 +1155,7 @@ export class ReleasesService {
                 .set({
                 status: 'failed',
                 phase: 'completed',
-                error: 'Deployment was interrupted by a restart.',
+                error: 'Deployment was interrupted by a restart. Deploy the release again.',
                 finishedAt: new Date(),
             })
                 .where('id', 'in', cutShort.map((deployment) => deployment.id))
@@ -1198,15 +1390,14 @@ export class ReleasesService {
     }
     /**
      * The first administrator the App's first deployment generated, for a person who may deploy it: until someone
-     * says it is saved, and for 24 hours (a preview App's until the App goes).
+     * says it is saved or the App is removed.
      */
     async readInitialAdmin(caller, appId) {
         const app = await this.requireApp(appId);
         await this.requireInitialAdminReader(caller, app);
-        await this.clearExpiredInitialAdmins();
         const found = await this.initialAdminOf(app);
         if (!found)
-            throw new ReleasesError('This App has no first administrator to show: it was saved, it expired, or the application did not generate one.', 'INITIAL_ADMIN_UNAVAILABLE', 'NOT_FOUND');
+            throw new ReleasesError('This App has no first administrator to show: it was saved or the application did not generate one.', 'INITIAL_ADMIN_UNAVAILABLE', 'NOT_FOUND');
         return found;
     }
     /** Forgets the generated first administrator once someone has saved it. */
@@ -1224,16 +1415,6 @@ export class ReleasesService {
             .where('initialAdmin', 'is not', null)
             .execute();
     }
-    /** Clears the first administrators whose 24 hours are over. */
-    async clearExpiredInitialAdmins(now = new Date()) {
-        await this.query()
-            .updateTable('relDeployments')
-            .set({ initialAdmin: null })
-            .where('initialAdmin', 'is not', null)
-            .where('initialAdminExpiresAt', 'is not', null)
-            .where('initialAdminExpiresAt', '<=', now)
-            .execute();
-    }
     async requireInitialAdminReader(caller, app) {
         // A password a person copies: never an agent's or a key's, whatever they may deploy.
         if (caller.kind !== 'human' &&
@@ -1250,18 +1431,14 @@ export class ReleasesService {
             .where('initialAdmin', 'is not', null)
             .orderBy('createdAt', 'desc')
             .execute();
-        const now = new Date();
         for (const deployment of rows.map(decodeDeployment)) {
-            if (deployment.initialAdminExpiresAt &&
-                deployment.initialAdminExpiresAt <= now)
-                continue;
             const admin = this.openInitialAdmin(deployment);
             if (!admin)
                 continue;
             return {
                 deploymentId: deployment.id,
                 ...admin,
-                expiresAt: iso(deployment.initialAdminExpiresAt),
+                expiresAt: null,
             };
         }
         return null;
@@ -1431,7 +1608,6 @@ export class ReleasesService {
         for (const [name, variable] of plan.oneShot)
             generated[name] = generateVariable(variable.generate);
         let initialAdmin = null;
-        let initialAdminExpiresAt = null;
         if (plan.initialAdmin) {
             const password = (await this.earlierInitialAdmin(app))?.password ?? randomPassword();
             generated[plan.initialAdmin.name] = password;
@@ -1441,10 +1617,6 @@ export class ReleasesService {
                 password,
             };
             initialAdmin = encryptText(JSON.stringify(admin), [deploymentId], this.options.secrets, 'initial-admin');
-            // A preview App's goes with the preview; any other's is deleted after a day.
-            initialAdminExpiresAt = app.previewOf
-                ? null
-                : new Date(Date.now() + INITIAL_ADMIN_TTL_MS);
         }
         const resolved = resolveVariables({
             manifest: release.variables,
@@ -1462,7 +1634,6 @@ export class ReleasesService {
                 : encryptText(JSON.stringify(resolved.env), [deploymentId], this.options.secrets, 'deployment-env'),
             variablesFingerprint: release.variables || names.length > 0 ? resolved.fingerprint : null,
             initialAdmin,
-            initialAdminExpiresAt,
         };
     }
     /**
@@ -1517,7 +1688,6 @@ export class ReleasesService {
     }
     // --- Internals ----------------------------------------------------------------------------------------------
     async startDeployment(caller, app, release, options, config, idempotency) {
-        await this.requireNoPendingDeployment(app.id);
         const deployment = await this.buildDeploymentRecord(app, release, caller, {
             kind: options.kind,
             rollbackTargetDeploymentId: options.rollbackTargetDeploymentId ?? null,
@@ -1526,7 +1696,6 @@ export class ReleasesService {
         });
         try {
             await this.options.database.transaction(async (connection) => {
-                await this.requireNoPendingDeployment(app.id, connection);
                 await connection.query
                     .insertInto('relDeployments')
                     .values(encodeDeployment(deployment))
@@ -1553,6 +1722,13 @@ export class ReleasesService {
                 await rm(deployment.configPath, { force: true });
             throw error;
         }
+        try {
+            await this.replaceQueued(deployment);
+        }
+        catch (error) {
+            await this.markFailed(deployment, error);
+            throw error;
+        }
         this.schedule(deployment);
         return {
             ...deploymentView(deployment, {
@@ -1642,6 +1818,7 @@ export class ReleasesService {
     }
     async removeApp(app, caller) {
         await this.withLock(`publish:${app.id}`, () => this.withLock(app.id, async () => {
+            await this.waitForAppDeployments(app.id);
             const current = await this.findApp(app.id);
             if (!current)
                 return;
@@ -1692,7 +1869,24 @@ export class ReleasesService {
             actor: actorOf(caller),
         });
     }
-    async summarize(apps) {
+    /** Each environment's driver status of these Apps, one call per environment; the error when its driver failed. */
+    async runtimeStatuses(apps) {
+        const statuses = new Map();
+        for (const environmentId of new Set(apps.map((app) => app.environmentId)))
+            try {
+                const session = await this.session(environmentId);
+                statuses.set(environmentId, await session.status(apps
+                    .filter((app) => app.environmentId === environmentId)
+                    .map((app) => app.id)));
+            }
+            catch (error) {
+                statuses.set(environmentId, error instanceof Error ? error : new Error(String(error)));
+            }
+        return statuses;
+    }
+    async summarize(apps, 
+    /** Statuses already read for these Apps (a superset is fine), so they are not asked again. */
+    known) {
         if (!apps.length)
             return [];
         if (this.restoring)
@@ -1707,10 +1901,10 @@ export class ReleasesService {
                 .execute(),
             this.query()
                 .selectFrom('relDeployments')
-                .select('appId')
-                .distinct()
+                .select(['id', 'appId', 'status', 'createdAt'])
                 .where('appId', 'in', ids)
-                .where('status', 'in', ['queued', 'deploying'])
+                .orderBy('createdAt', 'asc')
+                .where('status', 'in', ['queued', 'deploying', 'cancelling'])
                 .execute(),
             this.query()
                 .selectFrom('relApps')
@@ -1722,22 +1916,22 @@ export class ReleasesService {
         ]);
         const released = new Set(releases.map((row) => String(row.appId)));
         const pendingApps = new Set(pending.map((row) => String(row.appId)));
+        const pendingByApp = new Map();
+        for (const row of pending) {
+            const previous = pendingByApp.get(String(row.appId));
+            if (!previous ||
+                (previous.status === 'queued' && row.status !== 'queued'))
+                pendingByApp.set(String(row.appId), row);
+        }
         const versions = new Map(currentReleases.map((row) => [String(row.appId), String(row.version)]));
         const environments = new Map();
-        const statuses = new Map();
+        const statuses = known ?? (await this.runtimeStatuses(apps));
         const sessions = new Map();
         for (const environmentId of new Set(apps.map((app) => app.environmentId))) {
             environments.set(environmentId, await this.options.environments.find(environmentId));
-            const environmentApps = apps.filter((app) => app.environmentId === environmentId);
-            try {
-                const session = await this.session(environmentId);
-                sessions.set(environmentId, session);
-                statuses.set(environmentId, await session.status(environmentApps.map((app) => app.id)));
-            }
-            catch (error) {
-                sessions.set(environmentId, null);
-                statuses.set(environmentId, error instanceof Error ? error : new Error(String(error)));
-            }
+            sessions.set(environmentId, statuses.get(environmentId) instanceof Error
+                ? null
+                : await this.session(environmentId).catch(() => null));
         }
         return apps.map((app) => {
             const environment = environments.get(app.environmentId) ?? null;
@@ -1758,6 +1952,15 @@ export class ReleasesService {
                 currentVersion: versions.get(app.id) ?? null,
                 hasReleases: released.has(app.id),
                 hasPendingDeployment: pendingApps.has(app.id),
+                pendingDeployment: (() => {
+                    const row = pendingByApp.get(app.id);
+                    return row
+                        ? {
+                            status: row.status,
+                            queueAhead: this.progress.get(String(row.id))?.queueAhead ?? null,
+                        }
+                        : null;
+                })(),
             };
         });
     }
@@ -1790,6 +1993,8 @@ export class ReleasesService {
                 .driverEnvironment(environmentId)
                 .then(({ driver, target }) => driver.open(target, {
                 desired: () => this.desiredSet(environmentId),
+                deploymentBatchBusy: (excluding) => [...this.dispatch].some(([id, item]) => !excluding.has(id) &&
+                    (!item.environmentId || item.environmentId === environmentId)),
             }));
             session.catch(() => {
                 if (this.sessions.get(environmentId) === session)
@@ -1989,7 +2194,7 @@ export class ReleasesService {
     }
     schedule(deployment) {
         this.logDeployment(deployment, 'queued', 'Deployment queued');
-        const run = this.withLock(deployment.appId, () => this.runDeployment(deployment.id))
+        const run = this.dispatchDeployment(deployment)
             .catch((error) => this.diagnostic.error(`Deployment ${deployment.id} failed unexpectedly`, error))
             .finally(() => {
             this.running.delete(run);
@@ -1999,30 +2204,77 @@ export class ReleasesService {
         this.running.add(run);
         this.runs.set(deployment.id, run);
     }
-    async runDeployment(deploymentId) {
+    async dispatchDeployment(deployment) {
+        const app = await this.requireApp(deployment.appId);
+        const session = await this.session(app.environmentId);
+        try {
+            // Drivers without a native queue keep the previous per-App serialization.
+            if (!session.progress)
+                return await this.withLock(app.id, () => this.runDeployment(deployment.id, deployment.appId, app.environmentId));
+            const lifecycle = this.locks.get(app.id);
+            if (lifecycle)
+                await lifecycle.catch(() => undefined);
+            return await this.runDeployment(deployment.id, deployment.appId, app.environmentId);
+        }
+        finally {
+            // Metadata/initial-admission failures must not leave the batch busy forever either.
+            this.dispatch.delete(deployment.id);
+            try {
+                session.deploymentSettled?.();
+            }
+            catch (error) {
+                this.diagnostic.error('Failed to notify the deployment driver after settlement', error);
+            }
+        }
+    }
+    /** Lifecycle operations keep the deployment result current before reading its configuration. New dispatches wait on their App lock. */
+    async waitForAppDeployments(appId) {
+        const runs = [...this.dispatch]
+            .filter(([, value]) => value.appId === appId)
+            .map(([id]) => this.runs.get(id))
+            .filter((run) => Boolean(run));
+        await Promise.allSettled(runs);
+    }
+    async runDeployment(deploymentId, appId, environmentId) {
+        const dispatch = { appId, submitted: false, cancelled: false, environmentId };
+        this.dispatch.set(deploymentId, dispatch);
         const deployment = await this.requireDeploymentById(deploymentId);
+        if (deployment.status !== 'queued') {
+            this.dispatch.delete(deploymentId);
+            return;
+        }
         const app = await this.requireApp(deployment.appId);
         const release = await this.requireRelease(app.id, deployment.releaseId);
-        const previous = await this.currentDeployment(app);
+        let settlement;
         let phaseWrites = Promise.resolve();
         let lastSequence = 0;
+        let lastPhase;
+        let switchStarted = false;
         let phaseError;
-        await this.updateDeployment(deploymentId, {
-            status: 'deploying',
-            phase: 'resolving',
-            startedAt: new Date(),
-            error: null,
-            previousDeploymentId: app.currentDeploymentId,
-        });
         let outcome;
         try {
-            this.logDeployment(deployment, 'resolving', 'Deployment started');
             const session = await this.session(app.environmentId);
             const logsEnabled = this.options.logging?.deployments?.enabled !== false;
             const spec = await this.deploymentSpec({ ...app, enabled: true }, deployment, 'running');
             // A driver that runs no images runs the archive; one that does says what it ran (`DeploymentEvent.artifact`).
             if (spec.images === undefined)
                 await this.updateDeployment(deploymentId, artifactColumns({ kind: 'tarball' }));
+            if (dispatch.cancelled) {
+                await this.markCancelled(deployment);
+                return;
+            }
+            if (!session.progress) {
+                this.logDeployment(deployment, 'resolving', 'Deployment started');
+                await this.updateDeployment(deploymentId, {
+                    status: 'deploying',
+                    phase: 'resolving',
+                    startedAt: new Date(),
+                });
+            }
+            dispatch.submitted = true;
+            // Reserve in runtime submission order without holding up Host admission or cancellation.
+            settlement = this.reserveSettlement(app.id);
+            let started = false;
             const observed = await session.apply(spec, (entry) => {
                 if (typeof entry.sequence === 'number') {
                     if (entry.sequence <= lastSequence)
@@ -2039,8 +2291,46 @@ export class ReleasesService {
                     });
                 if (typeof entry.phase === 'string') {
                     const phase = entry.phase;
+                    // A completed capacity rejection may re-enter a cancellable wait before one retry.
+                    if (entry.cancellable === true)
+                        switchStarted = false;
+                    const first = !started &&
+                        phase !== 'queued' &&
+                        phase !== 'cancelled' &&
+                        phase !== 'cancelling';
+                    if (first)
+                        started = true;
+                    if (['switching', 'starting', 'health_check'].includes(phase))
+                        switchStarted = true;
+                    this.progress.set(deploymentId, {
+                        queueAhead: phase === 'queued' && typeof entry.queueAhead === 'number'
+                            ? entry.queueAhead
+                            : null,
+                        cancellable: !switchStarted &&
+                            !['completed', 'cancelled', 'cancelling'].includes(phase),
+                    });
+                    const changed = lastPhase !== phase;
+                    lastPhase = phase;
                     phaseWrites = phaseWrites
-                        .then(() => this.updateDeployment(deploymentId, { phase }))
+                        .then(async () => {
+                        // Queue-position events are transient; they cause no database writes.
+                        if (phase === 'queued' || !changed)
+                            return;
+                        await this.query()
+                            .updateTable('relDeployments')
+                            .set({
+                            phase,
+                            ...(first
+                                ? {
+                                    status: 'deploying',
+                                    startedAt: new Date(),
+                                }
+                                : {}),
+                        })
+                            .where('id', '=', deploymentId)
+                            .where('status', 'in', ['queued', 'deploying'])
+                            .execute();
+                    })
                         .catch((error) => {
                         phaseError =
                             error instanceof Error ? error : new Error(String(error));
@@ -2059,24 +2349,64 @@ export class ReleasesService {
             await phaseWrites;
             if (phaseError)
                 throw phaseError;
+            if (observed.state === 'cancelled') {
+                await this.markCancelled(deployment);
+                return;
+            }
             if (observed.state === 'failed' || observed.state === 'unknown')
                 throw new Error(observed.error ?? 'The runtime did not report the deployment.');
-            await this.markSucceeded(deployment, app.id, previous);
             outcome = { ok: true };
         }
         catch (error) {
             await phaseWrites.catch(() => undefined);
             outcome = { ok: false, error: errorMessage(error) };
-            await this.markFailed(deployment, error);
         }
         finally {
-            const policy = this.options.logging?.deployments;
-            await pruneJournals(path.dirname(this.deploymentLogPath(app.id, deployment.id)), {
-                retentionDays: policy?.retentionDays ?? 30,
-                maxSizeMB: policy?.maxTotalSizeMB ?? 1024,
-            }, `${deployment.id}.log`).catch((error) => reportLoggingFailure('Deployment log cleanup failed', error));
+            const settle = settlement ?? this.reserveSettlement(app.id);
+            const result = settle(async () => {
+                if (!outcome)
+                    return;
+                // The predecessor is read only after its writes, cleanup and outcome event have finished.
+                const previous = await this.currentDeployment(await this.requireApp(app.id));
+                if (outcome.ok) {
+                    try {
+                        await this.markSucceeded(deployment, app.id, previous);
+                    }
+                    catch (error) {
+                        outcome = { ok: false, error: errorMessage(error) };
+                    }
+                }
+                if (!outcome.ok)
+                    await this.markFailed(deployment, new Error(outcome.error), previous);
+                await this.emitOutcome(deployment, app, release, outcome);
+            });
+            try {
+                // A queued cancellation is already terminal; it must not wait for the running predecessor.
+                if (outcome)
+                    await result;
+            }
+            finally {
+                this.dispatch.delete(deploymentId);
+                this.progress.delete(deploymentId);
+                const policy = this.options.logging?.deployments;
+                await pruneJournals(path.dirname(this.deploymentLogPath(app.id, deployment.id)), {
+                    retentionDays: policy?.retentionDays ?? 30,
+                    maxSizeMB: policy?.maxTotalSizeMB ?? 1024,
+                }, `${deployment.id}.log`).catch((error) => reportLoggingFailure('Deployment log cleanup failed', error));
+            }
         }
-        await this.emitOutcome(deployment, app, release, outcome);
+    }
+    /** Reuse the local lock chain to order results while the Host continues executing its queue. */
+    reserveSettlement(appId) {
+        let finish;
+        const ready = new Promise((resolve) => {
+            finish = resolve;
+        });
+        const result = this.withLock(`deployment-result:${appId}`, async () => (await ready)());
+        return (work) => {
+            finish(work);
+            return result;
+        };
     }
     async markSucceeded(deployment, appId, previous) {
         this.logDeployment(deployment, 'completed', 'Deployment succeeded');
@@ -2084,7 +2414,12 @@ export class ReleasesService {
         await this.options.database.transaction(async (connection) => {
             await connection.query
                 .updateTable('relDeployments')
-                .set({ status: 'succeeded', phase: 'completed', finishedAt })
+                .set({
+                status: 'succeeded',
+                phase: 'completed',
+                finishedAt,
+                previousDeploymentId: previous?.id ?? null,
+            })
                 .where('id', '=', deployment.id)
                 .execute();
             await connection.query
@@ -2101,7 +2436,7 @@ export class ReleasesService {
         if (previous?.configPath && previous.id !== deployment.id)
             await rm(previous.configPath, { force: true }).catch(() => undefined);
     }
-    async markFailed(deployment, error) {
+    async markFailed(deployment, error, previous) {
         this.logDeployment(deployment, 'completed', 'Deployment failed', error);
         const finishedAt = new Date();
         await this.options.database.transaction(async (connection) => {
@@ -2112,6 +2447,9 @@ export class ReleasesService {
                 phase: 'completed',
                 error: errorMessage(error),
                 finishedAt,
+                ...(previous !== undefined
+                    ? { previousDeploymentId: previous?.id ?? null }
+                    : {}),
             })
                 .where('id', '=', deployment.id)
                 .execute();
@@ -2179,15 +2517,17 @@ export class ReleasesService {
                 };
             }
             const current = await this.requireDeploymentById(deployment.id);
-            if (current.status !== 'deploying' && current.status !== 'queued')
+            if (!['deploying', 'queued', 'cancelling'].includes(current.status))
                 return;
             const app = await this.requireApp(deployment.appId);
             const release = await this.requireRelease(app.id, deployment.releaseId);
             let outcome;
+            if (status?.state === 'cancelled') {
+                await this.markCancelled(deployment);
+                return;
+            }
             if (status?.state === 'running') {
-                const previous = deployment.previousDeploymentId
-                    ? await this.requireDeployment(app.id, deployment.previousDeploymentId).catch(() => null)
-                    : null;
+                const previous = await this.currentDeployment(app);
                 this.logDeployment(deployment, 'switching', 'The runtime reports the deployment finished');
                 await this.markSucceeded(deployment, app.id, previous);
                 outcome = { ok: true };
@@ -2195,7 +2535,7 @@ export class ReleasesService {
             else {
                 const error = status
                     ? (status.error ?? 'The deployment failed.')
-                    : 'Deployment was interrupted by a restart.';
+                    : 'Deployment was interrupted by a restart. Deploy the release again.';
                 await this.markFailed(deployment, new Error(error));
                 outcome = { ok: false, error };
             }
@@ -2211,15 +2551,120 @@ export class ReleasesService {
         this.runs.set(deployment.id, run);
         return run;
     }
-    async requireNoPendingDeployment(appId, connection) {
-        const pending = await (connection?.query ?? this.query())
+    progressView(view) {
+        const progress = this.progress.get(view.id);
+        return {
+            ...view,
+            queueAhead: progress?.queueAhead ?? null,
+            cancellable: ['queued', 'deploying'].includes(view.status) &&
+                (progress?.cancellable ?? view.phase === 'queued'),
+        };
+    }
+    async cancelDeployment(caller, appId, deploymentId) {
+        const app = await this.requireApp(appId);
+        await this.options.guard.requireApp(caller, 'deploy', app);
+        await this.withLock(`cancel:${deploymentId}`, async () => {
+            const deployment = await this.requireDeployment(appId, deploymentId);
+            if (deployment.status === 'cancelled' ||
+                deployment.status === 'cancelling')
+                return;
+            if (!['queued', 'deploying'].includes(deployment.status))
+                throw new ReleasesError('The deployment has finished.', 'DEPLOYMENT_NOT_CANCELLABLE', 'FAILED_PRECONDITION');
+            const result = await this.requestCancellation(deployment, false);
+            if (result === 'tooLate' || result === 'notFound')
+                throw new ReleasesError('The deployment has started switching or has finished.', 'DEPLOYMENT_NOT_CANCELLABLE', 'FAILED_PRECONDITION');
+        });
+        return this.getDeployment(caller, appId, deploymentId);
+    }
+    async replaceQueued(replacement) {
+        const rows = await this.query()
             .selectFrom('relDeployments')
-            .select('id')
-            .where('appId', '=', appId)
-            .where('status', 'in', ['queued', 'deploying'])
-            .executeTakeFirst();
-        if (pending)
-            throw new ReleasesError('A deployment is already in progress.', 'DEPLOYMENT_IN_PROGRESS', 'FAILED_PRECONDITION');
+            .selectAll()
+            .where('appId', '=', replacement.appId)
+            .where('status', '=', 'queued')
+            .where('id', '!=', replacement.id)
+            .execute();
+        for (const row of rows)
+            await this.requestCancellation(decodeDeployment(row), true, replacement.id);
+    }
+    async requestCancellation(deployment, queuedOnly, replacedBy) {
+        const dispatch = this.dispatch.get(deployment.id);
+        let result;
+        if ((dispatch && !dispatch.submitted) ||
+            (!dispatch && deployment.status === 'queued')) {
+            if (dispatch)
+                dispatch.cancelled = true;
+            result = 'cancelled';
+        }
+        else {
+            const app = await this.requireApp(deployment.appId);
+            const session = await this.session(app.environmentId);
+            if (!session.cancel)
+                throw new ReleasesError('The deployment driver does not support cancellation.', 'CANCELLATION_UNSUPPORTED', 'FAILED_PRECONDITION');
+            result = await session.cancel(deployment.id, app.id, queuedOnly);
+        }
+        if (result === 'cancelled')
+            await this.markCancelled(deployment, replacedBy);
+        else if (result === 'cancelling') {
+            this.logDeployment(deployment, 'cancelling', 'Cancellation requested; waiting for the current step to finish');
+            await this.query()
+                .updateTable('relDeployments')
+                .set({ status: 'cancelling' })
+                .where('id', '=', deployment.id)
+                .where('status', 'in', ['queued', 'deploying'])
+                .execute();
+        }
+        return result;
+    }
+    markCancelled(deployment, replacedBy) {
+        return this.withLock(`settle:${deployment.id}`, () => this.markCancelledUnlocked(deployment, replacedBy));
+    }
+    async markCancelledUnlocked(deployment, replacedBy) {
+        const current = await this.requireDeploymentById(deployment.id);
+        if (!['queued', 'deploying', 'cancelling'].includes(current.status))
+            return;
+        const reason = replacedBy
+            ? 'Replaced by a newer deployment.'
+            : 'Deployment cancelled.';
+        const finishedAt = new Date();
+        if (deployment.configPath)
+            await rm(deployment.configPath, { force: true });
+        await this.options.database.transaction(async (connection) => {
+            await connection.query
+                .updateTable('relDeployments')
+                .set({
+                status: 'cancelled',
+                phase: 'cancelled',
+                error: reason,
+                finishedAt,
+            })
+                .where('id', '=', deployment.id)
+                .where('status', 'in', ['queued', 'deploying', 'cancelling'])
+                .execute();
+            await this.settleRequest(connection, deployment, 'cancelled', finishedAt);
+        });
+        if (this.options.logging?.deployments?.enabled !== false)
+            this.appendDeploymentLog(deployment, {
+                time: finishedAt.toISOString(),
+                level: 'info',
+                appId: deployment.appId,
+                deploymentId: deployment.id,
+                phase: 'cancelled',
+                msg: reason,
+                reason: replacedBy ? 'superseded' : 'requested',
+                ...(replacedBy ? { replacedBy } : {}),
+            });
+        const app = await this.requireApp(deployment.appId);
+        const release = await this.requireRelease(app.id, deployment.releaseId);
+        await this.options.events
+            .emit({
+            type: 'deployment.cancelled',
+            app: appView(app),
+            release: releaseView(release),
+            deployment: deploymentView(await this.requireDeploymentById(deployment.id), { version: release.version, checksum: release.checksum }),
+            actor: { userId: deployment.actorId, kind: deployment.actorKind },
+        })
+            .catch((error) => this.diagnostic.error('Failed to announce deployment cancellation', error));
     }
     async findRelease(appId, releaseId) {
         const row = await this.query()
@@ -2330,8 +2775,6 @@ export class ReleasesService {
         return this.options.database.connection().query;
     }
 }
-// --- Variables --------------------------------------------------------------------------------------------------
-const INITIAL_ADMIN_TTL_MS = 24 * 60 * 60 * 1000;
 /** A variable's value from the App or the environment, when one of them sets it. */
 function suppliedValue(context, name) {
     return (context.appVariables.find((variable) => variable.name === name)?.value ??
@@ -2576,7 +3019,8 @@ function normalizeBuild(value) {
         throw new ReleasesError('The build is text of at most 255 characters.', 'INVALID_BUILD', 'INVALID_ARGUMENT');
     return value;
 }
-function runtimeStatus(status, appId) {
+/** What the driver observed of an App, as an App summary reports its runtime. */
+export function runtimeStatus(status, appId) {
     if (!status || status instanceof Error)
         return {
             available: false,
@@ -2598,11 +3042,14 @@ function runtimeStatus(status, appId) {
         };
     return {
         available: true,
-        state: item.state,
+        state: item.state === 'cancelled' ? 'failed' : item.state,
         version: item.version,
         startedAt: item.startedAt,
         error: item.error,
         lastAccessedAt: item.lastAccessedAt ?? null,
+        ...(item.holds?.length ? { holds: item.holds } : {}),
+        ...(item.waiting ? { waiting: item.waiting } : {}),
+        ...(item.unrestored ? { unrestored: true } : {}),
     };
 }
 function decodeActorKind(value) {
@@ -2686,7 +3133,6 @@ function decodeDeployment(row) {
         env: nullableString(row.env),
         variablesFingerprint: nullableString(row.variablesFingerprint),
         initialAdmin: nullableString(row.initialAdmin),
-        initialAdminExpiresAt: decodeOptionalDate(row.initialAdminExpiresAt),
         initialAdminSavedBy: nullableString(row.initialAdminSavedBy),
         initialAdminSavedAt: decodeOptionalDate(row.initialAdminSavedAt),
         createdAt: decodeDate(row.createdAt),