anbaric-cloud-hosting 1.68.3 → 1.69.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "anbaric-cloud-hosting",
-  "version": "1.68.3",
+  "version": "1.69.0",
   "description": "Anbaric Cloud hosting service: Postgres-backed job persistence and queuing exposed over an HTTP API",
   "license": "MIT",
   "author": "chris@anbaric.ai",
@@ -18,18 +18,18 @@
     "@aws-sdk/client-s3": "^3.1110.0",
     "@aws-sdk/client-secrets-manager": "^3.700.0",
     "@aws-sdk/client-servicediscovery": "^3.1110.0",
-    "anbaric-data-store": "^1.68.3",
-    "anbaric-plugins": "^1.68.3",
-    "anbaric-tsapi": "^1.68.3",
-    "anbaric-web": "^1.68.3",
+    "anbaric-data-store": "^1.69.0",
+    "anbaric-plugins": "^1.69.0",
+    "anbaric-tsapi": "^1.69.0",
+    "anbaric-web": "^1.69.0",
     "esbuild": "^0.28.2",
     "pg": "^8.16.0"
   },
   "devDependencies": {
     "@types/node": "^26.2.0",
     "@types/pg": "^8.15.0",
-    "anbaric-impl-cloud": "^1.68.3",
-    "anbaric-state-machine": "^1.68.3",
+    "anbaric-impl-cloud": "^1.69.0",
+    "anbaric-state-machine": "^1.69.0",
     "tsx": "^4.20.0",
     "typescript": "^7.0.2"
   },

package/src/app-management/BaseBuildLayer.ts

@@ -3,7 +3,7 @@ import {mkdir, readFile, rm, symlink, writeFile} from "node:fs/promises";
 import {join} from "node:path";
 import {adminPing} from "../app-admin/adminPing";
 import {DocGenerator} from "../docs/DocGenerator";
-import {AppSize, BuildLayer, DeploymentStatus, DeploymentSummary} from "./BuildLayer";
+import {AppSize, BuildLayer, DeploymentStatus, DeploymentSummary, DeployOptions} from "./BuildLayer";
 
 type Deployment = {
     appName : string,
@@ -14,6 +14,8 @@ type Deployment = {
     adminPort : number,
     consumerPort : number,
     log : Array<string>,
+    // Whether this deploy keeps the version it replaces up until it answers.
+    zeroDowntime : boolean,
     process? : ChildProcess,
     replaces? : Deployment,
     draining? : { inFlight : number, since : number },
@@ -47,7 +49,7 @@ abstract class BaseBuildLayer implements BuildLayer {
 
     constructor(protected appsDir : string, private consumerPortBase : number = 8800,
                 private probe : Probe = adminProbe,
-                private livenessTimeoutMs : number = LIVENESS_TIMEOUT_MS,
+                protected livenessTimeoutMs : number = LIVENESS_TIMEOUT_MS,
                 private drainTimeoutMs : number = DRAIN_TIMEOUT_MS,
                 private drainPollIntervalMs : number = DRAIN_POLL_INTERVAL_MS) {}
 
@@ -63,7 +65,7 @@ abstract class BaseBuildLayer implements BuildLayer {
 
     protected async rehydrate() : Promise<void> {}
 
-    deploy(appName : string, appPort : number, tarball : Buffer) : DeploymentSummary {
+    deploy(appName : string, appPort : number, tarball : Buffer, options : DeployOptions = {}) : DeploymentSummary {
         const existing = this.deployments.get(appName);
 
         const deployment : Deployment = {
@@ -76,6 +78,7 @@ abstract class BaseBuildLayer implements BuildLayer {
             adminPort: APP_ADMIN_PORT,
             consumerPort: existing?.consumerPort ?? this.consumerPortBase + this.nextAppIndex++,
             log: [],
+            zeroDowntime: options.zeroDowntime ?? false,
             replaces: existing,
         };
         this.deployments.set(appName, deployment);
@@ -247,24 +250,43 @@ abstract class BaseBuildLayer implements BuildLayer {
         const manifest = JSON.parse(await readFile(join(appDir, "package.json"), "utf8"));
         await this.linkWorkspacePackages(appDir, manifest);
 
-        // The version being replaced finishes its work first; the build above
-        // ran while it carried on, so the drain is the only pause it sees.
+        /* The version being replaced finishes its work first; the build above
+           ran while it carried on, so the drain is the only pause it sees. A
+           layer that can roll in place keeps it up past that, still serving,
+           until the new version has taken over. */
         if (deployment.replaces) {
             if (deployment.replaces.status === "running") await this.drain(deployment, deployment.replaces);
             if (!this.isCurrent(deployment)) return;
-            await this.stop(deployment.replaces);
+            if (! this.rollsInPlace(deployment)) await this.stop(deployment.replaces);
         }
 
         if (!this.isCurrent(deployment)) return;
         await this.start(deployment, appDir, manifest.main);
-        await this.awaitLive(deployment);
+        if (this.rollsInPlace(deployment)) {
+            await this.awaitRolled(deployment);
+        } else {
+            await this.awaitLive(deployment);
+        }
     }
 
     protected isCurrent(deployment : Deployment) : boolean {
         return this.deployments.get(deployment.appName) === deployment && deployment.status === "building";
     }
 
-    private async awaitLive(deployment : Deployment) : Promise<void> {
+    /* Whether this deploy replaces the old version without stopping it first.
+       Only a layer whose backend can run two versions of one app side by side
+       and swap between them says yes; the default cannot, and the option on
+       the deploy is then simply not honoured. */
+    protected rollsInPlace(_deployment : Deployment) : boolean {
+        return false;
+    }
+
+    // Waits for a roll-in-place to finish; a layer that rolls must say how.
+    protected async awaitRolled(deployment : Deployment) : Promise<void> {
+        await this.awaitLive(deployment);
+    }
+
+    protected async awaitLive(deployment : Deployment) : Promise<void> {
         const deadline = Date.now() + this.livenessTimeoutMs;
 
         while (Date.now() < deadline) {

package/src/app-management/BuildLayer.ts

@@ -27,6 +27,15 @@ type DeploymentSummary = {
     draining? : Draining,
 };
 
+/* How a deploy replaces the running version. By default the old version is
+   drained and stopped, then the new one started, so there is a gap. With
+   zeroDowntime the new version is brought up beside the old once the drain is
+   done and the old is let go only when the new answers - on a build layer that
+   can do that; one that cannot ignores it. */
+type DeployOptions = {
+    zeroDowntime? : boolean,
+};
+
 interface BuildLayer {
 
     /* Moves a running app onto a different size of instance. The app is rolled
@@ -35,7 +44,7 @@ interface BuildLayer {
     resize(appName : string, size : AppSize) : Promise<DeploymentSummary | undefined>;
 
     ensureHydrated() : Promise<void>;
-    deploy(appName : string, appPort : number, tarball : Buffer) : DeploymentSummary;
+    deploy(appName : string, appPort : number, tarball : Buffer, options? : DeployOptions) : DeploymentSummary;
     status(appName : string) : (DeploymentSummary & { log : Array<string> }) | undefined;
     list() : Array<DeploymentSummary>;
     ping(appName : string) : Promise<boolean>;
@@ -61,5 +70,5 @@ interface BuildLayer {
 
 }
 
-export type { BuildLayer, DeploymentStatus, DeploymentSummary, Draining, AppSize };
+export type { BuildLayer, DeploymentStatus, DeploymentSummary, Draining, AppSize, DeployOptions };
 export { APP_SIZES, isAppSize };

package/src/app-management/FargateBuildLayer.ts

@@ -35,6 +35,10 @@ type FargateBuildLayerOptions = {
     sqlDatabaseUrlSecretArn? : string,
     sqlSchema? : string,
     docGenerator? : DocGenerator,
+    /* Put on every task definition and service this layer makes, and from
+       there on every task, so the cost of a tenant's apps can be told apart
+       on the bill. Empty means untagged. */
+    tags? : Record<string, string>,
 };
 
 type AwsClients = {
@@ -55,6 +59,7 @@ const defaultClients = (region : string) : AwsClients => ({
 
 const BUILD_TIMEOUT_MS = 900_000;
 const FARGATE_LIVENESS_TIMEOUT_MS = 420_000;
+const ROLL_POLL_INTERVAL_MS = 5_000;
 
 /* What each size is on Fargate, in the units ECS wants. The memory is what
    the console describes as "2× the RAM"; the CPU is not advertised, only felt. */
@@ -65,6 +70,22 @@ const TASK_SIZES : Record<AppSize, { cpu : string, memory : string }> = {
 
 const sizeOfTask = (memory? : string) : AppSize => memory === TASK_SIZES.large.memory ? "large" : "small";
 
+/* What makes a roll zero-downtime: ECS only retires the old task once the
+   new one is healthy, and without a health check "healthy" means "started",
+   before the app inside is listening. This asks the app's admin port the
+   same `ping` the platform's own probe sends. Only a task with zero-downtime
+   deploys carries it, so nothing already running is touched. The start
+   period is Fargate's longest, for apps that build at boot. */
+const adminHealthCheck = (adminPort : number) => ({
+    command: ["CMD-SHELL", `node -e "const s=require('net').connect(${adminPort},'127.0.0.1');`
+        + `s.setTimeout(3000,()=>process.exit(1));s.on('connect',()=>s.write('ping\\n'));`
+        + `s.on('data',d=>process.exit(String(d).startsWith('ok')?0:1));s.on('error',()=>process.exit(1))"`],
+    interval: 10,
+    timeout: 5,
+    retries: 3,
+    startPeriod: 300,
+});
+
 class FargateBuildLayer extends BaseBuildLayer {
 
     private aws : AwsClients;
@@ -135,6 +156,50 @@ class FargateBuildLayer extends BaseBuildLayer {
         await this.upsertService(deployment, taskDefinition, registryArn);
     }
 
+    /* ECS can run the new task beside the old one and swap the service over
+       once the new one is up, which is what a zero-downtime deploy buys: the
+       old version, already drained, keeps answering until then. */
+    protected rollsInPlace(deployment : Deployment) : boolean {
+        return deployment.zeroDowntime && deployment.replaces !== undefined;
+    }
+
+    /* The roll is done when ECS says so, not when something answers on the
+       app's name: during the roll that name resolves to the old task as
+       readily as the new, so a probe would call the deploy live too soon.
+       ECS reports one deployment, completed, once the old task is gone. */
+    protected async awaitRolled(deployment : Deployment) : Promise<void> {
+        const deadline = Date.now() + this.livenessTimeoutMs;
+
+        while (Date.now() < deadline) {
+            if (deployment.status !== "building") return;
+            if (await this.rolled(deployment.appName)) {
+                deployment.status = "running";
+                this.log(deployment, "app is live: the service has rolled over to the new version");
+                return;
+            }
+            await new Promise(resolve => setTimeout(resolve, ROLL_POLL_INTERVAL_MS));
+        }
+
+        /* The old version is still serving - ECS keeps it until the new one
+           is healthy - so nothing is torn down; the deploy is reported as
+           failed and the app's log says why it never came up. */
+        deployment.status = "failed";
+        this.log(deployment, `the service did not roll over within ${this.livenessTimeoutMs / 1000}s; the previous version is still serving`);
+        for (const line of await this.diagnostics(deployment).catch(() => [])) this.log(deployment, `  ${line}`);
+    }
+
+    private async rolled(appName : string) : Promise<boolean> {
+        const {services} = await this.aws.ecs.send(new DescribeServicesCommand({
+            cluster: this.options.cluster,
+            services: [this.serviceNameFor(appName)],
+        }));
+        const deployments : Array<{ status? : string, rolloutState? : string, runningCount? : number }> = services?.[0]?.deployments ?? [];
+        const primary = deployments.find(each => each.status === "PRIMARY");
+
+        return deployments.length === 1 && primary !== undefined
+            && (primary.rolloutState === "COMPLETED" || (primary.rolloutState === undefined && (primary.runningCount ?? 0) > 0));
+    }
+
     protected async stop(deployment : Deployment) : Promise<void> {
         if (this.deployments.get(deployment.appName) !== deployment) return;
 
@@ -192,6 +257,7 @@ class FargateBuildLayer extends BaseBuildLayer {
         const registered = await this.aws.ecs.send(new RegisterTaskDefinitionCommand({
             family: this.serviceNameFor(deployment.appName),
             requiresCompatibilities: ["FARGATE"],
+            ...this.taggedWith(),
             networkMode: "awsvpc",
             ...TASK_SIZES[deployment.size],
             executionRoleArn: this.options.appExecutionRoleArn,
@@ -203,6 +269,7 @@ class FargateBuildLayer extends BaseBuildLayer {
                 // The most Fargate allows between SIGTERM and SIGKILL, so an
                 // app signalled with steps still in hand gets to finish them.
                 stopTimeout: 120,
+                ...(deployment.zeroDowntime ? { healthCheck: adminHealthCheck(deployment.adminPort) } : {}),
                 portMappings: [
                     { containerPort: deployment.appPort, protocol: "tcp" },
                     { containerPort: deployment.adminPort, protocol: "tcp" },
@@ -280,6 +347,7 @@ class FargateBuildLayer extends BaseBuildLayer {
                 taskDefinition,
                 desiredCount: 1,
                 forceNewDeployment: true,
+                ...this.propagatingTags(),
             }));
             return;
         }
@@ -291,6 +359,8 @@ class FargateBuildLayer extends BaseBuildLayer {
             taskDefinition,
             desiredCount: 1,
             launchType: "FARGATE",
+            ...this.taggedWith(),
+            ...this.propagatingTags(),
             networkConfiguration: {
                 awsvpcConfiguration: {
                     subnets: this.options.subnets,
@@ -358,6 +428,7 @@ class FargateBuildLayer extends BaseBuildLayer {
             appPort,
             appHost: this.appHostFor(appName),
             size: sizeOfTask(taskDefinition.memory),
+            zeroDowntime: false,
             adminPort: port("ANBARIC_ADMIN_PORT", 8791),
             consumerPort: port("ANBARIC_CONSUMER_PORT", 0),
             log: [],
@@ -368,6 +439,16 @@ class FargateBuildLayer extends BaseBuildLayer {
         return `anbaric-app-${appName}`;
     }
 
+    private taggedWith() : { tags? : Array<{ key : string, value : string }> } {
+        const entries = Object.entries(this.options.tags ?? {});
+        return entries.length === 0 ? {} : { tags: entries.map(([key, value]) => ({ key, value })) };
+    }
+
+    // The task definition's tags become each task's, which is where the cost is.
+    private propagatingTags() : { propagateTags? : "TASK_DEFINITION" } {
+        return Object.keys(this.options.tags ?? {}).length === 0 ? {} : { propagateTags: "TASK_DEFINITION" };
+    }
+
     private buildKeyFor(appName : string) : string {
         return `builds/${appName}.tar.gz`;
     }

package/src/data-store/AppSettingsStore.ts

@@ -0,0 +1,30 @@
+/* What a builder has chosen for an app beyond its code: the things set in
+   the console that a redeploy must not forget. Kept by the platform rather
+   than inferred from the running infrastructure, so changing one never
+   restarts anything. */
+type AppSettings = {
+    appName : string,
+    /* A deploy brings the new version up beside the old and only then lets
+       the old go, instead of stopping the old first. Priced, and off unless
+       the builder turns it on. */
+    zeroDowntime : boolean,
+};
+
+const DEFAULT_SETTINGS : Omit<AppSettings, "appName"> = { zeroDowntime: false };
+
+interface AppSettingsStore {
+
+    // The defaults for an app nobody has configured.
+    get(appName : string) : Promise<AppSettings>;
+
+    set(appName : string, settings : Partial<Omit<AppSettings, "appName">>) : Promise<AppSettings>;
+
+    list() : Promise<Array<AppSettings>>;
+
+    // Forgotten with the app, so a later app of the same name starts afresh.
+    remove(appName : string) : Promise<void>;
+
+}
+
+export type { AppSettings, AppSettingsStore };
+export { DEFAULT_SETTINGS };

package/src/data-store/InMemoryAppSettingsStore.ts

@@ -0,0 +1,27 @@
+import {AppSettings, AppSettingsStore, DEFAULT_SETTINGS} from "./AppSettingsStore";
+
+class InMemoryAppSettingsStore implements AppSettingsStore {
+
+    private settings = new Map<string, AppSettings>();
+
+    async get(appName : string) : Promise<AppSettings> {
+        return { ...(this.settings.get(appName) ?? { appName, ...DEFAULT_SETTINGS }) };
+    }
+
+    async set(appName : string, settings : Partial<Omit<AppSettings, "appName">>) : Promise<AppSettings> {
+        const updated = { ...(await this.get(appName)), ...settings, appName };
+        this.settings.set(appName, updated);
+        return { ...updated };
+    }
+
+    async list() : Promise<Array<AppSettings>> {
+        return [...this.settings.values()].map(settings => ({ ...settings }));
+    }
+
+    async remove(appName : string) : Promise<void> {
+        this.settings.delete(appName);
+    }
+
+}
+
+export { InMemoryAppSettingsStore };

package/src/data-store/PlatformMigrations.ts

@@ -262,6 +262,17 @@ const PLATFORM_MIGRATIONS : Array<Migration> = [
         "CREATE INDEX IF NOT EXISTS audit_records_at ON anbaric_system.audit_records (at)",
     ),
 
+    /* What a builder has set for an app in the console, kept apart from the
+       app's running infrastructure so that changing a setting never restarts
+       it. An app with no row has the defaults. */
+    statements("007-app-settings",
+        `CREATE TABLE IF NOT EXISTS app_settings (
+            app_name      TEXT PRIMARY KEY,
+            zero_downtime BOOLEAN NOT NULL DEFAULT false,
+            updated_at    TIMESTAMPTZ NOT NULL DEFAULT now()
+        )`,
+    ),
+
 ];
 
 export { PLATFORM_MIGRATIONS };

package/src/data-store/PostgresAppSettingsStore.ts

@@ -0,0 +1,40 @@
+import {Pool} from "pg";
+import {AppSettings, AppSettingsStore, DEFAULT_SETTINGS} from "./AppSettingsStore";
+
+const rowToSettings = (row : any) : AppSettings => ({
+    appName: row.app_name,
+    zeroDowntime: row.zero_downtime,
+});
+
+class PostgresAppSettingsStore implements AppSettingsStore {
+
+    constructor(private pool : Pool) {}
+
+    async get(appName : string) : Promise<AppSettings> {
+        const result = await this.pool.query(
+            "SELECT app_name, zero_downtime FROM app_settings WHERE app_name = $1", [appName]);
+        return result.rows[0] ? rowToSettings(result.rows[0]) : { appName, ...DEFAULT_SETTINGS };
+    }
+
+    async set(appName : string, settings : Partial<Omit<AppSettings, "appName">>) : Promise<AppSettings> {
+        const merged = { ...(await this.get(appName)), ...settings };
+        const result = await this.pool.query(
+            `INSERT INTO app_settings (app_name, zero_downtime) VALUES ($1, $2)
+             ON CONFLICT (app_name) DO UPDATE SET zero_downtime = EXCLUDED.zero_downtime, updated_at = now()
+             RETURNING app_name, zero_downtime`,
+            [appName, merged.zeroDowntime]);
+        return rowToSettings(result.rows[0]);
+    }
+
+    async list() : Promise<Array<AppSettings>> {
+        const result = await this.pool.query("SELECT app_name, zero_downtime FROM app_settings ORDER BY app_name");
+        return result.rows.map(rowToSettings);
+    }
+
+    async remove(appName : string) : Promise<void> {
+        await this.pool.query("DELETE FROM app_settings WHERE app_name = $1", [appName]);
+    }
+
+}
+
+export { PostgresAppSettingsStore };

package/src/hosting/HostingServer.ts

@@ -13,6 +13,7 @@ import {MyAppsHandler} from "./handlers/MyAppsHandler";
 import {MembersHandler} from "./handlers/auth/MembersHandler";
 import {BillingHandler} from "./handlers/BillingHandler";
 import {CentralBilling} from "../billing/CentralBilling";
+import {AppSettingsStore} from "../data-store/AppSettingsStore";
 import {Subdomains} from "../subdomains/Subdomains";
 import {AuditsHandler} from "./handlers/AuditsHandler";
 import {AuthorizeCliHandler} from "./handlers/auth/AuthorizeCliHandler";
@@ -81,7 +82,8 @@ class HostingServer {
                 promptManagerFor? : (appId : string) => PromptManager,
                 fileStorageFor? : (appId : string) => FileStorage,
                 subdomains? : Subdomains,
-                billing? : CentralBilling) {
+                billing? : CentralBilling,
+                appSettings? : AppSettingsStore) {
         const pages = new PagesHandler();
         const ping = new PingHandler(tenant);
         const jobs = new JobsHandler(persistence);
@@ -130,7 +132,7 @@ class HostingServer {
             buildLayer.consumerUrlsFor = appName =>
                 [...new Set(registry.list().filter(consumer => consumer.appId === appName).map(consumer => consumer.url))];
             buildLayer.appRemoved = appName => void subdomains?.release(appName);
-            publicRouter.registerApi("apps", new AppsHandler(buildLayer, tenant, subdomains));
+            publicRouter.registerApi("apps", new AppsHandler(buildLayer, tenant, subdomains, appSettings));
             publicRouter.register("app", appProxy);
 
             // The directory a person who is not a builder sees instead of the

package/src/hosting/handlers/AppsHandler.ts

@@ -6,11 +6,12 @@ import {appAddress} from "../../app-management/appAddress";
 import {redactEmails} from "../../app-management/redactEmails";
 import {RequestHandler} from "../RequestHandler";
 import {Subdomains} from "../../subdomains/Subdomains";
+import {AppSettingsStore} from "../../data-store/AppSettingsStore";
 
 class AppsHandler implements RequestHandler {
 
     constructor(private buildLayer : BuildLayer, private tenant? : string,
-                private subdomains? : Subdomains) {}
+                private subdomains? : Subdomains, private settings? : AppSettingsStore) {}
 
     async handle(request : Request) : Promise<void> {
         // Changing what is deployed is a builder's job; a USER may look.
@@ -35,6 +36,9 @@ class AppsHandler implements RequestHandler {
             case "size":
                 if (request.id) return this.handleSize(request, request.id);
                 break;
+            case "zero-downtime":
+                if (request.id) return this.handleZeroDowntime(request, request.id);
+                break;
             case undefined:
                 if (request.id) return this.handleApp(request, request.id);
                 return this.handleCollection(request);
@@ -44,10 +48,30 @@ class AppsHandler implements RequestHandler {
 
     private changesApps(request : Request) : boolean {
         if (request.subresource === "deploy" || request.subresource === "docs") return true;
-        if (request.subresource === "subdomain" || request.subresource === "size") return request.method === "PUT";
+        if (["subdomain", "size", "zero-downtime"].includes(request.subresource ?? "")) return request.method === "PUT";
         return request.subresource === undefined && request.method === "DELETE";
     }
 
+    /* Whether the app's deploys keep the old version up until the new one
+       answers. A builder's choice, priced per app per month as the console
+       says; it takes effect from the next deploy and restarts nothing now. */
+    private async handleZeroDowntime(request : Request, appName : string) : Promise<void> {
+        if (request.method !== "PUT") return request.notFound();
+        if (! this.settings) return request.reply(409, { error: "This platform does not offer zero-downtime deploys" });
+        if (! this.buildLayer.status(appName)) return request.reply(404, { error: `No app named "${appName}"` });
+
+        const body = await request.body() as { on? : unknown } | undefined;
+        if (typeof body?.on !== "boolean") return request.reply(400, { error: "Expected a body of { on : boolean }" });
+
+        const settings = await this.settings.set(appName, { zeroDowntime: body.on });
+        return request.reply(200, { appName, zeroDowntime: settings.zeroDowntime });
+    }
+
+    private async zeroDowntimeOf(appName : string) : Promise<boolean | undefined> {
+        if (! this.settings) return undefined;
+        return (await this.settings.get(appName)).zeroDowntime;
+    }
+
     /* How big the app's instance is, which is a builder's to change and costs
        what the console told them it would. The app is rolled onto the new
        size; what it is doing is drained first, as any deploy does. */
@@ -110,7 +134,7 @@ class AppsHandler implements RequestHandler {
                 const tarball = await request.rawBody();
                 if (tarball.length === 0) return request.reply(400, { error: "Expected a gzipped tarball body" });
 
-                const summary = this.buildLayer.deploy(appName, appPort, tarball);
+                const summary = this.buildLayer.deploy(appName, appPort, tarball, { zeroDowntime: await this.zeroDowntimeOf(appName) });
 
                 /* An address is asked for alongside the build rather than
                    after it: allocation is idempotent, so a redeploy costs
@@ -134,11 +158,13 @@ class AppsHandler implements RequestHandler {
                 if (!status) return request.reply(404, { error: `No app named "${appName}"` });
                 const live = await this.buildLayer.ping(appName);
                 const subdomain = await this.addressOf(appName);
-                return request.reply(200, { ...status, live, subdomain, url: appAddress(appName, subdomain) });
+                const zeroDowntime = await this.zeroDowntimeOf(appName);
+                return request.reply(200, { ...status, live, subdomain, url: appAddress(appName, subdomain), zeroDowntime });
             }
             case "DELETE": {
                 const outcome = await this.buildLayer.teardown(appName);
                 if (!outcome) return request.reply(404, { error: `No app named "${appName}"` });
+                await this.settings?.remove(appName);
 
                 // The address is let go by the build layer's appRemoved hook,
                 // which an app that drains reaches minutes after this returns.
@@ -192,9 +218,11 @@ class AppsHandler implements RequestHandler {
         switch (request.method) {
             case "GET": {
                 const addresses = new Map((await this.addresses()).map(held => [held.appName, held.subdomain]));
+                const settings = new Map((await this.settings?.list() ?? []).map(each => [each.appName, each]));
                 return request.reply(200, this.buildLayer.list().map(app => {
                     const subdomain = addresses.get(app.appName);
-                    return { ...app, subdomain, url: appAddress(app.appName, subdomain) };
+                    const zeroDowntime = this.settings ? settings.get(app.appName)?.zeroDowntime ?? false : undefined;
+                    return { ...app, subdomain, url: appAddress(app.appName, subdomain), zeroDowntime };
                 }));
             }
         }