@podosoft/podokit 1.6.2 → 1.6.4

package/dist/deploy-checks.d.ts

@@ -30,6 +30,8 @@ export interface ConnectionBudget {
      */
     rolloutProcesses: number;
     rollout: number;
+    /** What the rollout count includes beyond the steady processes. */
+    rolloutOverlap: string;
     migration: number;
     peak: number;
 }
@@ -45,11 +47,27 @@ export interface ConnectionBudget {
  * starts the next surge pod as soon as the previous replacement is ready. When
  * replacements become ready faster than replaced pods exit, every old pod of a
  * workload can still hold its pools while all of its new pods run, twice its
- * replicas whatever the surge. Compose stops a container before it starts the
- * replacement, so it never runs both.
+ * replicas whatever the surge. Compose starts the new API containers beside the
+ * old ones and stops the old ones once the new ones are healthy, so it runs twice
+ * the API replicas; it stops the worker before it starts the replacement.
  */
 export declare function connectionBudget(profile: AnyDeployProfile, modules: Set<string>): ConnectionBudget | null;
 /** The plan warning for a budget over the limit, or null. */
 export declare function connectionBudgetWarning(profile: AnyDeployProfile, modules: Set<string>): string | null;
+/** The ADMIN_EMAILS entries the API reads, from a value that may still carry an env file's quotes. */
+export declare function adminEmailEntries(value: string): string[];
+/** Why an ADMIN_EMAILS entry cannot be an administrator's real address, or null when it can be. */
+export declare function adminEmailProblem(entry: string): string | null;
+/**
+ * Whether ADMIN_EMAILS names real addresses. A placeholder such as
+ * admin@example.com names an administrator nobody can be, and a generated API
+ * refuses to start in production with one, so the doctor fails before a rollout.
+ */
+export declare function adminEmailsFinding(value: string | undefined, source: string): DeployFinding;
+/**
+ * Whether a driver must read ADMIN_EMAILS from the API secret on the target: the
+ * auth module reads it, and runtimeConfig, which profileFindings checks, does not set it.
+ */
+export declare function adminEmailsOnTarget(projectRoot: string, profile: AnyDeployProfile): boolean;
 export declare function profileFindings(projectRoot: string, profile: AnyDeployProfile): DeployFinding[];
 export declare function formatFinding(finding: DeployFinding): string;

package/dist/deploy-checks.js

@@ -4,6 +4,10 @@ exports.DEFAULT_MAX_CONNECTIONS = void 0;
 exports.installedModules = installedModules;
 exports.connectionBudget = connectionBudget;
 exports.connectionBudgetWarning = connectionBudgetWarning;
+exports.adminEmailEntries = adminEmailEntries;
+exports.adminEmailProblem = adminEmailProblem;
+exports.adminEmailsFinding = adminEmailsFinding;
+exports.adminEmailsOnTarget = adminEmailsOnTarget;
 exports.profileFindings = profileFindings;
 exports.formatFinding = formatFinding;
 const app_proxy_timeouts_1 = require("./app-proxy-timeouts");
@@ -114,8 +118,9 @@ function poolSize(runtimeConfig, key, fallback) {
  * starts the next surge pod as soon as the previous replacement is ready. When
  * replacements become ready faster than replaced pods exit, every old pod of a
  * workload can still hold its pools while all of its new pods run, twice its
- * replicas whatever the surge. Compose stops a container before it starts the
- * replacement, so it never runs both.
+ * replicas whatever the surge. Compose starts the new API containers beside the
+ * old ones and stops the old ones once the new ones are healthy, so it runs twice
+ * the API replicas; it stops the worker before it starts the replacement.
  */
 function connectionBudget(profile, modules) {
     const postgres = profile.dependencies.postgres;
@@ -138,7 +143,9 @@ function connectionBudget(profile, modules) {
     }
     const worker = profile.workloads.worker;
     const steadyProcesses = profile.workloads.api.replicas + (worker?.replicas ?? 0);
-    const rolloutProcesses = profile.driver === "kubernetes-helm" ? 2 * steadyProcesses : steadyProcesses;
+    const rolloutProcesses = profile.driver === "kubernetes-helm"
+        ? 2 * steadyProcesses
+        : steadyProcesses + profile.workloads.api.replicas;
     const maxConnections = postgres.maxConnections ?? exports.DEFAULT_MAX_CONNECTIONS;
     const steady = steadyProcesses * perProcess;
     const rollout = rolloutProcesses * perProcess;
@@ -151,6 +158,9 @@ function connectionBudget(profile, modules) {
         steady,
         rolloutProcesses,
         rollout,
+        rolloutOverlap: profile.driver === "kubernetes-helm"
+            ? "replaced pods that have not exited"
+            : "old API containers that have not stopped",
         migration: MIGRATION_CONNECTIONS,
         peak: Math.max(rollout, steady + MIGRATION_CONNECTIONS),
     };
@@ -159,7 +169,7 @@ function budgetSummary(budget) {
     return (`each API or worker process holds up to ${budget.perProcess} (${budget.perProcessTerms.join(" + ")}); ` +
         `${budget.steadyProcesses} process(es) hold ${budget.steady}` +
         (budget.rolloutProcesses > budget.steadyProcesses
-            ? `, ${budget.rolloutProcesses} during a rollout, counting replaced pods that have not exited, hold ${budget.rollout}`
+            ? `, ${budget.rolloutProcesses} during a rollout, counting ${budget.rolloutOverlap}, hold ${budget.rollout}`
             : "") +
         `, and the migration adds ${budget.migration} to the ${budget.steady}`);
 }
@@ -257,6 +267,72 @@ function clientAddressFinding(profile) {
         : "";
     return { code, ok: true, message: `With exposure.edgeProxies ${edge}, ${summary}.${traefik}` };
 }
+/** Domains and top-level names reserved for documentation and testing (RFC 2606, RFC 6761), which receive no mail. */
+const PLACEHOLDER_DOMAINS = ["example.com", "example.net", "example.org"];
+const PLACEHOLDER_TOP_LEVEL_NAMES = new Set(["example", "test", "invalid", "localhost"]);
+/** The ADMIN_EMAILS entries the API reads, from a value that may still carry an env file's quotes. */
+function adminEmailEntries(value) {
+    return value
+        .trim()
+        .replace(/^(["'])([\s\S]*)\1$/, "$2")
+        .split(",")
+        .map((entry) => entry.trim().toLowerCase())
+        .filter(Boolean);
+}
+/** Why an ADMIN_EMAILS entry cannot be an administrator's real address, or null when it can be. */
+function adminEmailProblem(entry) {
+    const at = entry.lastIndexOf("@");
+    if (at >= 0) {
+        const domain = entry.slice(at + 1);
+        const topLevel = domain.split(".").at(-1) ?? "";
+        if (PLACEHOLDER_DOMAINS.some((reserved) => domain === reserved || domain.endsWith(`.${reserved}`)) ||
+            PLACEHOLDER_TOP_LEVEL_NAMES.has(topLevel)) {
+            return `a placeholder: ${domain} is reserved and receives no mail`;
+        }
+    }
+    return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(entry) ? null : "not an email address";
+}
+/**
+ * Whether ADMIN_EMAILS names real addresses. A placeholder such as
+ * admin@example.com names an administrator nobody can be, and a generated API
+ * refuses to start in production with one, so the doctor fails before a rollout.
+ */
+function adminEmailsFinding(value, source) {
+    const code = "admin-emails";
+    const entries = adminEmailEntries(value ?? "");
+    if (entries.length === 0) {
+        return {
+            code,
+            ok: true,
+            message: `${source} does not set ADMIN_EMAILS, so no account becomes an administrator through it; ` +
+                "administrators come from admin:bootstrap or the admin console.",
+        };
+    }
+    const problems = entries.flatMap((entry) => {
+        const problem = adminEmailProblem(entry);
+        return problem ? [`${entry} (${problem})`] : [];
+    });
+    if (problems.length) {
+        return {
+            code,
+            ok: false,
+            message: `ADMIN_EMAILS in ${source} lists ${problems.join(", ")}. In production it must list the administrators' ` +
+                "real addresses or stay empty; a generated API refuses to start in production with such an entry.",
+        };
+    }
+    return {
+        code,
+        ok: true,
+        message: `ADMIN_EMAILS in ${source} lists ${entries.length} administrator address${entries.length === 1 ? "" : "es"}.`,
+    };
+}
+/**
+ * Whether a driver must read ADMIN_EMAILS from the API secret on the target: the
+ * auth module reads it, and runtimeConfig, which profileFindings checks, does not set it.
+ */
+function adminEmailsOnTarget(projectRoot, profile) {
+    return installedModules(projectRoot).has("auth") && profile.runtimeConfig.ADMIN_EMAILS === undefined;
+}
 function profileFindings(projectRoot, profile) {
     const modules = installedModules(projectRoot);
     const findings = [
@@ -268,6 +344,9 @@ function profileFindings(projectRoot, profile) {
         findings.push(budget);
     if (modules.has("audit-log"))
         findings.push(auditLogFinding(profile.runtimeConfig));
+    if (modules.has("auth") && profile.runtimeConfig.ADMIN_EMAILS !== undefined) {
+        findings.push(adminEmailsFinding(profile.runtimeConfig.ADMIN_EMAILS, "runtimeConfig"));
+    }
     return findings;
 }
 function formatFinding(finding) {

package/dist/deploy-compose-render.js

@@ -19,6 +19,13 @@ const lockfile_1 = require("./lockfile");
 const toolchain_1 = require("./toolchain");
 /** The internal port every generated application service listens on. */
 const CONTAINER_PORT = 3000;
+/**
+ * Bun caches a host name's addresses for 30 seconds and keeps connecting to a
+ * cached address after it stops answering. The web server reaches the API by its
+ * service name, which a rollout moves to new containers, so with the cache it
+ * would keep calling the old API's address after that container is gone.
+ */
+const WEB_DNS_CONFIG = { BUN_CONFIG_DNS_TIME_TO_LIVE_SECONDS: "0" };
 function quote(value) {
     return JSON.stringify(value);
 }
@@ -262,7 +269,7 @@ function applicationService(profile, role, image, release, stateDigest, environm
                 "    healthcheck:\n      disable: true\n") +
         labelsBlock(profile, release, stateDigest, "    "));
 }
-function gatewayService(profile, image, release, stateDigest) {
+function gatewayService(profile, image) {
     const globalOptions = profile.exposure.trustedProxyCidrs.length
         ? `{
   servers {
@@ -318,7 +325,10 @@ function gatewayService(profile, image, release, stateDigest) {
         `    healthcheck:\n` +
         `      test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:2019/config/"]\n` +
         `      interval: 10s\n      timeout: 5s\n      retries: 6\n      start_period: 10s\n` +
-        labelsBlock(profile, release, stateDigest, "    "));
+        // No release or rollout-state label: the gateway runs none of the release's
+        // code, and a label that changes with every release would make Compose
+        // recreate it, which closes the published port until the new one starts.
+        `    labels:\n      io.podokit.project: ${quote(profile.target.project)}\n`);
 }
 function renderComposeDocument(profile, release, images, stateDigest, proxyTimeouts, workerCommand = ["bun", "dist/main-worker.js"]) {
     const environment = derivedRuntimeConfig(profile, proxyTimeouts);
@@ -339,7 +349,8 @@ function renderComposeDocument(profile, release, images, stateDigest, proxyTimeo
     services.push(applicationService(profile, "api", images.api, release, stateDigest, environment, {
         healthcheckPath: "/health/ready",
     }));
-    services.push(applicationService(profile, "web", images.web, release, stateDigest, environment, {
+    const webEnvironment = { ...WEB_DNS_CONFIG, ...environment };
+    services.push(applicationService(profile, "web", images.web, release, stateDigest, webEnvironment, {
         // The web server's own route: an API outage must not mark it unhealthy.
         healthcheckPath: "/healthz",
         publish: !hasGateway,
@@ -348,7 +359,7 @@ function renderComposeDocument(profile, release, images, stateDigest, proxyTimeo
         if (!images.gateway) {
             throw new Error("A resolved gateway image is required when WebSocket or API paths are configured.");
         }
-        services.push(gatewayService(profile, images.gateway, release, stateDigest));
+        services.push(gatewayService(profile, images.gateway));
     }
     if (profile.workloads.worker) {
         // The worker runs the API image with a different entry point, exactly as the

package/dist/deploy-compose.js

@@ -63,32 +63,21 @@ function inspectComposeEndpointFingerprint(context, runner = deploy_compose_exec
     return digest(["docker-endpoint", host, daemonId]);
 }
 /**
- * Reads key NAMES from an env file on the target host.
+ * Runs a read-only shell command against an env file on the target host, as the
+ * identity that will actually deploy, and returns its output.
  *
- * The file never leaves the host and no value is captured: a throwaway container
- * mounts it read-only and prints the left-hand sides only. This mirrors the
- * Kubernetes doctor asking `kubectl` for key names rather than Secret contents.
+ * The file never leaves the host: over SSH the command reads it in place, and
+ * otherwise a throwaway container mounts it read-only.
  */
-function readEnvFileKeys(profile, path, image, runner) {
-    const script = `sed -n 's/^[[:space:]]*\\([A-Z][A-Z0-9_]*\\)=.*/\\1/p' /podokit-env | sort -u`;
-    // Read it as the identity that will actually deploy.
-    //
+function readEnvFile(profile, path, image, runner, command) {
     // A container running as root proves only that root can read the file, and
     // `docker compose` on a remote target runs as the SSH user. Checking with the
     // wrong identity is how a doctor passes and the apply then dies on
     // "permission denied" -- which is exactly what happened.
     const destination = (0, deploy_compose_exec_1.sshDestination)(profile, runner);
-    if (destination) {
-        const remote = (0, deploy_compose_exec_1.checked)(runner, "ssh", [
-            destination,
-            `sed -n 's/^[[:space:]]*\\([A-Z][A-Z0-9_]*\\)=.*/\\1/p' ${path} | sort -u`,
-        ]);
-        return remote
-            .split("\n")
-            .map((line) => line.trim())
-            .filter(Boolean);
-    }
-    const output = (0, deploy_compose_exec_1.checked)(runner, "docker", (0, deploy_compose_exec_1.dockerArgs)(profile, [
+    if (destination)
+        return (0, deploy_compose_exec_1.checked)(runner, "ssh", [destination, command(path)]);
+    return (0, deploy_compose_exec_1.checked)(runner, "docker", (0, deploy_compose_exec_1.dockerArgs)(profile, [
         "run",
         "--rm",
         "--network",
@@ -104,13 +93,36 @@ function readEnvFileKeys(profile, path, image, runner) {
         "sh",
         image,
         "-c",
-        script,
+        command("/podokit-env"),
     ]));
-    return output
+}
+/**
+ * Reads key NAMES from an env file on the target host. No value is captured: the
+ * command prints the left-hand sides only, which mirrors the Kubernetes doctor
+ * asking `kubectl` for key names rather than Secret contents.
+ */
+function readEnvFileKeys(profile, path, image, runner) {
+    return readEnvFile(profile, path, image, runner, (file) => `sed -n 's/^[[:space:]]*\\([A-Z][A-Z0-9_]*\\)=.*/\\1/p' ${file} | sort -u`)
         .split("\n")
         .map((line) => line.trim())
         .filter(Boolean);
 }
+// ADMIN_EMAILS is an allowlist of addresses, not a credential, so the doctor reads
+// this one value of the API env file to catch a placeholder. Every other value stays unread.
+function envFileAdminEmails(profile, image, runner) {
+    const path = profile.secrets.api.path;
+    try {
+        const lines = readEnvFile(profile, path, image, runner, (file) => `sed -n 's/^[[:space:]]*ADMIN_EMAILS=//p' ${file}`)
+            .split("\n")
+            .map((line) => line.trim())
+            .filter(Boolean);
+        // Docker Compose applies the last assignment of a key in an env file.
+        return (0, deploy_checks_1.adminEmailsFinding)(lines.at(-1), path);
+    }
+    catch (error) {
+        return { code: "admin-emails", ok: false, message: `Cannot read ADMIN_EMAILS from ${path}: ${error.message}` };
+    }
+}
 function writeLedger(profile, image, ledger, runner) {
     const encoded = Buffer.from(JSON.stringify(ledger)).toString("base64");
     (0, deploy_compose_exec_1.checked)(runner, "docker", (0, deploy_compose_exec_1.stateScript)(profile, image, `echo ${encoded} | base64 -d > ${deploy_compose_exec_1.LEDGER_PATH}.tmp && mv ${deploy_compose_exec_1.LEDGER_PATH}.tmp ${deploy_compose_exec_1.LEDGER_PATH}`));
@@ -171,6 +183,30 @@ function managedDependencyServices(profile) {
             : []),
     ];
 }
+/** The one-shot job that creates the bucket and the application user, when storage is managed. */
+function objectStorageInit(profile) {
+    return profile.dependencies.objectStorage.mode === "managed"
+        ? (0, deploy_compose_render_1.serviceName)(profile, "object-storage-init")
+        : null;
+}
+/**
+ * Runs the object storage init job and fails unless it exits 0.
+ *
+ * It runs with the dependencies, not in the application rollout: new API
+ * containers start without their dependencies, so the bucket and the application
+ * user must be reconciled before they do. `up --wait` cannot run it, because it
+ * reports a listed service that exits as a failure even with status 0, and an
+ * attached `up` would print the job's output, which names the access key.
+ */
+function runObjectStorageInit(profile, runtime, service, runner) {
+    (0, deploy_compose_exec_1.runCompose)(profile, runtime, "compose", ["up", "-d", "--no-deps", service], runner, false);
+    for (const container of serviceContainers(profile, service, runner)) {
+        const status = (0, deploy_compose_exec_1.checked)(runner, "docker", (0, deploy_compose_exec_1.dockerArgs)(profile, ["wait", container.id])).trim();
+        if (status !== "0") {
+            throw new Error(`${service} exited with status ${status}; inspect it with docker logs ${container.id}.`);
+        }
+    }
+}
 function doctorComposeDeployment(projectRoot, profileName, runner = deploy_compose_exec_1.defaultRunner, release) {
     const profile = (0, deploy_driver_1.loadComposeProfile)(projectRoot, profileName);
     const findings = (0, deploy_checks_1.profileFindings)(projectRoot, profile);
@@ -267,6 +303,8 @@ function doctorComposeDeployment(projectRoot, profileName, runner = deploy_compo
             record("env-file-" + file.label, false, `Cannot read ${file.path}: ${error.message}`);
         }
     }
+    if ((0, deploy_checks_1.adminEmailsOnTarget)(projectRoot, profile))
+        findings.push(envFileAdminEmails(profile, probe, runner));
     if (profile.secrets.registryLogin && release) {
         const result = runner("docker", (0, deploy_compose_exec_1.dockerArgs)(profile, ["run", "--rm", "--entrypoint", "true", probe]), { capture: true });
         record("registry-access", result.status === 0, result.status === 0
@@ -359,12 +397,13 @@ function planComposeDeployment(projectRoot, profileName, release, runner = deplo
     const psOutput = composePs(profile, runner);
     const actions = [];
     const dependencies = managedDependencyServices(profile);
+    const init = objectStorageInit(profile);
     let order = 1;
     if (dependencies.length) {
         actions.push({
             order: order++,
             kind: "dependencies",
-            description: `Reconcile and wait for ${dependencies.join(", ")}.`,
+            description: `Reconcile and wait for ${dependencies.join(", ")}${init ? `, then run ${init}` : ""}.`,
         });
     }
     actions.push({
@@ -376,7 +415,7 @@ function planComposeDeployment(projectRoot, profileName, release, runner = deplo
     actions.push({
         order: order++,
         kind: "application",
-        description: `Roll out api, web${profile.workloads.worker ? ", worker" : ""}${(0, deploy_compose_profile_1.composeHasGateway)(profile) ? ", gateway" : ""} and wait for health.${(0, api_paths_1.directApiRoutes)(profile.exposure.apiPaths)}`,
+        description: `${rolloutDescription(profile)}${(0, api_paths_1.directApiRoutes)(profile.exposure.apiPaths)}`,
     });
     actions.push({
         order: order++,
@@ -418,6 +457,146 @@ function planComposeDeployment(projectRoot, profileName, release, runner = deplo
         warnings,
     };
 }
+/**
+ * Which application services are replaced side by side.
+ *
+ * The API always is. Web is only behind the gateway: without one it publishes the
+ * host port itself, and a second container cannot bind it.
+ */
+function sideBySideServices(profile) {
+    return [
+        { service: (0, deploy_compose_render_1.serviceName)(profile, "api"), replicas: profile.workloads.api.replicas },
+        ...((0, deploy_compose_profile_1.composeHasGateway)(profile)
+            ? [{ service: (0, deploy_compose_render_1.serviceName)(profile, "web"), replicas: profile.workloads.web.replicas }]
+            : []),
+    ];
+}
+function rolloutDescription(profile) {
+    const replacedInPlace = [
+        ...((0, deploy_compose_profile_1.composeHasGateway)(profile) ? [] : ["web"]),
+        ...(profile.workloads.worker ? ["worker"] : []),
+    ];
+    return (`Start the new ${(0, deploy_compose_profile_1.composeHasGateway)(profile) ? "api and web" : "api"} beside the running containers and wait for health, ` +
+        (replacedInPlace.length ? `replace ${replacedInPlace.join(" and ")} in place, ` : "") +
+        `then stop the old containers${(0, deploy_compose_profile_1.composeHasGateway)(profile) ? "; the gateway is replaced only when its definition changes" : ""}.`);
+}
+function serviceContainers(profile, service, runner) {
+    const output = (0, deploy_compose_exec_1.checked)(runner, "docker", (0, deploy_compose_exec_1.dockerArgs)(profile, [
+        "ps",
+        "-a",
+        "--filter",
+        `label=com.docker.compose.project=${profile.target.project}`,
+        "--filter",
+        `label=com.docker.compose.service=${service}`,
+        "--format",
+        '{{.ID}}\t{{.State}}\t{{.Label "com.docker.compose.config-hash"}}',
+    ]));
+    return output
+        .split("\n")
+        .map((line) => line.trim().split("\t"))
+        .flatMap(([id, state, configHash]) => id ? [{ id, running: state === "running", configHash: configHash ?? "" }] : []);
+}
+/** The hash Compose stamps on a container of this service as the project file now defines it. */
+function composeConfigHash(profile, runtime, service, runner) {
+    const output = (0, deploy_compose_exec_1.runCompose)(profile, runtime, "compose", ["config", "--hash", service], runner, true);
+    const line = output.split("\n").find((entry) => entry.trim().startsWith(`${service} `));
+    const hash = line?.trim().split(/\s+/)[1];
+    if (!hash)
+        throw new Error(`docker compose config --hash did not report a hash for ${service}.`);
+    return hash;
+}
+/**
+ * Starts a service's new containers next to the running ones and waits for them.
+ *
+ * Compose recreates a changed service by stopping its container before it starts
+ * the replacement, and it stops every changed service before it starts any, so a
+ * single-host deployment had no API and no web server until the new ones passed
+ * their healthchecks. Scaling up with `--no-recreate` keeps the old containers
+ * serving while Compose creates new ones from the new definition, and both answer
+ * to the service's name on the project network until the old ones are stopped.
+ *
+ * A service with nothing running, or already running its new definition, is left
+ * to the final `compose up`: there is no traffic to keep, or nothing to replace.
+ */
+function startBeside(profile, runtime, service, replicas, runner) {
+    const before = serviceContainers(profile, service, runner);
+    const running = before.filter((container) => container.running);
+    if (!running.length)
+        return { service, replaced: [], started: [] };
+    const expected = composeConfigHash(profile, runtime, service, runner);
+    if (running.length === replicas && running.every((container) => container.configHash === expected)) {
+        return { service, replaced: [], started: [] };
+    }
+    // A stopped container serves nothing, and `up` would start it again with the
+    // old definition.
+    const stopped = before.filter((container) => !container.running).map((container) => container.id);
+    if (stopped.length)
+        (0, deploy_compose_exec_1.checked)(runner, "docker", (0, deploy_compose_exec_1.dockerArgs)(profile, ["rm", "-f", ...stopped]));
+    const replaced = running.map((container) => container.id);
+    try {
+        (0, deploy_compose_exec_1.runCompose)(profile, runtime, "compose", [
+            "up",
+            "-d",
+            "--no-deps",
+            "--no-recreate",
+            "--wait",
+            "--scale",
+            `${service}=${running.length + replicas}`,
+            service,
+        ], runner, false);
+    }
+    catch (error) {
+        removeContainers(profile, startedSince(profile, service, replaced, runner), runner);
+        throw error;
+    }
+    return { service, replaced, started: startedSince(profile, service, replaced, runner) };
+}
+function startedSince(profile, service, replaced, runner) {
+    return serviceContainers(profile, service, runner)
+        .map((container) => container.id)
+        .filter((id) => !replaced.includes(id));
+}
+function removeContainers(profile, ids, runner) {
+    if (ids.length)
+        runner("docker", (0, deploy_compose_exec_1.dockerArgs)(profile, ["rm", "-f", ...ids]), { capture: true });
+}
+/**
+ * Replaces the application containers without leaving a moment with nothing to
+ * answer.
+ *
+ * The new API, and the new web server behind the gateway, start beside the running
+ * ones. Only when all of them are healthy do the old ones get a graceful stop, web
+ * before API so an old web server never outlives the API it was talking to. The
+ * worker, a web server that publishes the port itself, and a changed gateway are
+ * replaced in place by the final `compose up`, which also removes orphans. If a new
+ * container fails its healthcheck, the containers this rollout started are removed
+ * and the old ones keep serving.
+ */
+function rollOutApplication(profile, runtime, runner) {
+    const sideBySide = [];
+    try {
+        for (const { service, replicas } of sideBySideServices(profile)) {
+            sideBySide.push(startBeside(profile, runtime, service, replicas, runner));
+        }
+        if (!(0, deploy_compose_profile_1.composeHasGateway)(profile)) {
+            // `--remove-orphans`: a gateway left from a profile that had one still holds
+            // the port this web server publishes.
+            (0, deploy_compose_exec_1.runCompose)(profile, runtime, "compose", ["up", "-d", "--no-deps", "--wait", "--remove-orphans", (0, deploy_compose_render_1.serviceName)(profile, "web")], runner, false);
+        }
+    }
+    catch (error) {
+        for (const entry of sideBySide)
+            removeContainers(profile, entry.started, runner);
+        throw error;
+    }
+    for (const entry of [...sideBySide].reverse()) {
+        if (!entry.replaced.length)
+            continue;
+        (0, deploy_compose_exec_1.checked)(runner, "docker", (0, deploy_compose_exec_1.dockerArgs)(profile, ["stop", ...entry.replaced]));
+        (0, deploy_compose_exec_1.checked)(runner, "docker", (0, deploy_compose_exec_1.dockerArgs)(profile, ["rm", ...entry.replaced]));
+    }
+    (0, deploy_compose_exec_1.runCompose)(profile, runtime, "compose", ["up", "-d", "--wait", "--remove-orphans"], runner, false);
+}
 async function applyComposeDeployment(projectRoot, profileName, release, confirm, runner = deploy_compose_exec_1.defaultRunner, fetcher = fetch) {
     const findings = doctorComposeDeployment(projectRoot, profileName, runner, release);
     const failed = findings.filter((finding) => !finding.ok);
@@ -455,14 +634,19 @@ async function applyComposeDeployment(projectRoot, profileName, release, confirm
         const dependencies = managedDependencyServices(profile);
         if (dependencies.length) {
             (0, deploy_compose_exec_1.runCompose)(profile, runtime, "compose", ["up", "-d", "--wait", ...dependencies], runner, false);
+            const init = objectStorageInit(profile);
+            if (init)
+                runObjectStorageInit(profile, runtime, init, runner);
         }
         else {
             // The application network still has to exist before the migration project,
-            // which joins it as an external network, can start.
-            (0, deploy_compose_exec_1.runCompose)(profile, runtime, "compose", ["up", "-d", "--no-start"], runner, false);
+            // which joins it as an external network, can start. `--no-recreate` keeps
+            // this from stopping the running containers of a changed service before the
+            // migration has even run.
+            (0, deploy_compose_exec_1.runCompose)(profile, runtime, "compose", ["up", "-d", "--no-start", "--no-recreate"], runner, false);
         }
         (0, deploy_compose_exec_1.runCompose)(profile, runtime, "migration", ["run", "--rm", "--no-deps", "migrate"], runner, false);
-        (0, deploy_compose_exec_1.runCompose)(profile, runtime, "compose", ["up", "-d", "--wait", "--remove-orphans"], runner, false);
+        rollOutApplication(profile, runtime, runner);
         const ledger = (0, deploy_compose_exec_1.readLedger)(profile, plan.images.api, runner);
         ledger.entries.push({
             revision: ((0, deploy_compose_exec_1.currentLedgerEntry)(ledger)?.revision ?? 0) + 1,
@@ -652,7 +836,7 @@ async function rollbackComposeDeployment(projectRoot, profileName, revision, con
     acquireComposeLock(profile, preflight.images.api, holder, runner);
     try {
         const runtime = (0, deploy_compose_render_1.renderComposeDeployment)(projectRoot, profileName, profile, preflight.release, preflight.images, preflight.rolloutStateDigest);
-        (0, deploy_compose_exec_1.runCompose)(profile, runtime, "compose", ["up", "-d", "--wait", "--remove-orphans"], runner, false);
+        rollOutApplication(profile, runtime, runner);
         const ledger = (0, deploy_compose_exec_1.readLedger)(profile, preflight.images.api, runner);
         ledger.entries.push({
             revision: ((0, deploy_compose_exec_1.currentLedgerEntry)(ledger)?.revision ?? 0) + 1,

package/dist/deploy.js

@@ -100,6 +100,20 @@ function secretKeys(profile, secretName, runner) {
         .filter(Boolean)
         .sort();
 }
+// ADMIN_EMAILS is an allowlist of addresses, not a credential, so the doctor reads
+// this one value of the API Secret to catch a placeholder. Every other value stays unread.
+function secretAdminEmails(profile, runner) {
+    const name = profile.secrets.api.name;
+    const result = runner("kubectl", kubectlArgs(profile, ["get", "secret", name, "-o", 'go-template={{with index .data "ADMIN_EMAILS"}}{{.}}{{end}}']), { capture: true });
+    if (result.status !== 0) {
+        return {
+            code: "admin-emails",
+            ok: false,
+            message: `Cannot read ADMIN_EMAILS from Secret ${name}: ${result.stderr.trim() || "the Secret is missing or inaccessible"}.`,
+        };
+    }
+    return (0, deploy_checks_1.adminEmailsFinding)(Buffer.from(result.stdout.trim(), "base64").toString("utf8"), `Secret ${name}`);
+}
 function dependencySecretRequirements(profile) {
     const requirements = [];
     if (profile.dependencies.postgres.mode === "inCluster") {
@@ -270,6 +284,8 @@ function doctorDeployment(projectRoot, profileName, runner = defaultRunner) {
                     : `Secret ${requirement.name} contains the required key names.`,
         });
     }
+    if ((0, deploy_checks_1.adminEmailsOnTarget)(projectRoot, profile))
+        findings.push(secretAdminEmails(profile, runner));
     const imagePullType = runner("kubectl", kubectlArgs(profile, [
         "get",
         "secret",

package/dist/templates/fullstack/apps/api/src/database/database.spec.ts

@@ -1,5 +1,7 @@
 import { SQL } from "bun";
 import { describe, expect, it } from "bun:test";
+import { createSocket } from "node:dgram";
+import { Resolver } from "node:dns/promises";
 import { existsSync, mkdtempSync, rmSync } from "node:fs";
 import { connect, createServer, type AddressInfo } from "node:net";
 import { tmpdir } from "node:os";
@@ -62,6 +64,45 @@ function probedSql(): ProbedSql {
   };
 }
 
+interface LocalDns {
+  resolver: Resolver;
+  close: () => void;
+}
+
+// A resolver that asks a DNS server on a local UDP port instead of the system's,
+// whose answers depend on the machine. The server answers every query with
+// NXDOMAIN, or never answers when `answers` is false.
+async function localDns(answers: boolean): Promise<LocalDns> {
+  const server = createSocket("udp4");
+  if (answers) {
+    server.on("message", (query, peer) => {
+      // The header and the question, flagged as a response with RCODE 3 and no records.
+      let end = 12;
+      while (end < query.length && query[end] !== 0) end += (query[end] ?? 0) + 1;
+      const response = Buffer.from(query.subarray(0, end + 5));
+      response[2] = 0x80 | ((query[2] ?? 0) & 0x01);
+      response[3] = 0x83;
+      response.writeUInt16BE(1, 4);
+      response.fill(0, 6, 12);
+      server.send(response, peer.port, peer.address);
+    });
+  }
+  await new Promise<void>((resolve) => server.bind(0, "127.0.0.1", resolve));
+  // Bun waits at least a second for an answer; a single try keeps that the whole wait.
+  const resolver = new Resolver({ timeout: 200, tries: 1 });
+  resolver.setServers([`127.0.0.1:${server.address().port}`]);
+  return { resolver, close: () => server.close() };
+}
+
+async function lookupError(answers: boolean): Promise<unknown> {
+  const dns = await localDns(answers);
+  try {
+    return await dns.resolver.resolve4("database.invalid").then(() => undefined, (reason: unknown) => reason);
+  } finally {
+    dns.close();
+  }
+}
+
 describe("database provider", () => {
   it("uses DATABASE_URL as the canonical connection setting", () => {
     const provider = databaseProvider();
@@ -266,11 +307,16 @@ describe("isDatabaseUnavailable", () => {
   });
 
   it("recognizes a real lookup of a host that does not exist", async () => {
-    // .invalid never resolves (RFC 2606). A resolver that does not answer at all
-    // fails with ETIMEOUT after about 16 seconds, which is recognized too.
-    const error: unknown = await fetch("http://database.invalid/").then(() => undefined, (reason: unknown) => reason);
+    const error = await lookupError(true);
+    expect((error as { code?: unknown }).code).toBe("ENOTFOUND");
     expect(isDatabaseUnavailable(error)).toBe(true);
-  }, 30_000);
+  });
+
+  it("recognizes a real lookup that the resolver never answers", async () => {
+    const error = await lookupError(false);
+    expect((error as { code?: unknown }).code).toBe("ETIMEOUT");
+    expect(isDatabaseUnavailable(error)).toBe(true);
+  });
 
   it("recognizes a real refused connection to localhost", async () => {
     const server = createServer();