@specific.dev/spectest 0.69.0 → 0.71.0

package/dist/daemon.js

@@ -40,13 +40,15 @@ import { LOG_DELTA_MAX_BYTES, capMiddle, streamDelta } from "./harness/log-delta
 import { resolveHostPath as resolveVolumeHostPath, sanitizeSegment, } from "./harness/volume-paths.js";
 import { pollUntilReady } from "./harness/ready-poll.js";
 import { runWrapperRules } from "./harness/wrapper-rules.js";
+import { cpus } from "node:os";
 import { APP_DIR, WORKSPACE, resolveProjectPath } from "./project-files.js";
 import { isTextualContentType, looksBinary, omittedBody, parseContentLength, } from "./harness/http-body.js";
 import { encodeRegistry } from "./harness/names-registry.js";
 import { InterceptRegistry, parseTarget, runChain, } from "./harness/intercept.js";
 import { HOP_BY_HOP_HEADERS, augmentCorsResponse, corsPreflightResponse, isCorsPreflight, } from "./harness/http-proxy.js";
-import { certCovers as hostmatchCertCovers, hostWithoutPort, matchRoute, wildcardSuffix, } from "./harness/hostmatch.js";
-import { INGRESS_HTTPS_PORT, INGRESS_HTTP_PORT, bindRoute, certEntries, clearTables, emptyTables, planBind, registryTarget, routesFor, unbindRoute, } from "./harness/ingress-table.js";
+import { certCovers as hostmatchCertCovers, hostWithoutPort, matchRoute, selectCertName, wildcardSuffix, } from "./harness/hostmatch.js";
+import { INGRESS_HTTPS_PORT, INGRESS_HTTP_PORT, bindRoute, clearTables, emptyTables, planBind, registryTarget, routesFor, unbindRoute, } from "./harness/ingress-table.js";
+import { startTlsTerminator } from "./harness/tls-terminator.js";
 import { runContainerArgs } from "./harness/container-run.js";
 import { assertAbsolute, certificateHostnames, defaultKeyMode, expandServiceToken, isNoopChown, mountFlag, needsIdTables, numericId, resolveChownIds, } from "./harness/file-mounts.js";
 import { conflict, notFound, requireString, } from "./harness/methods.js";
@@ -66,29 +68,51 @@ function namedServices(cfg) {
 const DEFAULT_TEST_TIMEOUT_MS = 60_000;
 const NETWORK_NAME = process.env.SPECTEST_NETWORK ?? "spectest-net";
 // Stable hostname every service container resolves to the host (the
-// `spectest-br0` gateway) — so apps that build or pull images at runtime
-// can point a builder at `spectest-host:5000` (the zot Docker Hub mirror)
-// or `spectest-host:1234` (the shared buildkitd) without hard-coding the
-// gateway IP. Injected into each container's /etc/hosts in runContainer.
+// `spectest-br0` gateway) — so an app that runs its OWN BuildKit inside a
+// test can point its cache export at the host's build-cache registry,
+// `spectest-host:5007`, without hard-coding the gateway IP. Nothing else
+// lives behind it any more: images are pulled and built inside the VM
+// against the container store (CONTAINER_STORE.md). Injected into each
+// container's /etc/hosts in runContainer.
 const SPECTEST_HOST_NAME = "spectest-host";
-// The host image-cache gateway, discovered once from the same
-// `registry-mirrors` entry the in-VM dockerd already uses (baked into the
-// local provider's golden /etc/docker/daemon.json). `null` when there's
-// no host cache, so nothing is injected.
+// The host gateway, read once from the guest's own default route
+// (/proc/net/route: destination 0, gateway as a little-endian hex word).
+// `null` when the guest has no default route, in which case nothing is
+// injected.
 let _hostCacheGateway;
 function hostCacheGateway() {
     if (_hostCacheGateway !== undefined)
         return _hostCacheGateway;
+    _hostCacheGateway = null;
     try {
-        const cfg = JSON.parse(readFileSync("/etc/docker/daemon.json", "utf8"));
-        const first = cfg["registry-mirrors"]?.[0];
-        _hostCacheGateway = first ? new URL(first).hostname || null : null;
+        for (const line of readFileSync("/proc/net/route", "utf8").split("\n").slice(1)) {
+            const f = line.trim().split(/\s+/);
+            if (f.length < 3 || f[1] !== "00000000")
+                continue;
+            const hex = f[2];
+            const octets = [6, 4, 2, 0].map((i) => parseInt(hex.slice(i, i + 2), 16));
+            if (octets.every((o) => Number.isFinite(o))) {
+                _hostCacheGateway = octets.join(".");
+                break;
+            }
+        }
     }
     catch {
         _hostCacheGateway = null;
     }
     return _hostCacheGateway;
 }
+/** vCPUs this guest has, for BuildKit's `max-parallelism` — the number
+ *  Blacksmith sets too, and the one that matters now that the build
+ *  competes for the guest's own cores rather than the host's. */
+function cpuCount() {
+    try {
+        return Math.max(1, cpus().length);
+    }
+    catch {
+        return 4;
+    }
+}
 // WORKSPACE (/workspace) and APP_DIR (/opt/spectest/app) both live in
 // project-files.ts, next to the rule that decides which copy of a project
 // file is the current one.
@@ -418,42 +442,108 @@ async function hasBuildx() {
     }
     return _buildxAvailable;
 }
-// A single buildkitd runs on the host (see scripts/install-buildkitd.sh),
-// reachable from every VM at the bridge gateway. Building against it as a
-// `remote` buildx builder gives a persistent, shared layer/mount cache that
-// survives forks and warm-template misses — a fresh VM no longer rebuilds
-// from scratch. The build runs on the host (runc-isolated); `--load` pulls
-// the finished image back into the in-VM dockerd. Detected once; if the
-// builder can't be created or buildkitd is unreachable we fall back to the
-// in-VM builder, so a missing/dead buildkitd just means slower builds.
-const REMOTE_BUILDER_ADDR = process.env.SPECTEST_BUILDKIT_ADDR ?? "tcp://10.42.0.1:1234";
-const REMOTE_BUILDER_NAME = "spectest-remote";
-/** Parent of the per-build buildx config dirs (see isolatedBuildxConfig).
- *  On tmpfs: each holds a builder stub and an 8-byte node id. */
-let _remoteBuilder;
-async function ensureRemoteBuilder() {
-    if (_remoteBuilder !== undefined)
-        return _remoteBuilder;
-    if (!(await hasBuildx())) {
-        _remoteBuilder = false;
+// Every VM carries its project's container store (CONTAINER_STORE.md):
+// the disk every pull and every build lands on, mounted by the control
+// plane before this harness starts. The manifest below says where; the
+// in-VM buildkitd keeps its exported cache there too, so a fresh VM finds
+// every layer it built before. Detected once; if the daemon will not
+// start, dockerd's own BuildKit builds instead.
+const IMAGE_CACHE_MANIFEST = "/run/spectest-image-cache.json";
+const LOCAL_BUILDER_NAME = "spectest-local";
+const LOCAL_BUILDKIT_ADDR = "tcp://127.0.0.1:1234";
+/** The bring-up script the cache base bakes (`base.rs::BUILDKITD_UP_SH`). */
+const BUILDKITD_UP_PATH = "/usr/local/bin/spectest-buildkitd-up";
+/** Where the control plane mounted this VM's image cache: containerd's
+ *  root (read-write, this VM's own) and the layers disk (read-only, shared
+ *  by every VM of a generation). `null` when the VM carries no cache. */
+async function imageCachePaths() {
+    try {
+        const raw = await fs.readFile(IMAGE_CACHE_MANIFEST, "utf8");
+        const parsed = JSON.parse(raw);
+        const root = (parsed.disks ?? []).find((d) => d.role === "root" && d.path)?.path;
+        const layers = (parsed.disks ?? []).find((d) => d.role === "layers" && d.path)?.path;
+        return root && layers ? { root, layers } : null;
+    }
+    catch {
+        return null;
+    }
+}
+let _localBuilder;
+/**
+ * Start buildkitd inside this VM with its state on the cache disk, and
+ * register it as a buildx `remote` builder.
+ *
+ * Lazy on purpose: it runs on the first build a project actually does,
+ * so a project with no dockerfile service never pays for a builder. And
+ * best-effort: a daemon that will not start falls back to the shared host
+ * one, which is a slower build and not a failed one.
+ */
+async function ensureLocalBuildkitd() {
+    if (_localBuilder !== undefined)
+        return _localBuilder;
+    _localBuilder = false;
+    const paths = await imageCachePaths();
+    if (!paths)
+        return false;
+    const state = paths.root;
+    if (!(await hasBuildx()))
+        return false;
+    // The bring-up itself is a script baked into the image cache's base
+    // snapshot (`base.rs::BUILDKITD_UP_SH`), so production and the real-VM
+    // test drive exactly the same daemon with exactly the same config. All
+    // it takes from us is where the state lives — the registry routing is
+    // its own (Docker Hub via `mirror.gcr.io`, deliberately not the host
+    // zot instance).
+    const started = await shx("/bin/bash", [BUILDKITD_UP_PATH, state], 900_000);
+    if (started.code !== 0) {
+        // eslint-disable-next-line no-console
+        console.warn(`[build] in-VM buildkitd would not start; building with dockerd's own BuildKit:\n${(started.stderr || started.stdout).trim()}`);
         return false;
     }
-    // Idempotent: a repeat create with the same name errors ("existing
-    // instance"), which we treat as already-present.
-    const create = await docker(["buildx", "create", "--name", REMOTE_BUILDER_NAME, "--driver", "remote", REMOTE_BUILDER_ADDR], 30_000);
+    const create = await docker(["buildx", "create", "--name", LOCAL_BUILDER_NAME, "--driver", "remote", LOCAL_BUILDKIT_ADDR], 30_000);
     if (create.code !== 0 && !/existing instance|already exists/i.test(create.stderr)) {
-        _remoteBuilder = false;
+        // eslint-disable-next-line no-console
+        console.warn(`[build] could not register the in-VM builder:\n${create.stderr.trim()}`);
         return false;
     }
-    // `inspect --bootstrap` actually dials buildkitd, so it's our reachability
-    // probe. If buildkitd is down this fails and we fall back.
-    const boot = await docker(["buildx", "inspect", "--bootstrap", REMOTE_BUILDER_NAME], 60_000);
-    _remoteBuilder = boot.code === 0;
-    if (!_remoteBuilder) {
+    // `inspect --bootstrap` dials the daemon, so it is both the readiness
+    // wait and the proof it is really answering.
+    const boot = await docker(["buildx", "inspect", "--bootstrap", LOCAL_BUILDER_NAME], 120_000);
+    if (boot.code !== 0) {
         // eslint-disable-next-line no-console
-        console.warn(`[build] remote buildkitd at ${REMOTE_BUILDER_ADDR} unreachable; using in-VM builder:\n${boot.stderr.trim()}`);
+        console.warn(`[build] in-VM buildkitd never answered; building with dockerd's own BuildKit:\n${boot.stderr.trim()}`);
+        return false;
     }
-    return _remoteBuilder;
+    // eslint-disable-next-line no-console
+    console.log(`[build] building in this VM against the image cache (root ${paths.root}, layers ${paths.layers})`);
+    // Imports come from the merged cache on the read-only layers disk and
+    // from this lineage's own exports on the root; exports go to the root,
+    // where the merge picks them up (image_cache/merge.rs).
+    _localCacheDir = `${state}/spectest-buildkit-cache`;
+    _localCacheImports = [`${paths.layers}/spectest-buildkit-cache`, `${state}/spectest-buildkit-cache`];
+    _localBuilder = true;
+    return true;
+}
+/** The exported-cache directory on the cache disk, once the in-VM builder is up. */
+let _localCacheDir = null;
+/** The cache directories a build imports from: the merged one on the
+ *  layers disk, then this lineage's own exports. */
+let _localCacheImports = [];
+/**
+ * A reference as containerd names it. buildx's `-t` on a remote builder
+ * stores an unqualified name (`probe:bx`) that dockerd then cannot
+ * resolve (measured, CONTAINER_STORE.md), so the in-VM build names its
+ * output in full.
+ */
+function qualifyImageRef(ref) {
+    const slash = ref.indexOf("/");
+    const first = slash < 0 ? "" : ref.slice(0, slash);
+    const isRegistry = first.includes(".") || first.includes(":") || first === "localhost";
+    if (slash < 0)
+        return `docker.io/library/${ref}`;
+    if (!isRegistry)
+        return `docker.io/${ref}`;
+    return ref;
 }
 async function ensureNetwork() {
     const inspect = await docker(["network", "inspect", NETWORK_NAME], 30_000);
@@ -508,7 +598,12 @@ async function ensureVolumes(svc) {
         // created here nor recorded for the delta-restore wipe.
         if (!(await isExistingNonDirectory(host))) {
             await fs.mkdir(host, { recursive: true });
-            if (vol.source?.startsWith("/") && !host.startsWith("/var/cache/spectest/")) {
+            // …and neither is the pre-disks cache tree, kept for a server still
+            // serving older SDKs. Leaving it out of the manifest is what
+            // protects a project running this SDK against a server whose
+            // teardown guard predates it.
+            const durable = host.startsWith("/var/cache/spectest/");
+            if (vol.source?.startsWith("/") && !durable) {
                 await recordAbsoluteVolumeDir(host);
             }
         }
@@ -1002,24 +1097,47 @@ async function runServiceBuild(name, image, tag, caSuffix) {
         // predates per-Dockerfile ignores — and is written ONLY when the
         // project ships none of its own (see readProjectDockerignore).
         await fs.writeFile(`${dfPath}.dockerignore`, serviceDockerignore(image.exclude));
-        const useRemote = await ensureRemoteBuilder();
-        // Both the remote builder and a local buildx are BuildKit, so both emit
-        // per-step timing on stderr under `--progress=plain` (parsed below). Only
-        // the legacy in-VM builder takes no progress flag.
-        const useBuildKit = useRemote || (await hasBuildx());
+        // The in-VM buildkitd on the container store first: the build runs
+        // inside the guest's own isolation boundary, against this project's
+        // own layer cache, and the finished image is already in the store
+        // dockerd reads. dockerd's built-in BuildKit is the fallback (a guest
+        // with no store, or a daemon that would not start).
+        const useLocal = await ensureLocalBuildkitd();
+        // Both the in-VM daemon and dockerd's buildx are BuildKit, so both
+        // emit per-step timing on stderr under `--progress=plain` (parsed
+        // below). Only the legacy builder takes no progress flag.
+        const useBuildKit = useLocal || (await hasBuildx());
         const buildEnv = {};
         let buildArgs;
         // The user's `buildArgs`, as `--build-arg` flags; a plain client flag,
         // so every builder — host buildkitd, in-VM BuildKit, legacy — takes it.
         const argFlags = buildArgFlags(image.buildArgs);
-        if (useRemote) {
-            // Build on the host-side shared buildkitd (persistent cross-VM cache);
-            // `--load` brings the finished image back into the in-VM dockerd so
-            // runContainer can `docker run` it. The build context (WORKSPACE, minus
-            // .dockerignore) streams to buildkitd over the bridge.
+        if (useLocal && _localCacheDir) {
+            // The in-VM builder is BuildKit's containerd worker on this VM's
+            // own image store (CONTAINER_STORE.md): the output is an image
+            // record in dockerd's namespace, unpacked, so there is no `--load`
+            // and nothing crosses a socket. The cache directory on the image cache
+            // disk is what outlives the VM; `mode=max` keeps every
+            // intermediate layer, uncompressed so an import never inflates,
+            // and one tag per service so exports do not replace each other.
+            const cacheTag = name.replace(/[^a-z0-9-]/gi, "-").toLowerCase();
             buildArgs = [
                 "buildx", "build",
-                "--builder", REMOTE_BUILDER_NAME,
+                "--builder", LOCAL_BUILDER_NAME,
+                "--progress=plain",
+                "--output", `type=image,name=${qualifyImageRef(tag)},unpack=true`,
+                ..._localCacheImports.flatMap((src) => ["--cache-from", `type=local,src=${src}`]),
+                "--cache-to", `type=local,dest=${_localCacheDir},mode=max,compression=uncompressed,force-compression=true,tag=${cacheTag}`,
+                ...argFlags,
+                "-f", dfPath, WORKSPACE,
+            ];
+        }
+        else if (useLocal) {
+            // The daemon came up but reported no cache directory: build on it
+            // and `--load` the result into dockerd.
+            buildArgs = [
+                "buildx", "build",
+                "--builder", LOCAL_BUILDER_NAME,
                 "--load",
                 "--progress=plain",
                 ...argFlags,
@@ -1286,6 +1404,43 @@ async function waitForReady(svc) {
     msg += output ? `\nRecent container logs:\n${output}` : `\n(the container logged nothing)`;
     throw new Error(msg);
 }
+/**
+ * Add the container's own account of its death to an error raised by its
+ * `setup` hook.
+ *
+ * `waitForReady` already does this, because a container that never becomes
+ * ready is obviously the container's fault. A `setup` hook is the case that
+ * was missing, and it is the one that reads most misleadingly: the hook
+ * talks to the service over the network, so when the container dies
+ * mid-hook what surfaces is a name that no longer resolves or a rollout
+ * that never finished — a symptom from the far end of a connection to
+ * something that is not there any more. The reason is in a log that goes
+ * with the VM at teardown.
+ *
+ * Only for a container that is **gone**: one that is still up did not cause
+ * this, and its log would bury the real error. Best-effort throughout — a
+ * diagnostic must never replace the failure it explains.
+ */
+async function withContainerPostMortem(name, err) {
+    try {
+        const state = await docker(["inspect", "-f", "{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}}", name], 15_000);
+        const [running, code, oom] = state.stdout.trim().split(/\s+/);
+        if (state.code !== 0 || running !== "false")
+            return err;
+        const logs = await docker(["logs", "--tail=120", name], 30_000);
+        const output = `${logs.stdout}\n${logs.stderr}`.trim();
+        const base = err instanceof Error ? err : new Error(String(err));
+        base.message +=
+            `\n\nThe "${name}" container exited (code ${code}` +
+                `${oom === "true" ? ", OOM-killed" : ""}) while its setup hook was running, ` +
+                `which is why the hook could not reach it.` +
+                (output ? `\nIts last output:\n${output}` : `\nIt logged nothing.`);
+        return base;
+    }
+    catch {
+        return err;
+    }
+}
 /** Validate the `dependsOn` graph and return the name→service map used to
  *  walk it. Rules live in `harness/service-graph.ts`. */
 function validateServiceGraph(services) {
@@ -1356,43 +1511,29 @@ const INGRESS_HTTP_SERVERS = new Map();
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const INGRESS_HTTPS_SERVERS = new Map();
 /**
- * Servers replaced by a rebind and now draining. On Bun 1.3.14 a request
- * arriving on a kept-alive connection of a `stop(false)`-drained server
- * dispatches into freed per-server state and can SEGFAULT the process
- * (use-after-free class fixed upstream by oven-sh/bun#36790, first in Bun
- * 1.4.0; observed here as `panic: Segmentation fault at address 0xA` in
- * `server.zig onRequestFor` on ~2-3 % of runtime-TLS rebinds). Until the
- * Bun bump lands, shrink the number of requests a drained server can ever
- * see: every response it still serves carries `Connection: close` (one
- * more request per surviving connection, not unlimited), and a grace timer
- * force-closes whatever is left ({@link REBIND_DRAIN_GRACE_MS}).
+ * The :443 TLS terminator, once per harness process.
+ *
+ * There is deliberately no draining machinery beside it any more. :443
+ * used to be a `Bun.serve` that had to be swapped for every new
+ * certificate, and no swap can be made safe: `stop(false)` FINs an idle
+ * kept-alive connection within 1-2 ms, so a client that wrote a request in
+ * that instant failed with `SocketError: other side closed` — in whatever
+ * unrelated test happened to be talking through the ingress.
+ * {@link startTlsTerminator} chooses the leaf per handshake instead, so
+ * the listener is bound once and lives as long as the project does.
  */
-const DRAINING_INGRESS = new WeakSet();
-/** How long a drained listener may keep serving in-flight work before its
- *  remaining connections are force-closed. Long enough for a slow proxied
- *  response to finish, short enough to bound the 1.3.14 UAF window. */
-const REBIND_DRAIN_GRACE_MS = 15_000;
-/** Stamp `Connection: close` on a response served by a draining listener so
- *  the kept-alive connection retires instead of lingering as a UAF trigger.
- *  Proxied responses can carry immutable headers; rewrap when needed. */
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-function withDrainClose(server, res) {
-    if (!DRAINING_INGRESS.has(server))
-        return res;
-    try {
-        res.headers.set("connection", "close");
-        return res;
-    }
-    catch {
-        const headers = new Headers(res.headers);
-        headers.set("connection", "close");
-        return new Response(res.body, {
-            status: res.status,
-            statusText: res.statusText,
-            headers,
-        });
-    }
-}
+let HTTPS_TERMINATOR;
+/**
+ * The in-flight first bind of :443, if one is running.
+ *
+ * {@link ensureHttpsIngress} is async where the swap it replaced was
+ * synchronous, and that reintroduces an interleaving the old code could
+ * not have: two `ctx.startService({ tls })` calls issued together by a
+ * fake handler would both find :443 absent and both try to bind it, and
+ * the loser gets EADDRINUSE — a provisioning failure in place of the
+ * dropped connection this change exists to remove. One promise, shared.
+ */
+let HTTPS_INGRESS_STARTING;
 /**
  * The live ingress tables — per-port routes and the :443 SNI cert table.
  *
@@ -1441,6 +1582,12 @@ function stopIngressServers() {
         }
     }
     INGRESS_HTTPS_SERVERS.clear();
+    // Force-closed for the same reason the listeners are: the containers
+    // behind these routes are going away, so a connection still riding them
+    // has nothing left to reach.
+    HTTPS_TERMINATOR?.close();
+    HTTPS_TERMINATOR = undefined;
+    HTTPS_INGRESS_STARTING = undefined;
     clearTables(INGRESS);
 }
 function buildIngress(project) {
@@ -1655,7 +1802,7 @@ async function startIngress() {
     }
     // ── HTTPS listener on INGRESS_HTTPS_PORT: SNI per certificated hostname.
     if (INGRESS.certByHost.size > 0)
-        rebindHttpsListener(Bun);
+        await ensureHttpsIngress(Bun);
     // Seed the resolver's names registry: ingress hostnames (fakes, TLS
     // proxies, dnsName(→ingress)) → bridge gateway, plus ingress-targeted
     // wildcards. Service-targeted wildcards wait for the post-container pass
@@ -1671,81 +1818,85 @@ function requireBun() {
     }
     return Bun;
 }
-/** Flatten the live SNI cert table into Bun's TLS-entry array. */
-function tlsEntriesFromCerts() {
-    return certEntries(INGRESS);
-}
 /**
- * (Re)bind the :443 listener from the current cert table + route map.
+ * Bring :443 up, once.
  *
- * Bun fixes a server's TLS config at `Bun.serve` time — `reload()` accepts a
- * new `tls` option and silently keeps serving the old certificates (measured
- * on Bun 1.3.14: after reloading with a second SNI entry, the new hostname
- * still gets the first one's leaf). So adding a certificate really does mean
- * a second listener.
+ * Two pieces: a plaintext `Bun.serve` on an ephemeral loopback port that
+ * carries the HTTPS route table (and therefore all the routing, fake
+ * dispatch, interception, reverse-proxying and `server.upgrade()`
+ * WebSocket bridging that :80 already gets, unchanged), and the TLS
+ * terminator in front of it on :443.
  *
- * **Bind the new one before stopping the old one.** Doing it the other way
- * round — which is what this used to do, with `stop(true)` — has two teeth:
- * the force-close kills every established connection on :443, and the gap
- * before the new listener binds refuses new ones. Neither is limited to the
- * hostname being added; they hit all the unrelated traffic the listener is
- * carrying. On a project that mints a certificate per provisioned database
- * while deploys stream through the same port, that surfaced as the app under
- * test dying with `SocketError: other side closed` or `ECONNREFUSED
- * <gateway>:443` — a different test each run, and nothing pointing at
- * ingress.
+ * **This is idempotent, and that is the feature.** Adding a certificate
+ * mutates {@link IngressTables.certByHost} and nothing else: the
+ * terminator reads that table on every handshake, so the new hostname is
+ * served by the next connection and no established connection is
+ * disturbed. The listener this replaced had to be stopped and re-served
+ * for each new certificate, and every such swap severed the idle
+ * kept-alive connections it was carrying — see
+ * `harness/tls-terminator.ts` for the measurements and the history.
  *
- * Overlapping the two needs SO_REUSEPORT on both sockets ({@link
- * bindIngressServer} sets it unconditionally for that reason), after which
- * the old listener is drained with `stop(false)` so requests in flight
- * finish. During the overlap the kernel may hand a new connection to either
- * socket, which is safe: the only hostname the two disagree about is the one
- * being added, and it does not resolve until the caller writes the names
- * registry after this returns.
+ * The route `Map` is the persistent module object, so the plaintext
+ * server closes over the same table and later route additions need no
+ * restart either.
  *
- * The route Map is the persistent module object, so the new listener closes
- * over the same table and later route additions need no rebind at all.
+ * The loopback port is ephemeral rather than fixed so it can never
+ * collide with a fake's declared port; it is bound on 127.0.0.1, which no
+ * container can reach (containers arrive at the bridge gateway), so the
+ * only way in is through the terminator.
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-function rebindHttpsListener(Bun) {
-    // The *same* table object the previous listener closed over: the new
-    // listener must serve the routes bound since, and any bound later.
+async function ensureHttpsIngress(Bun) {
+    if (INGRESS_HTTPS_SERVERS.has(INGRESS_HTTPS_PORT))
+        return;
+    if (!HTTPS_INGRESS_STARTING) {
+        // Cleared on both outcomes: once bound, the check above short-circuits
+        // every later caller, and a failed bind must be retryable rather than
+        // remembered as a rejection for the life of the project.
+        HTTPS_INGRESS_STARTING = startHttpsIngress(Bun).finally(() => {
+            HTTPS_INGRESS_STARTING = undefined;
+        });
+    }
+    return HTTPS_INGRESS_STARTING;
+}
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+async function startHttpsIngress(Bun) {
     const routes = routesFor(INGRESS, INGRESS_HTTPS_PORT);
-    const old = INGRESS_HTTPS_SERVERS.get(INGRESS_HTTPS_PORT);
-    const server = bindIngressServer(Bun, INGRESS_HTTPS_PORT, routes, `https :${INGRESS_HTTPS_PORT}`, tlsEntriesFromCerts());
-    if (old) {
+    // `proto` is passed rather than inferred: this listener speaks plain
+    // HTTP, but everything reaching it arrived over TLS, so X-Forwarded-Proto
+    // must say https.
+    const plain = bindIngressServer(Bun, 0, routes, `https :${INGRESS_HTTPS_PORT}`, {
+        proto: "https",
+        hostname: "127.0.0.1",
+    });
+    INGRESS_HTTPS_SERVERS.set(INGRESS_HTTPS_PORT, plain);
+    try {
+        HTTPS_TERMINATOR = await startTlsTerminator({
+            port: INGRESS_HTTPS_PORT,
+            hostname: "0.0.0.0",
+            upstreamPort: plain.port,
+            certFor: (serverName) => {
+                if (serverName === undefined)
+                    return INGRESS.certByHost.values().next().value;
+                const name = selectCertName(INGRESS.certByHost.keys(), serverName.toLowerCase());
+                return name === undefined ? undefined : INGRESS.certByHost.get(name);
+            },
+        });
+    }
+    catch (err) {
+        // Leave nothing half-built: a registered plaintext server with no
+        // terminator would make `planBind` believe :443 is up.
+        INGRESS_HTTPS_SERVERS.delete(INGRESS_HTTPS_PORT);
         try {
-            // Graceful: stop accepting, let in-flight requests finish. The old
-            // server stays alive until they do, which is the point — a long
-            // upload through ingress must not be collateral damage of another
-            // service being provisioned.
-            old.stop(false);
-            // Bun 1.3.14 landmine: a request arriving later on one of the old
-            // server's kept-alive connections dispatches into freed state and can
-            // segfault the daemon (see {@link DRAINING_INGRESS}). Mark it so any
-            // response it still serves closes its connection, and force-close the
-            // stragglers once in-flight work has had a fair window to finish.
-            DRAINING_INGRESS.add(old);
-            const graceTimer = setTimeout(() => {
-                try {
-                    old.stop(true);
-                }
-                catch {
-                    /* already fully stopped */
-                }
-            }, REBIND_DRAIN_GRACE_MS);
-            // Don't let the grace timer keep the process alive on shutdown.
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            graceTimer.unref?.();
+            plain.stop(true);
         }
-        catch (err) {
-            // eslint-disable-next-line no-console
-            console.warn("[ingress] failed to drain the previous https listener:", err);
+        catch {
+            /* already gone */
         }
+        throw err;
     }
-    INGRESS_HTTPS_SERVERS.set(INGRESS_HTTPS_PORT, server);
     // eslint-disable-next-line no-console
-    console.log(`[ingress] https :${INGRESS_HTTPS_PORT} for ${[...routes.keys()].join(", ")}`);
+    console.log(`[ingress] https :${INGRESS_HTTPS_PORT} (tls terminator -> :${plain.port}) for ${[...routes.keys()].join(", ")}`);
 }
 /**
  * RFC 6125 wildcard match: `*.example.com` covers `api.example.com` but NOT
@@ -1800,10 +1951,13 @@ async function bindRuntimeTls(hostname, service, port) {
         INGRESS_HTTP_SERVERS.set(INGRESS_HTTP_PORT, bindIngressServer(Bun, INGRESS_HTTP_PORT, routesFor(INGRESS, INGRESS_HTTP_PORT), `port ${INGRESS_HTTP_PORT}`));
     }
     if (plan.needsCert) {
+        // The terminator reads this table on every handshake, so the leaf is
+        // live the moment it lands — no listener is touched, and no connection
+        // already on :443 notices.
         INGRESS.certByHost.set(host, await generateHostCert(host, [host]));
     }
-    if (plan.needsHttpsRebind)
-        rebindHttpsListener(Bun);
+    if (plan.needsHttpsListener)
+        await ensureHttpsIngress(Bun);
     // Resolve the hostname to the daemon gateway (where :443/:80 listen).
     // A wildcard can only live in the resolver's suffix table.
     const gw = await bridgeGatewayIp();
@@ -1851,20 +2005,26 @@ async function unbindRuntimeTls(hostname) {
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function bindIngressServer(
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-Bun, port, byHost, listenerLabel, tlsEntries) {
-    // A TLS listener terminates https; everything else is plain http. Used
-    // to stamp X-Forwarded-Proto so upstreams that build absolute URLs or
-    // redirect see the scheme the client actually used, not our http hop.
-    const proto = tlsEntries ? "https" : "http";
+Bun, port, byHost, listenerLabel, 
+// Every ingress listener speaks plain HTTP now — the one behind :443
+// sits under the TLS terminator. So the scheme a client actually used
+// can no longer be inferred from the socket and is declared instead; it
+// stamps X-Forwarded-Proto, which upstreams that build absolute URLs or
+// redirect depend on.
+serve = { proto: "http" }) {
+    const proto = serve.proto;
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const opts = {
         port,
-        hostname: "0.0.0.0",
-        // SO_REUSEPORT on every ingress listener, so a replacement can be bound
-        // while the old one is still serving. That overlap is the only way to
-        // add an SNI certificate without a gap — see {@link rebindHttpsListener}
-        // — and it only works if *both* sockets opt in: a second plain bind
-        // fails with "Is port 443 in use?".
+        hostname: serve.hostname ?? "0.0.0.0",
+        // SO_REUSEPORT on every ingress listener. It was introduced so a
+        // replacement :443 could overlap the original during a certificate
+        // swap; the terminator removed that swap, and this stays for the
+        // remaining case — the `/load` teardown force-closes and rebinds, and
+        // a socket the kernel has not finished releasing would otherwise fail
+        // the next bind with "Is port 443 in use?". Note the flip side, which
+        // {@link stopIngressServers} depends on: a half-alive listener shares
+        // the port silently instead of colliding loudly.
         reusePort: true,
         // Bun.serve defaults to a 10s idleTimeout, which kills any proxied
         // request whose upstream takes >10s to produce bytes — under parallel
@@ -1876,7 +2036,7 @@ Bun, port, byHost, listenerLabel, tlsEntries) {
         // short-lived, leaked-connection risk is bounded by the fork.
         idleTimeout: 0,
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        fetch: (req, server) => dispatchIngress(req, server, byHost, listenerLabel, proto).then((res) => withDrainClose(server, res)),
+        fetch: (req, server) => dispatchIngress(req, server, byHost, listenerLabel, proto),
         websocket: {
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             async open(ws) {
@@ -1953,8 +2113,6 @@ Bun, port, byHost, listenerLabel, tlsEntries) {
             },
         },
     };
-    if (tlsEntries)
-        opts.tls = tlsEntries;
     return Bun.serve(opts);
 }
 /**
@@ -2018,6 +2176,28 @@ server, byHost, listenerLabel, proto) {
         : await runChain(chain, req, upstream, recordInterceptedRequest);
     return augmentCorsResponse(req, res);
 }
+/**
+ * The address to report as the client's in `X-Forwarded-For`.
+ *
+ * On :80 that is simply the socket's peer. On :443 the peer is the TLS
+ * terminator on loopback, so the real address has to be recovered from it:
+ * the terminator keys `source port -> client address` for the life of each
+ * upstream connection, and the source port is what `requestIP` reports
+ * here. Gated on `proto === "https"` so a genuinely loopback caller on :80
+ * can never pick up an unrelated terminator connection's port.
+ *
+ * Falling back to the socket's own address means the worst case is the
+ * pre-terminator answer for a plain HTTP hop, never a wrong tenant.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function ingressClientIp(server, req, proto) {
+    const peer = server.requestIP?.(req);
+    if (!peer?.address)
+        return undefined;
+    if (proto !== "https")
+        return peer.address;
+    return HTTPS_TERMINATOR?.clientIpFor(peer.port) ?? peer.address;
+}
 /**
  * Reverse-proxy a request to `http://<service>:<port>` on
  * `spectest-net`. Handles plain HTTP/1.1 + 2 and WebSocket upgrades:
@@ -2067,7 +2247,7 @@ server, service, port, listenerLabel, proto) {
     // Standard reverse-proxy provenance headers: the upstream sees the
     // public scheme/host it was reached through and the client's address,
     // even though we rewrite Host below to the service-net name.
-    const clientIp = server.requestIP?.(req)?.address;
+    const clientIp = ingressClientIp(server, req, proto);
     const priorXff = req.headers.get("x-forwarded-for");
     const xff = clientIp ? (priorXff ? `${priorXff}, ${clientIp}` : clientIp) : priorXff;
     if (xff)
@@ -2860,18 +3040,14 @@ async function bootstrapInner() {
     // at "image ready" waiting for an unrelated slow build elsewhere.
     //
     // Prep concurrency: registry pulls always run in parallel (network-bound,
-    // low VM RAM). Dockerfile builds parallelize *only* when the host
-    // buildkitd is in play — there the build executes host-side under runc, so
-    // N concurrent builds don't touch the VM's memory ceiling. When we fall
-    // back to the in-VM builder, two or more concurrent builds routinely OOM a
-    // single VM on monorepos with parallel pnpm/npm installs (each install
-    // fans out to ~16 fetchers + lifecycle workers, ~70 MB/process), so we
-    // serialize that case behind a FIFO chain — but only the in-VM builds
-    // serialize; pulls and starts run freely alongside them. The remote-builder
-    // probe is memoized, so this up-front call is free; skip it with no builds.
+    // low VM RAM). Dockerfile builds run inside the VM, and two or more
+    // concurrent builds routinely OOM a single VM on monorepos with parallel
+    // pnpm/npm installs (each install fans out to ~16 fetchers + lifecycle
+    // workers, ~70 MB/process), so builds serialize behind a FIFO chain —
+    // but only the builds; pulls and starts run freely alongside them.
     const tags = new Map();
     const builds = services.filter((s) => s.image.type === "dockerfile");
-    const buildsRunHostSide = builds.length > 0 && (await ensureRemoteBuilder());
+    const buildsRunHostSide = false;
     // A promise chain is a fair FIFO mutex: when builds run in-VM, each build
     // waits for the previous to settle. Pulls and host-side builds bypass it.
     let inVmBuildChain = Promise.resolve();
@@ -2952,11 +3128,16 @@ async function bootstrapInner() {
         if (svc.setup) {
             progressService(svc.name, { status: "probing", detail: "running setup" });
             const helpers = await ensureHelpers(svc.name, svc);
-            await svc.setup({
-                name: svc.name,
-                helpers,
-                ...(await spectestContext({ service: svc.name, includeSelf: true })),
-            });
+            try {
+                await svc.setup({
+                    name: svc.name,
+                    helpers,
+                    ...(await spectestContext({ service: svc.name, includeSelf: true })),
+                });
+            }
+            catch (err) {
+                throw await withContainerPostMortem(svc.name, err);
+            }
         }
         progressService(svc.name, { status: "ready", detail: undefined });
         const ti = timings.get(svc.name);