@specific.dev/spectest 0.69.0 → 0.71.0

package/src/daemon.ts

@@ -92,6 +92,7 @@ import {
 import { pollUntilReady } from "./harness/ready-poll.js";
 import { runWrapperRules } from "./harness/wrapper-rules.js";
 import type { WrapperDiagnostic } from "./harness/wrapper-rules.js";
+import { cpus } from "node:os";
 import { APP_DIR, WORKSPACE, resolveProjectPath } from "./project-files.js";
 import {
   isTextualContentType,
@@ -116,6 +117,7 @@ import {
   certCovers as hostmatchCertCovers,
   hostWithoutPort,
   matchRoute,
+  selectCertName,
   wildcardCoversHost,
   wildcardSuffix,
 } from "./harness/hostmatch.js";
@@ -123,7 +125,6 @@ import {
   INGRESS_HTTPS_PORT,
   INGRESS_HTTP_PORT,
   bindRoute,
-  certEntries,
   clearTables,
   emptyTables,
   planBind,
@@ -132,6 +133,7 @@ import {
   unbindRoute,
   type Route as IngressRoute,
 } from "./harness/ingress-table.js";
+import { startTlsTerminator, type TlsTerminator } from "./harness/tls-terminator.js";
 import { runContainerArgs } from "./harness/container-run.js";
 import {
   assertAbsolute,
@@ -239,30 +241,49 @@ const DEFAULT_TEST_TIMEOUT_MS = 60_000;
 const NETWORK_NAME = process.env.SPECTEST_NETWORK ?? "spectest-net";
 
 // Stable hostname every service container resolves to the host (the
-// `spectest-br0` gateway) — so apps that build or pull images at runtime
-// can point a builder at `spectest-host:5000` (the zot Docker Hub mirror)
-// or `spectest-host:1234` (the shared buildkitd) without hard-coding the
-// gateway IP. Injected into each container's /etc/hosts in runContainer.
+// `spectest-br0` gateway) — so an app that runs its OWN BuildKit inside a
+// test can point its cache export at the host's build-cache registry,
+// `spectest-host:5007`, without hard-coding the gateway IP. Nothing else
+// lives behind it any more: images are pulled and built inside the VM
+// against the container store (CONTAINER_STORE.md). Injected into each
+// container's /etc/hosts in runContainer.
 const SPECTEST_HOST_NAME = "spectest-host";
 
-// The host image-cache gateway, discovered once from the same
-// `registry-mirrors` entry the in-VM dockerd already uses (baked into the
-// local provider's golden /etc/docker/daemon.json). `null` when there's
-// no host cache, so nothing is injected.
+// The host gateway, read once from the guest's own default route
+// (/proc/net/route: destination 0, gateway as a little-endian hex word).
+// `null` when the guest has no default route, in which case nothing is
+// injected.
 let _hostCacheGateway: string | null | undefined;
 function hostCacheGateway(): string | null {
   if (_hostCacheGateway !== undefined) return _hostCacheGateway;
+  _hostCacheGateway = null;
   try {
-    const cfg = JSON.parse(
-      readFileSync("/etc/docker/daemon.json", "utf8"),
-    ) as { "registry-mirrors"?: string[] };
-    const first = cfg["registry-mirrors"]?.[0];
-    _hostCacheGateway = first ? new URL(first).hostname || null : null;
+    for (const line of readFileSync("/proc/net/route", "utf8").split("\n").slice(1)) {
+      const f = line.trim().split(/\s+/);
+      if (f.length < 3 || f[1] !== "00000000") continue;
+      const hex = f[2];
+      const octets = [6, 4, 2, 0].map((i) => parseInt(hex.slice(i, i + 2), 16));
+      if (octets.every((o) => Number.isFinite(o))) {
+        _hostCacheGateway = octets.join(".");
+        break;
+      }
+    }
   } catch {
     _hostCacheGateway = null;
   }
   return _hostCacheGateway;
 }
+/** vCPUs this guest has, for BuildKit's `max-parallelism` — the number
+ *  Blacksmith sets too, and the one that matters now that the build
+ *  competes for the guest's own cores rather than the host's. */
+function cpuCount(): number {
+  try {
+    return Math.max(1, cpus().length);
+  } catch {
+    return 4;
+  }
+}
+
 // WORKSPACE (/workspace) and APP_DIR (/opt/spectest/app) both live in
 // project-files.ts, next to the rule that decides which copy of a project
 // file is the current one.
@@ -687,46 +708,110 @@ async function hasBuildx(): Promise<boolean> {
   return _buildxAvailable;
 }
 
-// A single buildkitd runs on the host (see scripts/install-buildkitd.sh),
-// reachable from every VM at the bridge gateway. Building against it as a
-// `remote` buildx builder gives a persistent, shared layer/mount cache that
-// survives forks and warm-template misses — a fresh VM no longer rebuilds
-// from scratch. The build runs on the host (runc-isolated); `--load` pulls
-// the finished image back into the in-VM dockerd. Detected once; if the
-// builder can't be created or buildkitd is unreachable we fall back to the
-// in-VM builder, so a missing/dead buildkitd just means slower builds.
-const REMOTE_BUILDER_ADDR = process.env.SPECTEST_BUILDKIT_ADDR ?? "tcp://10.42.0.1:1234";
-const REMOTE_BUILDER_NAME = "spectest-remote";
-/** Parent of the per-build buildx config dirs (see isolatedBuildxConfig).
- *  On tmpfs: each holds a builder stub and an 8-byte node id. */
-let _remoteBuilder: boolean | undefined;
-async function ensureRemoteBuilder(): Promise<boolean> {
-  if (_remoteBuilder !== undefined) return _remoteBuilder;
-  if (!(await hasBuildx())) {
-    _remoteBuilder = false;
+// Every VM carries its project's container store (CONTAINER_STORE.md):
+// the disk every pull and every build lands on, mounted by the control
+// plane before this harness starts. The manifest below says where; the
+// in-VM buildkitd keeps its exported cache there too, so a fresh VM finds
+// every layer it built before. Detected once; if the daemon will not
+// start, dockerd's own BuildKit builds instead.
+const IMAGE_CACHE_MANIFEST = "/run/spectest-image-cache.json";
+const LOCAL_BUILDER_NAME = "spectest-local";
+const LOCAL_BUILDKIT_ADDR = "tcp://127.0.0.1:1234";
+/** The bring-up script the cache base bakes (`base.rs::BUILDKITD_UP_SH`). */
+const BUILDKITD_UP_PATH = "/usr/local/bin/spectest-buildkitd-up";
+
+/** Where the control plane mounted this VM's image cache: containerd's
+ *  root (read-write, this VM's own) and the layers disk (read-only, shared
+ *  by every VM of a generation). `null` when the VM carries no cache. */
+async function imageCachePaths(): Promise<{ root: string; layers: string } | null> {
+  try {
+    const raw = await fs.readFile(IMAGE_CACHE_MANIFEST, "utf8");
+    const parsed = JSON.parse(raw) as { disks?: { role?: string; path?: string }[] };
+    const root = (parsed.disks ?? []).find((d) => d.role === "root" && d.path)?.path;
+    const layers = (parsed.disks ?? []).find((d) => d.role === "layers" && d.path)?.path;
+    return root && layers ? { root, layers } : null;
+  } catch {
+    return null;
+  }
+}
+
+let _localBuilder: boolean | undefined;
+/**
+ * Start buildkitd inside this VM with its state on the cache disk, and
+ * register it as a buildx `remote` builder.
+ *
+ * Lazy on purpose: it runs on the first build a project actually does,
+ * so a project with no dockerfile service never pays for a builder. And
+ * best-effort: a daemon that will not start falls back to the shared host
+ * one, which is a slower build and not a failed one.
+ */
+async function ensureLocalBuildkitd(): Promise<boolean> {
+  if (_localBuilder !== undefined) return _localBuilder;
+  _localBuilder = false;
+  const paths = await imageCachePaths();
+  if (!paths) return false;
+  const state = paths.root;
+  if (!(await hasBuildx())) return false;
+
+  // The bring-up itself is a script baked into the image cache's base
+  // snapshot (`base.rs::BUILDKITD_UP_SH`), so production and the real-VM
+  // test drive exactly the same daemon with exactly the same config. All
+  // it takes from us is where the state lives — the registry routing is
+  // its own (Docker Hub via `mirror.gcr.io`, deliberately not the host
+  // zot instance).
+  const started = await shx("/bin/bash", [BUILDKITD_UP_PATH, state], 900_000);
+  if (started.code !== 0) {
+    // eslint-disable-next-line no-console
+    console.warn(
+      `[build] in-VM buildkitd would not start; building with dockerd's own BuildKit:\n${(started.stderr || started.stdout).trim()}`,
+    );
     return false;
   }
-  // Idempotent: a repeat create with the same name errors ("existing
-  // instance"), which we treat as already-present.
   const create = await docker(
-    ["buildx", "create", "--name", REMOTE_BUILDER_NAME, "--driver", "remote", REMOTE_BUILDER_ADDR],
+    ["buildx", "create", "--name", LOCAL_BUILDER_NAME, "--driver", "remote", LOCAL_BUILDKIT_ADDR],
     30_000,
   );
   if (create.code !== 0 && !/existing instance|already exists/i.test(create.stderr)) {
-    _remoteBuilder = false;
+    // eslint-disable-next-line no-console
+    console.warn(`[build] could not register the in-VM builder:\n${create.stderr.trim()}`);
     return false;
   }
-  // `inspect --bootstrap` actually dials buildkitd, so it's our reachability
-  // probe. If buildkitd is down this fails and we fall back.
-  const boot = await docker(["buildx", "inspect", "--bootstrap", REMOTE_BUILDER_NAME], 60_000);
-  _remoteBuilder = boot.code === 0;
-  if (!_remoteBuilder) {
+  // `inspect --bootstrap` dials the daemon, so it is both the readiness
+  // wait and the proof it is really answering.
+  const boot = await docker(["buildx", "inspect", "--bootstrap", LOCAL_BUILDER_NAME], 120_000);
+  if (boot.code !== 0) {
     // eslint-disable-next-line no-console
-    console.warn(
-      `[build] remote buildkitd at ${REMOTE_BUILDER_ADDR} unreachable; using in-VM builder:\n${boot.stderr.trim()}`,
-    );
+    console.warn(`[build] in-VM buildkitd never answered; building with dockerd's own BuildKit:\n${boot.stderr.trim()}`);
+    return false;
   }
-  return _remoteBuilder;
+  // eslint-disable-next-line no-console
+  console.log(`[build] building in this VM against the image cache (root ${paths.root}, layers ${paths.layers})`);
+  // Imports come from the merged cache on the read-only layers disk and
+  // from this lineage's own exports on the root; exports go to the root,
+  // where the merge picks them up (image_cache/merge.rs).
+  _localCacheDir = `${state}/spectest-buildkit-cache`;
+  _localCacheImports = [`${paths.layers}/spectest-buildkit-cache`, `${state}/spectest-buildkit-cache`];
+  _localBuilder = true;
+  return true;
+}
+/** The exported-cache directory on the cache disk, once the in-VM builder is up. */
+let _localCacheDir: string | null = null;
+/** The cache directories a build imports from: the merged one on the
+ *  layers disk, then this lineage's own exports. */
+let _localCacheImports: string[] = [];
+/**
+ * A reference as containerd names it. buildx's `-t` on a remote builder
+ * stores an unqualified name (`probe:bx`) that dockerd then cannot
+ * resolve (measured, CONTAINER_STORE.md), so the in-VM build names its
+ * output in full.
+ */
+function qualifyImageRef(ref: string): string {
+  const slash = ref.indexOf("/");
+  const first = slash < 0 ? "" : ref.slice(0, slash);
+  const isRegistry = first.includes(".") || first.includes(":") || first === "localhost";
+  if (slash < 0) return `docker.io/library/${ref}`;
+  if (!isRegistry) return `docker.io/${ref}`;
+  return ref;
 }
 
 async function ensureNetwork(): Promise<void> {
@@ -787,7 +872,12 @@ async function ensureVolumes(svc: NamedService): Promise<string[]> {
     // created here nor recorded for the delta-restore wipe.
     if (!(await isExistingNonDirectory(host))) {
       await fs.mkdir(host, { recursive: true });
-      if (vol.source?.startsWith("/") && !host.startsWith("/var/cache/spectest/")) {
+      // …and neither is the pre-disks cache tree, kept for a server still
+      // serving older SDKs. Leaving it out of the manifest is what
+      // protects a project running this SDK against a server whose
+      // teardown guard predates it.
+      const durable = host.startsWith("/var/cache/spectest/");
+      if (vol.source?.startsWith("/") && !durable) {
         await recordAbsoluteVolumeDir(host);
       }
     }
@@ -1347,24 +1437,46 @@ async function runServiceBuild(
     // predates per-Dockerfile ignores — and is written ONLY when the
     // project ships none of its own (see readProjectDockerignore).
     await fs.writeFile(`${dfPath}.dockerignore`, serviceDockerignore(image.exclude));
-    const useRemote = await ensureRemoteBuilder();
-    // Both the remote builder and a local buildx are BuildKit, so both emit
-    // per-step timing on stderr under `--progress=plain` (parsed below). Only
-    // the legacy in-VM builder takes no progress flag.
-    const useBuildKit = useRemote || (await hasBuildx());
+    // The in-VM buildkitd on the container store first: the build runs
+    // inside the guest's own isolation boundary, against this project's
+    // own layer cache, and the finished image is already in the store
+    // dockerd reads. dockerd's built-in BuildKit is the fallback (a guest
+    // with no store, or a daemon that would not start).
+    const useLocal = await ensureLocalBuildkitd();
+    // Both the in-VM daemon and dockerd's buildx are BuildKit, so both
+    // emit per-step timing on stderr under `--progress=plain` (parsed
+    // below). Only the legacy builder takes no progress flag.
+    const useBuildKit = useLocal || (await hasBuildx());
     const buildEnv: Record<string, string> = {};
     let buildArgs: string[];
     // The user's `buildArgs`, as `--build-arg` flags; a plain client flag,
     // so every builder — host buildkitd, in-VM BuildKit, legacy — takes it.
     const argFlags = buildArgFlags(image.buildArgs);
-    if (useRemote) {
-      // Build on the host-side shared buildkitd (persistent cross-VM cache);
-      // `--load` brings the finished image back into the in-VM dockerd so
-      // runContainer can `docker run` it. The build context (WORKSPACE, minus
-      // .dockerignore) streams to buildkitd over the bridge.
+    if (useLocal && _localCacheDir) {
+      // The in-VM builder is BuildKit's containerd worker on this VM's
+      // own image store (CONTAINER_STORE.md): the output is an image
+      // record in dockerd's namespace, unpacked, so there is no `--load`
+      // and nothing crosses a socket. The cache directory on the image cache
+      // disk is what outlives the VM; `mode=max` keeps every
+      // intermediate layer, uncompressed so an import never inflates,
+      // and one tag per service so exports do not replace each other.
+      const cacheTag = name.replace(/[^a-z0-9-]/gi, "-").toLowerCase();
       buildArgs = [
         "buildx", "build",
-        "--builder", REMOTE_BUILDER_NAME,
+        "--builder", LOCAL_BUILDER_NAME,
+        "--progress=plain",
+        "--output", `type=image,name=${qualifyImageRef(tag)},unpack=true`,
+        ..._localCacheImports.flatMap((src) => ["--cache-from", `type=local,src=${src}`]),
+        "--cache-to", `type=local,dest=${_localCacheDir},mode=max,compression=uncompressed,force-compression=true,tag=${cacheTag}`,
+        ...argFlags,
+        "-f", dfPath, WORKSPACE,
+      ];
+    } else if (useLocal) {
+      // The daemon came up but reported no cache directory: build on it
+      // and `--load` the result into dockerd.
+      buildArgs = [
+        "buildx", "build",
+        "--builder", LOCAL_BUILDER_NAME,
         "--load",
         "--progress=plain",
         ...argFlags,
@@ -1673,6 +1785,45 @@ async function waitForReady(svc: NamedService): Promise<void> {
   throw new Error(msg);
 }
 
+/**
+ * Add the container's own account of its death to an error raised by its
+ * `setup` hook.
+ *
+ * `waitForReady` already does this, because a container that never becomes
+ * ready is obviously the container's fault. A `setup` hook is the case that
+ * was missing, and it is the one that reads most misleadingly: the hook
+ * talks to the service over the network, so when the container dies
+ * mid-hook what surfaces is a name that no longer resolves or a rollout
+ * that never finished — a symptom from the far end of a connection to
+ * something that is not there any more. The reason is in a log that goes
+ * with the VM at teardown.
+ *
+ * Only for a container that is **gone**: one that is still up did not cause
+ * this, and its log would bury the real error. Best-effort throughout — a
+ * diagnostic must never replace the failure it explains.
+ */
+async function withContainerPostMortem(name: string, err: unknown): Promise<unknown> {
+  try {
+    const state = await docker(
+      ["inspect", "-f", "{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}}", name],
+      15_000,
+    );
+    const [running, code, oom] = state.stdout.trim().split(/\s+/);
+    if (state.code !== 0 || running !== "false") return err;
+    const logs = await docker(["logs", "--tail=120", name], 30_000);
+    const output = `${logs.stdout}\n${logs.stderr}`.trim();
+    const base = err instanceof Error ? err : new Error(String(err));
+    base.message +=
+      `\n\nThe "${name}" container exited (code ${code}` +
+      `${oom === "true" ? ", OOM-killed" : ""}) while its setup hook was running, ` +
+      `which is why the hook could not reach it.` +
+      (output ? `\nIts last output:\n${output}` : `\nIt logged nothing.`);
+    return base;
+  } catch {
+    return err;
+  }
+}
+
 /** Validate the `dependsOn` graph and return the name→service map used to
  *  walk it. Rules live in `harness/service-graph.ts`. */
 function validateServiceGraph(services: NamedService[]): Map<string, NamedService> {
@@ -1769,42 +1920,29 @@ const INGRESS_HTTP_SERVERS = new Map<number, any>();
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const INGRESS_HTTPS_SERVERS = new Map<number, any>();
 /**
- * Servers replaced by a rebind and now draining. On Bun 1.3.14 a request
- * arriving on a kept-alive connection of a `stop(false)`-drained server
- * dispatches into freed per-server state and can SEGFAULT the process
- * (use-after-free class fixed upstream by oven-sh/bun#36790, first in Bun
- * 1.4.0; observed here as `panic: Segmentation fault at address 0xA` in
- * `server.zig onRequestFor` on ~2-3 % of runtime-TLS rebinds). Until the
- * Bun bump lands, shrink the number of requests a drained server can ever
- * see: every response it still serves carries `Connection: close` (one
- * more request per surviving connection, not unlimited), and a grace timer
- * force-closes whatever is left ({@link REBIND_DRAIN_GRACE_MS}).
+ * The :443 TLS terminator, once per harness process.
+ *
+ * There is deliberately no draining machinery beside it any more. :443
+ * used to be a `Bun.serve` that had to be swapped for every new
+ * certificate, and no swap can be made safe: `stop(false)` FINs an idle
+ * kept-alive connection within 1-2 ms, so a client that wrote a request in
+ * that instant failed with `SocketError: other side closed` — in whatever
+ * unrelated test happened to be talking through the ingress.
+ * {@link startTlsTerminator} chooses the leaf per handshake instead, so
+ * the listener is bound once and lives as long as the project does.
  */
-const DRAINING_INGRESS = new WeakSet<object>();
-/** How long a drained listener may keep serving in-flight work before its
- *  remaining connections are force-closed. Long enough for a slow proxied
- *  response to finish, short enough to bound the 1.3.14 UAF window. */
-const REBIND_DRAIN_GRACE_MS = 15_000;
-
-/** Stamp `Connection: close` on a response served by a draining listener so
- *  the kept-alive connection retires instead of lingering as a UAF trigger.
- *  Proxied responses can carry immutable headers; rewrap when needed. */
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-function withDrainClose(server: any, res: Response): Response {
-  if (!DRAINING_INGRESS.has(server)) return res;
-  try {
-    res.headers.set("connection", "close");
-    return res;
-  } catch {
-    const headers = new Headers(res.headers);
-    headers.set("connection", "close");
-    return new Response(res.body, {
-      status: res.status,
-      statusText: res.statusText,
-      headers,
-    });
-  }
-}
+let HTTPS_TERMINATOR: TlsTerminator | undefined;
+/**
+ * The in-flight first bind of :443, if one is running.
+ *
+ * {@link ensureHttpsIngress} is async where the swap it replaced was
+ * synchronous, and that reintroduces an interleaving the old code could
+ * not have: two `ctx.startService({ tls })` calls issued together by a
+ * fake handler would both find :443 absent and both try to bind it, and
+ * the loser gets EADDRINUSE — a provisioning failure in place of the
+ * dropped connection this change exists to remove. One promise, shared.
+ */
+let HTTPS_INGRESS_STARTING: Promise<void> | undefined;
 /**
  * The live ingress tables — per-port routes and the :443 SNI cert table.
  *
@@ -1852,6 +1990,12 @@ function stopIngressServers(): void {
     }
   }
   INGRESS_HTTPS_SERVERS.clear();
+  // Force-closed for the same reason the listeners are: the containers
+  // behind these routes are going away, so a connection still riding them
+  // has nothing left to reach.
+  HTTPS_TERMINATOR?.close();
+  HTTPS_TERMINATOR = undefined;
+  HTTPS_INGRESS_STARTING = undefined;
   clearTables(INGRESS);
 }
 
@@ -2089,7 +2233,7 @@ async function startIngress(): Promise<void> {
     console.log(`[ingress] http :${port} for ${[...byHost.keys()].join(", ")}`);
   }
   // ── HTTPS listener on INGRESS_HTTPS_PORT: SNI per certificated hostname.
-  if (INGRESS.certByHost.size > 0) rebindHttpsListener(Bun);
+  if (INGRESS.certByHost.size > 0) await ensureHttpsIngress(Bun);
 
   // Seed the resolver's names registry: ingress hostnames (fakes, TLS
   // proxies, dnsName(→ingress)) → bridge gateway, plus ingress-targeted
@@ -2110,86 +2254,85 @@ function requireBun(): any {
   return Bun;
 }
 
-/** Flatten the live SNI cert table into Bun's TLS-entry array. */
-function tlsEntriesFromCerts(): Array<{ cert: string; key: string; serverName: string }> {
-  return certEntries(INGRESS);
-}
 
 /**
- * (Re)bind the :443 listener from the current cert table + route map.
+ * Bring :443 up, once.
  *
- * Bun fixes a server's TLS config at `Bun.serve` time — `reload()` accepts a
- * new `tls` option and silently keeps serving the old certificates (measured
- * on Bun 1.3.14: after reloading with a second SNI entry, the new hostname
- * still gets the first one's leaf). So adding a certificate really does mean
- * a second listener.
+ * Two pieces: a plaintext `Bun.serve` on an ephemeral loopback port that
+ * carries the HTTPS route table (and therefore all the routing, fake
+ * dispatch, interception, reverse-proxying and `server.upgrade()`
+ * WebSocket bridging that :80 already gets, unchanged), and the TLS
+ * terminator in front of it on :443.
  *
- * **Bind the new one before stopping the old one.** Doing it the other way
- * round — which is what this used to do, with `stop(true)` — has two teeth:
- * the force-close kills every established connection on :443, and the gap
- * before the new listener binds refuses new ones. Neither is limited to the
- * hostname being added; they hit all the unrelated traffic the listener is
- * carrying. On a project that mints a certificate per provisioned database
- * while deploys stream through the same port, that surfaced as the app under
- * test dying with `SocketError: other side closed` or `ECONNREFUSED
- * <gateway>:443` — a different test each run, and nothing pointing at
- * ingress.
+ * **This is idempotent, and that is the feature.** Adding a certificate
+ * mutates {@link IngressTables.certByHost} and nothing else: the
+ * terminator reads that table on every handshake, so the new hostname is
+ * served by the next connection and no established connection is
+ * disturbed. The listener this replaced had to be stopped and re-served
+ * for each new certificate, and every such swap severed the idle
+ * kept-alive connections it was carrying — see
+ * `harness/tls-terminator.ts` for the measurements and the history.
  *
- * Overlapping the two needs SO_REUSEPORT on both sockets ({@link
- * bindIngressServer} sets it unconditionally for that reason), after which
- * the old listener is drained with `stop(false)` so requests in flight
- * finish. During the overlap the kernel may hand a new connection to either
- * socket, which is safe: the only hostname the two disagree about is the one
- * being added, and it does not resolve until the caller writes the names
- * registry after this returns.
+ * The route `Map` is the persistent module object, so the plaintext
+ * server closes over the same table and later route additions need no
+ * restart either.
  *
- * The route Map is the persistent module object, so the new listener closes
- * over the same table and later route additions need no rebind at all.
+ * The loopback port is ephemeral rather than fixed so it can never
+ * collide with a fake's declared port; it is bound on 127.0.0.1, which no
+ * container can reach (containers arrive at the bridge gateway), so the
+ * only way in is through the terminator.
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-function rebindHttpsListener(Bun: any): void {
-  // The *same* table object the previous listener closed over: the new
-  // listener must serve the routes bound since, and any bound later.
+async function ensureHttpsIngress(Bun: any): Promise<void> {
+  if (INGRESS_HTTPS_SERVERS.has(INGRESS_HTTPS_PORT)) return;
+  if (!HTTPS_INGRESS_STARTING) {
+    // Cleared on both outcomes: once bound, the check above short-circuits
+    // every later caller, and a failed bind must be retryable rather than
+    // remembered as a rejection for the life of the project.
+    HTTPS_INGRESS_STARTING = startHttpsIngress(Bun).finally(() => {
+      HTTPS_INGRESS_STARTING = undefined;
+    });
+  }
+  return HTTPS_INGRESS_STARTING;
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+async function startHttpsIngress(Bun: any): Promise<void> {
   const routes = routesFor(INGRESS, INGRESS_HTTPS_PORT);
-  const old = INGRESS_HTTPS_SERVERS.get(INGRESS_HTTPS_PORT);
-  const server = bindIngressServer(
-    Bun,
-    INGRESS_HTTPS_PORT,
-    routes,
-    `https :${INGRESS_HTTPS_PORT}`,
-    tlsEntriesFromCerts(),
-  );
-  if (old) {
+  // `proto` is passed rather than inferred: this listener speaks plain
+  // HTTP, but everything reaching it arrived over TLS, so X-Forwarded-Proto
+  // must say https.
+  const plain = bindIngressServer(Bun, 0, routes, `https :${INGRESS_HTTPS_PORT}`, {
+    proto: "https",
+    hostname: "127.0.0.1",
+  });
+  INGRESS_HTTPS_SERVERS.set(INGRESS_HTTPS_PORT, plain);
+  try {
+    HTTPS_TERMINATOR = await startTlsTerminator({
+      port: INGRESS_HTTPS_PORT,
+      hostname: "0.0.0.0",
+      upstreamPort: plain.port,
+      certFor: (serverName) => {
+        if (serverName === undefined) return INGRESS.certByHost.values().next().value;
+        const name = selectCertName(INGRESS.certByHost.keys(), serverName.toLowerCase());
+        return name === undefined ? undefined : INGRESS.certByHost.get(name);
+      },
+    });
+  } catch (err) {
+    // Leave nothing half-built: a registered plaintext server with no
+    // terminator would make `planBind` believe :443 is up.
+    INGRESS_HTTPS_SERVERS.delete(INGRESS_HTTPS_PORT);
     try {
-      // Graceful: stop accepting, let in-flight requests finish. The old
-      // server stays alive until they do, which is the point — a long
-      // upload through ingress must not be collateral damage of another
-      // service being provisioned.
-      old.stop(false);
-      // Bun 1.3.14 landmine: a request arriving later on one of the old
-      // server's kept-alive connections dispatches into freed state and can
-      // segfault the daemon (see {@link DRAINING_INGRESS}). Mark it so any
-      // response it still serves closes its connection, and force-close the
-      // stragglers once in-flight work has had a fair window to finish.
-      DRAINING_INGRESS.add(old);
-      const graceTimer = setTimeout(() => {
-        try {
-          old.stop(true);
-        } catch {
-          /* already fully stopped */
-        }
-      }, REBIND_DRAIN_GRACE_MS);
-      // Don't let the grace timer keep the process alive on shutdown.
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      (graceTimer as any).unref?.();
-    } catch (err) {
-      // eslint-disable-next-line no-console
-      console.warn("[ingress] failed to drain the previous https listener:", err);
+      plain.stop(true);
+    } catch {
+      /* already gone */
     }
+    throw err;
   }
-  INGRESS_HTTPS_SERVERS.set(INGRESS_HTTPS_PORT, server);
   // eslint-disable-next-line no-console
-  console.log(`[ingress] https :${INGRESS_HTTPS_PORT} for ${[...routes.keys()].join(", ")}`);
+  console.log(
+    `[ingress] https :${INGRESS_HTTPS_PORT} (tls terminator -> :${plain.port}) for ${[...routes.keys()].join(", ")}`,
+  );
 }
 
 /**
@@ -2258,9 +2401,12 @@ async function bindRuntimeTls(hostname: string, service: string, port: number):
     );
   }
   if (plan.needsCert) {
+    // The terminator reads this table on every handshake, so the leaf is
+    // live the moment it lands — no listener is touched, and no connection
+    // already on :443 notices.
     INGRESS.certByHost.set(host, await generateHostCert(host, [host]));
   }
-  if (plan.needsHttpsRebind) rebindHttpsListener(Bun);
+  if (plan.needsHttpsListener) await ensureHttpsIngress(Bun);
 
   // Resolve the hostname to the daemon gateway (where :443/:80 listen).
   // A wildcard can only live in the resolver's suffix table.
@@ -2312,22 +2458,27 @@ function bindIngressServer(
   port: number,
   byHost: Map<string, Route>,
   listenerLabel: string,
-  tlsEntries?: Array<{ cert: string; key: string; serverName: string }>,
+  // Every ingress listener speaks plain HTTP now — the one behind :443
+  // sits under the TLS terminator. So the scheme a client actually used
+  // can no longer be inferred from the socket and is declared instead; it
+  // stamps X-Forwarded-Proto, which upstreams that build absolute URLs or
+  // redirect depend on.
+  serve: { proto: "http" | "https"; hostname?: string } = { proto: "http" },
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
 ): any {
-  // A TLS listener terminates https; everything else is plain http. Used
-  // to stamp X-Forwarded-Proto so upstreams that build absolute URLs or
-  // redirect see the scheme the client actually used, not our http hop.
-  const proto = tlsEntries ? "https" : "http";
+  const proto = serve.proto;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const opts: Record<string, any> = {
     port,
-    hostname: "0.0.0.0",
-    // SO_REUSEPORT on every ingress listener, so a replacement can be bound
-    // while the old one is still serving. That overlap is the only way to
-    // add an SNI certificate without a gap — see {@link rebindHttpsListener}
-    // — and it only works if *both* sockets opt in: a second plain bind
-    // fails with "Is port 443 in use?".
+    hostname: serve.hostname ?? "0.0.0.0",
+    // SO_REUSEPORT on every ingress listener. It was introduced so a
+    // replacement :443 could overlap the original during a certificate
+    // swap; the terminator removed that swap, and this stays for the
+    // remaining case — the `/load` teardown force-closes and rebinds, and
+    // a socket the kernel has not finished releasing would otherwise fail
+    // the next bind with "Is port 443 in use?". Note the flip side, which
+    // {@link stopIngressServers} depends on: a half-alive listener shares
+    // the port silently instead of colliding loudly.
     reusePort: true,
     // Bun.serve defaults to a 10s idleTimeout, which kills any proxied
     // request whose upstream takes >10s to produce bytes — under parallel
@@ -2340,9 +2491,7 @@ function bindIngressServer(
     idleTimeout: 0,
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     fetch: (req: Request, server: any): Response | Promise<Response> =>
-      dispatchIngress(req, server, byHost, listenerLabel, proto).then((res) =>
-        withDrainClose(server, res),
-      ),
+      dispatchIngress(req, server, byHost, listenerLabel, proto),
     websocket: {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       async open(ws: any) {
@@ -2412,7 +2561,6 @@ function bindIngressServer(
       },
     },
   };
-  if (tlsEntries) opts.tls = tlsEntries;
   return Bun.serve(opts);
 }
 
@@ -2489,6 +2637,27 @@ async function dispatchIngress(
   return augmentCorsResponse(req, res);
 }
 
+/**
+ * The address to report as the client's in `X-Forwarded-For`.
+ *
+ * On :80 that is simply the socket's peer. On :443 the peer is the TLS
+ * terminator on loopback, so the real address has to be recovered from it:
+ * the terminator keys `source port -> client address` for the life of each
+ * upstream connection, and the source port is what `requestIP` reports
+ * here. Gated on `proto === "https"` so a genuinely loopback caller on :80
+ * can never pick up an unrelated terminator connection's port.
+ *
+ * Falling back to the socket's own address means the worst case is the
+ * pre-terminator answer for a plain HTTP hop, never a wrong tenant.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function ingressClientIp(server: any, req: Request, proto: string): string | undefined {
+  const peer = server.requestIP?.(req) as { address?: string; port?: number } | undefined;
+  if (!peer?.address) return undefined;
+  if (proto !== "https") return peer.address;
+  return HTTPS_TERMINATOR?.clientIpFor(peer.port) ?? peer.address;
+}
+
 /**
  * Reverse-proxy a request to `http://<service>:<port>` on
  * `spectest-net`. Handles plain HTTP/1.1 + 2 and WebSocket upgrades:
@@ -2548,7 +2717,7 @@ async function proxyToService(
   // Standard reverse-proxy provenance headers: the upstream sees the
   // public scheme/host it was reached through and the client's address,
   // even though we rewrite Host below to the service-net name.
-  const clientIp = server.requestIP?.(req)?.address as string | undefined;
+  const clientIp = ingressClientIp(server, req, proto);
   const priorXff = req.headers.get("x-forwarded-for");
   const xff = clientIp ? (priorXff ? `${priorXff}, ${clientIp}` : clientIp) : priorXff;
   if (xff) fwdHeaders.set("x-forwarded-for", xff);
@@ -3429,18 +3598,14 @@ async function bootstrapInner(): Promise<BootstrapTimings> {
   // at "image ready" waiting for an unrelated slow build elsewhere.
   //
   // Prep concurrency: registry pulls always run in parallel (network-bound,
-  // low VM RAM). Dockerfile builds parallelize *only* when the host
-  // buildkitd is in play — there the build executes host-side under runc, so
-  // N concurrent builds don't touch the VM's memory ceiling. When we fall
-  // back to the in-VM builder, two or more concurrent builds routinely OOM a
-  // single VM on monorepos with parallel pnpm/npm installs (each install
-  // fans out to ~16 fetchers + lifecycle workers, ~70 MB/process), so we
-  // serialize that case behind a FIFO chain — but only the in-VM builds
-  // serialize; pulls and starts run freely alongside them. The remote-builder
-  // probe is memoized, so this up-front call is free; skip it with no builds.
+  // low VM RAM). Dockerfile builds run inside the VM, and two or more
+  // concurrent builds routinely OOM a single VM on monorepos with parallel
+  // pnpm/npm installs (each install fans out to ~16 fetchers + lifecycle
+  // workers, ~70 MB/process), so builds serialize behind a FIFO chain —
+  // but only the builds; pulls and starts run freely alongside them.
   const tags = new Map<string, string>();
   const builds = services.filter((s) => s.image.type === "dockerfile");
-  const buildsRunHostSide = builds.length > 0 && (await ensureRemoteBuilder());
+  const buildsRunHostSide = false;
   // A promise chain is a fair FIFO mutex: when builds run in-VM, each build
   // waits for the previous to settle. Pulls and host-side builds bypass it.
   let inVmBuildChain: Promise<unknown> = Promise.resolve();
@@ -3527,11 +3692,15 @@ async function bootstrapInner(): Promise<BootstrapTimings> {
     if (svc.setup) {
       progressService(svc.name, { status: "probing", detail: "running setup" });
       const helpers = await ensureHelpers(svc.name, svc);
-      await svc.setup({
-        name: svc.name,
-        helpers,
-        ...(await spectestContext({ service: svc.name, includeSelf: true })),
-      });
+      try {
+        await svc.setup({
+          name: svc.name,
+          helpers,
+          ...(await spectestContext({ service: svc.name, includeSelf: true })),
+        });
+      } catch (err) {
+        throw await withContainerPostMortem(svc.name, err);
+      }
     }
     progressService(svc.name, { status: "ready", detail: undefined });
     const ti = timings.get(svc.name);