@specific.dev/spectest 0.69.0 → 0.70.0

package/dist/harness/tls-terminator.js

@@ -0,0 +1,199 @@
+/**
+ * The :443 TLS terminator — one listener, for the life of the harness.
+ *
+ * ## Why this exists
+ *
+ * Ingress certificates are minted *while tests run*: a fake that
+ * provisions a database hands the app a CA-trusted `https://<new-host>/`
+ * endpoint, and that hostname did not exist a second earlier. Bun fixes a
+ * server's TLS configuration at `Bun.serve` time — `reload({ tls })` is
+ * accepted and then ignored (re-measured on Bun 1.4.0: the added
+ * `serverName` still gets the first entry's leaf) — so for as long as
+ * :443 was a `Bun.serve`, every new certificate meant swapping the
+ * listener.
+ *
+ * **No listener swap can be made safe.** Binding the replacement first
+ * under SO_REUSEPORT and draining the original with `stop(false)` saves a
+ * request that is already *in flight*, and that is all it saves: measured
+ * on Bun 1.4.0, `stop(false)` sends FIN to an **idle kept-alive
+ * connection within 1–2 ms**. A client that writes a request into that
+ * socket in the same instant gets `SocketError: other side closed`
+ * (`UND_ERR_SOCKET` under undici), and Node's `fetch` does not retry it.
+ * That is a flake in whatever unrelated test happened to be talking
+ * through the ingress — never in the one that provisioned the service —
+ * and it is why the app under test would die far from any ingress code.
+ *
+ * So the listener must stop being swapped, which means the certificate
+ * must be chosen **per handshake** rather than baked into the socket.
+ * That is what every real server does, and under Bun it is reachable
+ * through `node:tls`'s `SNICallback` (verified working on Bun 1.4.0:
+ * adding an entry to the table is served on the next handshake, with no
+ * restart and no effect on any established connection).
+ *
+ * ## The shape, and why it is a byte pipe
+ *
+ * The terminator does TLS and *nothing else*: it accepts on :443, picks a
+ * leaf by SNI, and pipes the decrypted stream to a plaintext `Bun.serve`
+ * bound on loopback. Everything downstream — routing, fakes,
+ * interceptors, reverse-proxying, and `server.upgrade()` WebSocket
+ * bridging — is the same `Bun.serve` code that already serves :80, run
+ * unchanged. Terminating HTTP here instead (`node:https`, `(req, res)`)
+ * would have meant reimplementing the WebSocket bridge against a
+ * different API, which is exactly the kind of trade this change exists to
+ * avoid: one flake removed, another introduced somewhere subtler.
+ *
+ * Measured cost of the extra hop on this host (Bun 1.4.0, undici client,
+ * 50 MB bodies): ~11 % fewer small requests per second, download
+ * throughput unchanged, upload ~500 MB/s against ~800 MB/s direct. Ingress
+ * traffic in a test VM is nowhere near either bound.
+ *
+ * ## Recovering the client's address
+ *
+ * The plaintext server's peer is the terminator, so `server.requestIP()`
+ * reports loopback and `x-forwarded-for` would lose the caller. The
+ * upstream socket's *source port* is unique among live connections, so the
+ * terminator records `source port → real client address` for the life of
+ * each connection and {@link TlsTerminator.clientIpFor} hands it back.
+ * Verified end to end: a client connecting from 127.0.0.2 is reported as
+ * 127.0.0.2 by the plaintext server behind the pipe.
+ */
+import net from "node:net";
+import tls from "node:tls";
+/**
+ * Start the terminator. Resolves once :443 is accepting, so a caller can
+ * register the hostname in DNS immediately afterwards and know the port is
+ * live.
+ */
+/**
+ * Is this the ordinary end of a connection rather than a fault? A peer
+ * that goes away mid-stream, or a half-finished handshake, is routine on a
+ * listener that fronts browsers.
+ */
+function isRoutineDisconnect(err) {
+    const code = err?.code;
+    return (code === "ECONNRESET" ||
+        code === "EPIPE" ||
+        code === "ERR_STREAM_PREMATURE_CLOSE" ||
+        code === "ECONNABORTED");
+}
+export function startTlsTerminator(opts) {
+    // Parsing a PEM into a SecureContext is the expensive half of choosing a
+    // certificate, and the table is stable — cache by the certificate's own
+    // bytes so a handshake is a Map lookup. Keyed on the PEM rather than the
+    // hostname so a re-minted leaf for the same name cannot be served stale.
+    const contexts = new Map();
+    const contextFor = (leaf) => {
+        let ctx = contexts.get(leaf.cert);
+        if (!ctx) {
+            ctx = tls.createSecureContext({ cert: leaf.cert, key: leaf.key });
+            contexts.set(leaf.cert, ctx);
+        }
+        return ctx;
+    };
+    const warn = (message, err) => {
+        // A client hanging up is not a fault, and this listener carries every
+        // browser in the VM: Chromium opens speculative connections and drops
+        // them, a fetch is abandoned when its test ends, and each of those
+        // arrives here as a reset. Logging them would bury the boot log a user
+        // reads with `spectest env logs` in noise that means nothing.
+        if (isRoutineDisconnect(err))
+            return;
+        if (opts.onWarning)
+            opts.onWarning(message, err);
+        // eslint-disable-next-line no-console
+        else
+            console.warn(`[ingress] ${message}:`, err);
+    };
+    /** upstream source port → the address the TLS client connected from. */
+    const clientIpByUpstreamPort = new Map();
+    const live = new Set();
+    // A default leaf is only ever used for a client that sends no SNI at
+    // all; `SNICallback` decides every other handshake. This mirrors what
+    // Bun did with the first entry of its `tls` array.
+    const fallback = opts.certFor(undefined);
+    const server = tls.createServer({
+        ...(fallback ? { cert: fallback.cert, key: fallback.key } : {}),
+        SNICallback: (serverName, cb) => {
+            const leaf = opts.certFor(serverName) ?? fallback;
+            // No certificate at all is a handshake we cannot complete. Answer
+            // with the fallback context rather than an error so the client
+            // gets a certificate-mismatch alert it can report, which is the
+            // same thing it saw when Bun served the first entry.
+            cb(null, leaf ? contextFor(leaf) : undefined);
+        },
+    }, (client) => {
+        // No idle timeout anywhere on the ingress path: it fronts app
+        // endpoints that legitimately take minutes (a deploy), and a
+        // timeout here would surface as a truncated response with no
+        // explanation.
+        client.setTimeout(0);
+        live.add(client);
+        // Nothing reads from `client` until `pipe` below, so a socket that
+        // is written to before the upstream is connected stays paused and
+        // its bytes are buffered rather than dropped — which is the normal
+        // case, since a client sends its first request the instant the
+        // handshake completes.
+        const upstream = net.connect(opts.upstreamPort, "127.0.0.1");
+        upstream.setTimeout(0);
+        upstream.setNoDelay(true);
+        client.setNoDelay(true);
+        let recordedPort;
+        upstream.on("connect", () => {
+            recordedPort = upstream.localPort;
+            if (recordedPort !== undefined && client.remoteAddress) {
+                clientIpByUpstreamPort.set(recordedPort, client.remoteAddress);
+            }
+            // `pipe` carries backpressure and the half-close in both
+            // directions, which is what a WebSocket bridge and a streamed
+            // upload each need.
+            client.pipe(upstream);
+            upstream.pipe(client);
+        });
+        const teardown = () => {
+            if (recordedPort !== undefined)
+                clientIpByUpstreamPort.delete(recordedPort);
+            live.delete(client);
+            client.destroy();
+            upstream.destroy();
+        };
+        client.on("error", (err) => {
+            warn("tls client connection failed", err);
+            teardown();
+        });
+        upstream.on("error", (err) => {
+            warn(`ingress upstream :${opts.upstreamPort} failed`, err);
+            teardown();
+        });
+        client.on("close", teardown);
+        upstream.on("close", teardown);
+    });
+    // A client that aborts mid-handshake, or offers a protocol version we
+    // do not speak, emits this. It must be handled: an unhandled 'error' on
+    // a net server is thrown, and taking the daemon down over one bad
+    // handshake would be a far worse failure than the one being fixed.
+    server.on("tlsClientError", (err) => warn("tls handshake failed", err));
+    server.on("error", (err) => warn(`tls listener :${opts.port} failed`, err));
+    return new Promise((resolve, reject) => {
+        server.once("error", reject);
+        server.listen(opts.port, opts.hostname, () => {
+            server.removeListener("error", reject);
+            resolve({
+                port: opts.port,
+                clientIpFor: (upstreamPort) => upstreamPort === undefined ? undefined : clientIpByUpstreamPort.get(upstreamPort),
+                connectionCount: () => live.size,
+                close: () => {
+                    try {
+                        server.close();
+                    }
+                    catch (err) {
+                        warn("closing the tls listener failed", err);
+                    }
+                    for (const sock of [...live])
+                        sock.destroy();
+                    live.clear();
+                    clientIpByUpstreamPort.clear();
+                },
+            });
+        });
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@specific.dev/spectest",
-  "version": "0.69.0",
+  "version": "0.70.0",
   "description": "Spectest SDK for defining test environments in TypeScript.",
   "type": "module",
   "main": "./dist/index.js",

package/src/daemon.ts

@@ -116,6 +116,7 @@ import {
   certCovers as hostmatchCertCovers,
   hostWithoutPort,
   matchRoute,
+  selectCertName,
   wildcardCoversHost,
   wildcardSuffix,
 } from "./harness/hostmatch.js";
@@ -123,7 +124,6 @@ import {
   INGRESS_HTTPS_PORT,
   INGRESS_HTTP_PORT,
   bindRoute,
-  certEntries,
   clearTables,
   emptyTables,
   planBind,
@@ -132,6 +132,7 @@ import {
   unbindRoute,
   type Route as IngressRoute,
 } from "./harness/ingress-table.js";
+import { startTlsTerminator, type TlsTerminator } from "./harness/tls-terminator.js";
 import { runContainerArgs } from "./harness/container-run.js";
 import {
   assertAbsolute,
@@ -1769,42 +1770,29 @@ const INGRESS_HTTP_SERVERS = new Map<number, any>();
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const INGRESS_HTTPS_SERVERS = new Map<number, any>();
 /**
- * Servers replaced by a rebind and now draining. On Bun 1.3.14 a request
- * arriving on a kept-alive connection of a `stop(false)`-drained server
- * dispatches into freed per-server state and can SEGFAULT the process
- * (use-after-free class fixed upstream by oven-sh/bun#36790, first in Bun
- * 1.4.0; observed here as `panic: Segmentation fault at address 0xA` in
- * `server.zig onRequestFor` on ~2-3 % of runtime-TLS rebinds). Until the
- * Bun bump lands, shrink the number of requests a drained server can ever
- * see: every response it still serves carries `Connection: close` (one
- * more request per surviving connection, not unlimited), and a grace timer
- * force-closes whatever is left ({@link REBIND_DRAIN_GRACE_MS}).
+ * The :443 TLS terminator, once per harness process.
+ *
+ * There is deliberately no draining machinery beside it any more. :443
+ * used to be a `Bun.serve` that had to be swapped for every new
+ * certificate, and no swap can be made safe: `stop(false)` FINs an idle
+ * kept-alive connection within 1-2 ms, so a client that wrote a request in
+ * that instant failed with `SocketError: other side closed` — in whatever
+ * unrelated test happened to be talking through the ingress.
+ * {@link startTlsTerminator} chooses the leaf per handshake instead, so
+ * the listener is bound once and lives as long as the project does.
  */
-const DRAINING_INGRESS = new WeakSet<object>();
-/** How long a drained listener may keep serving in-flight work before its
- *  remaining connections are force-closed. Long enough for a slow proxied
- *  response to finish, short enough to bound the 1.3.14 UAF window. */
-const REBIND_DRAIN_GRACE_MS = 15_000;
-
-/** Stamp `Connection: close` on a response served by a draining listener so
- *  the kept-alive connection retires instead of lingering as a UAF trigger.
- *  Proxied responses can carry immutable headers; rewrap when needed. */
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-function withDrainClose(server: any, res: Response): Response {
-  if (!DRAINING_INGRESS.has(server)) return res;
-  try {
-    res.headers.set("connection", "close");
-    return res;
-  } catch {
-    const headers = new Headers(res.headers);
-    headers.set("connection", "close");
-    return new Response(res.body, {
-      status: res.status,
-      statusText: res.statusText,
-      headers,
-    });
-  }
-}
+let HTTPS_TERMINATOR: TlsTerminator | undefined;
+/**
+ * The in-flight first bind of :443, if one is running.
+ *
+ * {@link ensureHttpsIngress} is async where the swap it replaced was
+ * synchronous, and that reintroduces an interleaving the old code could
+ * not have: two `ctx.startService({ tls })` calls issued together by a
+ * fake handler would both find :443 absent and both try to bind it, and
+ * the loser gets EADDRINUSE — a provisioning failure in place of the
+ * dropped connection this change exists to remove. One promise, shared.
+ */
+let HTTPS_INGRESS_STARTING: Promise<void> | undefined;
 /**
  * The live ingress tables — per-port routes and the :443 SNI cert table.
  *
@@ -1852,6 +1840,12 @@ function stopIngressServers(): void {
     }
   }
   INGRESS_HTTPS_SERVERS.clear();
+  // Force-closed for the same reason the listeners are: the containers
+  // behind these routes are going away, so a connection still riding them
+  // has nothing left to reach.
+  HTTPS_TERMINATOR?.close();
+  HTTPS_TERMINATOR = undefined;
+  HTTPS_INGRESS_STARTING = undefined;
   clearTables(INGRESS);
 }
 
@@ -2089,7 +2083,7 @@ async function startIngress(): Promise<void> {
     console.log(`[ingress] http :${port} for ${[...byHost.keys()].join(", ")}`);
   }
   // ── HTTPS listener on INGRESS_HTTPS_PORT: SNI per certificated hostname.
-  if (INGRESS.certByHost.size > 0) rebindHttpsListener(Bun);
+  if (INGRESS.certByHost.size > 0) await ensureHttpsIngress(Bun);
 
   // Seed the resolver's names registry: ingress hostnames (fakes, TLS
   // proxies, dnsName(→ingress)) → bridge gateway, plus ingress-targeted
@@ -2110,86 +2104,85 @@ function requireBun(): any {
   return Bun;
 }
 
-/** Flatten the live SNI cert table into Bun's TLS-entry array. */
-function tlsEntriesFromCerts(): Array<{ cert: string; key: string; serverName: string }> {
-  return certEntries(INGRESS);
-}
 
 /**
- * (Re)bind the :443 listener from the current cert table + route map.
+ * Bring :443 up, once.
  *
- * Bun fixes a server's TLS config at `Bun.serve` time — `reload()` accepts a
- * new `tls` option and silently keeps serving the old certificates (measured
- * on Bun 1.3.14: after reloading with a second SNI entry, the new hostname
- * still gets the first one's leaf). So adding a certificate really does mean
- * a second listener.
+ * Two pieces: a plaintext `Bun.serve` on an ephemeral loopback port that
+ * carries the HTTPS route table (and therefore all the routing, fake
+ * dispatch, interception, reverse-proxying and `server.upgrade()`
+ * WebSocket bridging that :80 already gets, unchanged), and the TLS
+ * terminator in front of it on :443.
  *
- * **Bind the new one before stopping the old one.** Doing it the other way
- * round — which is what this used to do, with `stop(true)` — has two teeth:
- * the force-close kills every established connection on :443, and the gap
- * before the new listener binds refuses new ones. Neither is limited to the
- * hostname being added; they hit all the unrelated traffic the listener is
- * carrying. On a project that mints a certificate per provisioned database
- * while deploys stream through the same port, that surfaced as the app under
- * test dying with `SocketError: other side closed` or `ECONNREFUSED
- * <gateway>:443` — a different test each run, and nothing pointing at
- * ingress.
+ * **This is idempotent, and that is the feature.** Adding a certificate
+ * mutates {@link IngressTables.certByHost} and nothing else: the
+ * terminator reads that table on every handshake, so the new hostname is
+ * served by the next connection and no established connection is
+ * disturbed. The listener this replaced had to be stopped and re-served
+ * for each new certificate, and every such swap severed the idle
+ * kept-alive connections it was carrying — see
+ * `harness/tls-terminator.ts` for the measurements and the history.
  *
- * Overlapping the two needs SO_REUSEPORT on both sockets ({@link
- * bindIngressServer} sets it unconditionally for that reason), after which
- * the old listener is drained with `stop(false)` so requests in flight
- * finish. During the overlap the kernel may hand a new connection to either
- * socket, which is safe: the only hostname the two disagree about is the one
- * being added, and it does not resolve until the caller writes the names
- * registry after this returns.
+ * The route `Map` is the persistent module object, so the plaintext
+ * server closes over the same table and later route additions need no
+ * restart either.
  *
- * The route Map is the persistent module object, so the new listener closes
- * over the same table and later route additions need no rebind at all.
+ * The loopback port is ephemeral rather than fixed so it can never
+ * collide with a fake's declared port; it is bound on 127.0.0.1, which no
+ * container can reach (containers arrive at the bridge gateway), so the
+ * only way in is through the terminator.
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-function rebindHttpsListener(Bun: any): void {
-  // The *same* table object the previous listener closed over: the new
-  // listener must serve the routes bound since, and any bound later.
+async function ensureHttpsIngress(Bun: any): Promise<void> {
+  if (INGRESS_HTTPS_SERVERS.has(INGRESS_HTTPS_PORT)) return;
+  if (!HTTPS_INGRESS_STARTING) {
+    // Cleared on both outcomes: once bound, the check above short-circuits
+    // every later caller, and a failed bind must be retryable rather than
+    // remembered as a rejection for the life of the project.
+    HTTPS_INGRESS_STARTING = startHttpsIngress(Bun).finally(() => {
+      HTTPS_INGRESS_STARTING = undefined;
+    });
+  }
+  return HTTPS_INGRESS_STARTING;
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+async function startHttpsIngress(Bun: any): Promise<void> {
   const routes = routesFor(INGRESS, INGRESS_HTTPS_PORT);
-  const old = INGRESS_HTTPS_SERVERS.get(INGRESS_HTTPS_PORT);
-  const server = bindIngressServer(
-    Bun,
-    INGRESS_HTTPS_PORT,
-    routes,
-    `https :${INGRESS_HTTPS_PORT}`,
-    tlsEntriesFromCerts(),
-  );
-  if (old) {
+  // `proto` is passed rather than inferred: this listener speaks plain
+  // HTTP, but everything reaching it arrived over TLS, so X-Forwarded-Proto
+  // must say https.
+  const plain = bindIngressServer(Bun, 0, routes, `https :${INGRESS_HTTPS_PORT}`, {
+    proto: "https",
+    hostname: "127.0.0.1",
+  });
+  INGRESS_HTTPS_SERVERS.set(INGRESS_HTTPS_PORT, plain);
+  try {
+    HTTPS_TERMINATOR = await startTlsTerminator({
+      port: INGRESS_HTTPS_PORT,
+      hostname: "0.0.0.0",
+      upstreamPort: plain.port,
+      certFor: (serverName) => {
+        if (serverName === undefined) return INGRESS.certByHost.values().next().value;
+        const name = selectCertName(INGRESS.certByHost.keys(), serverName.toLowerCase());
+        return name === undefined ? undefined : INGRESS.certByHost.get(name);
+      },
+    });
+  } catch (err) {
+    // Leave nothing half-built: a registered plaintext server with no
+    // terminator would make `planBind` believe :443 is up.
+    INGRESS_HTTPS_SERVERS.delete(INGRESS_HTTPS_PORT);
     try {
-      // Graceful: stop accepting, let in-flight requests finish. The old
-      // server stays alive until they do, which is the point — a long
-      // upload through ingress must not be collateral damage of another
-      // service being provisioned.
-      old.stop(false);
-      // Bun 1.3.14 landmine: a request arriving later on one of the old
-      // server's kept-alive connections dispatches into freed state and can
-      // segfault the daemon (see {@link DRAINING_INGRESS}). Mark it so any
-      // response it still serves closes its connection, and force-close the
-      // stragglers once in-flight work has had a fair window to finish.
-      DRAINING_INGRESS.add(old);
-      const graceTimer = setTimeout(() => {
-        try {
-          old.stop(true);
-        } catch {
-          /* already fully stopped */
-        }
-      }, REBIND_DRAIN_GRACE_MS);
-      // Don't let the grace timer keep the process alive on shutdown.
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      (graceTimer as any).unref?.();
-    } catch (err) {
-      // eslint-disable-next-line no-console
-      console.warn("[ingress] failed to drain the previous https listener:", err);
+      plain.stop(true);
+    } catch {
+      /* already gone */
     }
+    throw err;
   }
-  INGRESS_HTTPS_SERVERS.set(INGRESS_HTTPS_PORT, server);
   // eslint-disable-next-line no-console
-  console.log(`[ingress] https :${INGRESS_HTTPS_PORT} for ${[...routes.keys()].join(", ")}`);
+  console.log(
+    `[ingress] https :${INGRESS_HTTPS_PORT} (tls terminator -> :${plain.port}) for ${[...routes.keys()].join(", ")}`,
+  );
 }
 
 /**
@@ -2258,9 +2251,12 @@ async function bindRuntimeTls(hostname: string, service: string, port: number):
     );
   }
   if (plan.needsCert) {
+    // The terminator reads this table on every handshake, so the leaf is
+    // live the moment it lands — no listener is touched, and no connection
+    // already on :443 notices.
     INGRESS.certByHost.set(host, await generateHostCert(host, [host]));
   }
-  if (plan.needsHttpsRebind) rebindHttpsListener(Bun);
+  if (plan.needsHttpsListener) await ensureHttpsIngress(Bun);
 
   // Resolve the hostname to the daemon gateway (where :443/:80 listen).
   // A wildcard can only live in the resolver's suffix table.
@@ -2312,22 +2308,27 @@ function bindIngressServer(
   port: number,
   byHost: Map<string, Route>,
   listenerLabel: string,
-  tlsEntries?: Array<{ cert: string; key: string; serverName: string }>,
+  // Every ingress listener speaks plain HTTP now — the one behind :443
+  // sits under the TLS terminator. So the scheme a client actually used
+  // can no longer be inferred from the socket and is declared instead; it
+  // stamps X-Forwarded-Proto, which upstreams that build absolute URLs or
+  // redirect depend on.
+  serve: { proto: "http" | "https"; hostname?: string } = { proto: "http" },
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
 ): any {
-  // A TLS listener terminates https; everything else is plain http. Used
-  // to stamp X-Forwarded-Proto so upstreams that build absolute URLs or
-  // redirect see the scheme the client actually used, not our http hop.
-  const proto = tlsEntries ? "https" : "http";
+  const proto = serve.proto;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const opts: Record<string, any> = {
     port,
-    hostname: "0.0.0.0",
-    // SO_REUSEPORT on every ingress listener, so a replacement can be bound
-    // while the old one is still serving. That overlap is the only way to
-    // add an SNI certificate without a gap — see {@link rebindHttpsListener}
-    // — and it only works if *both* sockets opt in: a second plain bind
-    // fails with "Is port 443 in use?".
+    hostname: serve.hostname ?? "0.0.0.0",
+    // SO_REUSEPORT on every ingress listener. It was introduced so a
+    // replacement :443 could overlap the original during a certificate
+    // swap; the terminator removed that swap, and this stays for the
+    // remaining case — the `/load` teardown force-closes and rebinds, and
+    // a socket the kernel has not finished releasing would otherwise fail
+    // the next bind with "Is port 443 in use?". Note the flip side, which
+    // {@link stopIngressServers} depends on: a half-alive listener shares
+    // the port silently instead of colliding loudly.
     reusePort: true,
     // Bun.serve defaults to a 10s idleTimeout, which kills any proxied
     // request whose upstream takes >10s to produce bytes — under parallel
@@ -2340,9 +2341,7 @@ function bindIngressServer(
     idleTimeout: 0,
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     fetch: (req: Request, server: any): Response | Promise<Response> =>
-      dispatchIngress(req, server, byHost, listenerLabel, proto).then((res) =>
-        withDrainClose(server, res),
-      ),
+      dispatchIngress(req, server, byHost, listenerLabel, proto),
     websocket: {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       async open(ws: any) {
@@ -2412,7 +2411,6 @@ function bindIngressServer(
       },
     },
   };
-  if (tlsEntries) opts.tls = tlsEntries;
   return Bun.serve(opts);
 }
 
@@ -2489,6 +2487,27 @@ async function dispatchIngress(
   return augmentCorsResponse(req, res);
 }
 
+/**
+ * The address to report as the client's in `X-Forwarded-For`.
+ *
+ * On :80 that is simply the socket's peer. On :443 the peer is the TLS
+ * terminator on loopback, so the real address has to be recovered from it:
+ * the terminator keys `source port -> client address` for the life of each
+ * upstream connection, and the source port is what `requestIP` reports
+ * here. Gated on `proto === "https"` so a genuinely loopback caller on :80
+ * can never pick up an unrelated terminator connection's port.
+ *
+ * Falling back to the socket's own address means the worst case is the
+ * pre-terminator answer for a plain HTTP hop, never a wrong tenant.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function ingressClientIp(server: any, req: Request, proto: string): string | undefined {
+  const peer = server.requestIP?.(req) as { address?: string; port?: number } | undefined;
+  if (!peer?.address) return undefined;
+  if (proto !== "https") return peer.address;
+  return HTTPS_TERMINATOR?.clientIpFor(peer.port) ?? peer.address;
+}
+
 /**
  * Reverse-proxy a request to `http://<service>:<port>` on
  * `spectest-net`. Handles plain HTTP/1.1 + 2 and WebSocket upgrades:
@@ -2548,7 +2567,7 @@ async function proxyToService(
   // Standard reverse-proxy provenance headers: the upstream sees the
   // public scheme/host it was reached through and the client's address,
   // even though we rewrite Host below to the service-net name.
-  const clientIp = server.requestIP?.(req)?.address as string | undefined;
+  const clientIp = ingressClientIp(server, req, proto);
   const priorXff = req.headers.get("x-forwarded-for");
   const xff = clientIp ? (priorXff ? `${priorXff}, ${clientIp}` : clientIp) : priorXff;
   if (xff) fwdHeaders.set("x-forwarded-for", xff);

package/src/harness/hostmatch.ts

@@ -68,6 +68,41 @@ export function certCovers(serverNames: Iterable<string>, hostname: string): boo
   return names.some((n) => isWildcard(n) && wildcardCoversHost(n, hostname));
 }
 
+/**
+ * Pick the certificate server name that should serve `hostname`'s TLS
+ * handshake: the exact entry first, then the longest matching wildcard.
+ *
+ * This is the same precedence {@link matchRoute} applies to routes and the
+ * resolver applies to DNS — but over the *certificate* rule, so a wildcard
+ * covers exactly one label ({@link wildcardCoversHost}) rather than any
+ * depth.
+ *
+ * It exists because the SNI lookup is now ours to make. Until the :443
+ * listener stopped being rebuilt for every new certificate, Bun performed
+ * this match internally from the `tls` array it was served with; the
+ * terminator picks a certificate per handshake instead, so the rule has to
+ * be written down — and, being written down, it is testable, which Bun's
+ * internal version never was.
+ */
+export function selectCertName(
+  serverNames: Iterable<string>,
+  hostname: string,
+): string | undefined {
+  const names = [...serverNames];
+  if (names.includes(hostname)) return hostname;
+  let best: string | undefined;
+  let bestLen = -1;
+  for (const name of names) {
+    if (!isWildcard(name) || !wildcardCoversHost(name, hostname)) continue;
+    const len = wildcardSuffix(name).length;
+    if (len > bestLen) {
+      best = name;
+      bestLen = len;
+    }
+  }
+  return best;
+}
+
 /**
  * Pick the route for `host`: exact match first, then the longest matching
  * wildcard suffix.