@specific.dev/spectest 0.69.0 → 0.70.0

package/src/harness/tls-terminator.ts

@@ -0,0 +1,244 @@
+/**
+ * The :443 TLS terminator — one listener, for the life of the harness.
+ *
+ * ## Why this exists
+ *
+ * Ingress certificates are minted *while tests run*: a fake that
+ * provisions a database hands the app a CA-trusted `https://<new-host>/`
+ * endpoint, and that hostname did not exist a second earlier. Bun fixes a
+ * server's TLS configuration at `Bun.serve` time — `reload({ tls })` is
+ * accepted and then ignored (re-measured on Bun 1.4.0: the added
+ * `serverName` still gets the first entry's leaf) — so for as long as
+ * :443 was a `Bun.serve`, every new certificate meant swapping the
+ * listener.
+ *
+ * **No listener swap can be made safe.** Binding the replacement first
+ * under SO_REUSEPORT and draining the original with `stop(false)` saves a
+ * request that is already *in flight*, and that is all it saves: measured
+ * on Bun 1.4.0, `stop(false)` sends FIN to an **idle kept-alive
+ * connection within 1–2 ms**. A client that writes a request into that
+ * socket in the same instant gets `SocketError: other side closed`
+ * (`UND_ERR_SOCKET` under undici), and Node's `fetch` does not retry it.
+ * That is a flake in whatever unrelated test happened to be talking
+ * through the ingress — never in the one that provisioned the service —
+ * and it is why the app under test would die far from any ingress code.
+ *
+ * So the listener must stop being swapped, which means the certificate
+ * must be chosen **per handshake** rather than baked into the socket.
+ * That is what every real server does, and under Bun it is reachable
+ * through `node:tls`'s `SNICallback` (verified working on Bun 1.4.0:
+ * adding an entry to the table is served on the next handshake, with no
+ * restart and no effect on any established connection).
+ *
+ * ## The shape, and why it is a byte pipe
+ *
+ * The terminator does TLS and *nothing else*: it accepts on :443, picks a
+ * leaf by SNI, and pipes the decrypted stream to a plaintext `Bun.serve`
+ * bound on loopback. Everything downstream — routing, fakes,
+ * interceptors, reverse-proxying, and `server.upgrade()` WebSocket
+ * bridging — is the same `Bun.serve` code that already serves :80, run
+ * unchanged. Terminating HTTP here instead (`node:https`, `(req, res)`)
+ * would have meant reimplementing the WebSocket bridge against a
+ * different API, which is exactly the kind of trade this change exists to
+ * avoid: one flake removed, another introduced somewhere subtler.
+ *
+ * Measured cost of the extra hop on this host (Bun 1.4.0, undici client,
+ * 50 MB bodies): ~11 % fewer small requests per second, download
+ * throughput unchanged, upload ~500 MB/s against ~800 MB/s direct. Ingress
+ * traffic in a test VM is nowhere near either bound.
+ *
+ * ## Recovering the client's address
+ *
+ * The plaintext server's peer is the terminator, so `server.requestIP()`
+ * reports loopback and `x-forwarded-for` would lose the caller. The
+ * upstream socket's *source port* is unique among live connections, so the
+ * terminator records `source port → real client address` for the life of
+ * each connection and {@link TlsTerminator.clientIpFor} hands it back.
+ * Verified end to end: a client connecting from 127.0.0.2 is reported as
+ * 127.0.0.2 by the plaintext server behind the pipe.
+ */
+import net from "node:net";
+import tls from "node:tls";
+
+import type { Leaf } from "./ingress-table";
+
+export interface TlsTerminatorOptions {
+  /** Public port to accept TLS on (443 in the harness). */
+  port: number;
+  /** Interface to accept on. */
+  hostname: string;
+  /** Loopback port of the plaintext ingress server to pipe into. */
+  upstreamPort: number;
+  /**
+   * The leaf to serve for an SNI server name, read live on every
+   * handshake — this is what lets a certificate be added with no restart.
+   * `undefined` server name means the client sent no SNI.
+   */
+  certFor(serverName: string | undefined): Leaf | undefined;
+  /** Non-fatal diagnostics (a handshake a client aborted, a pipe error). */
+  onWarning?(message: string, err: unknown): void;
+}
+
+export interface TlsTerminator {
+  /** The port actually bound. */
+  readonly port: number;
+  /**
+   * The real client address behind an upstream connection, keyed by the
+   * source port the plaintext server sees. `undefined` for a port this
+   * terminator did not open.
+   */
+  clientIpFor(upstreamPort: number | undefined): string | undefined;
+  /** Open TLS connections right now (diagnostics and tests). */
+  connectionCount(): number;
+  /** Stop accepting and force-close every connection. */
+  close(): void;
+}
+
+/**
+ * Start the terminator. Resolves once :443 is accepting, so a caller can
+ * register the hostname in DNS immediately afterwards and know the port is
+ * live.
+ */
+/**
+ * Is this the ordinary end of a connection rather than a fault? A peer
+ * that goes away mid-stream, or a half-finished handshake, is routine on a
+ * listener that fronts browsers.
+ */
+function isRoutineDisconnect(err: unknown): boolean {
+  const code = (err as { code?: string } | undefined)?.code;
+  return (
+    code === "ECONNRESET" ||
+    code === "EPIPE" ||
+    code === "ERR_STREAM_PREMATURE_CLOSE" ||
+    code === "ECONNABORTED"
+  );
+}
+
+export function startTlsTerminator(opts: TlsTerminatorOptions): Promise<TlsTerminator> {
+  // Parsing a PEM into a SecureContext is the expensive half of choosing a
+  // certificate, and the table is stable — cache by the certificate's own
+  // bytes so a handshake is a Map lookup. Keyed on the PEM rather than the
+  // hostname so a re-minted leaf for the same name cannot be served stale.
+  const contexts = new Map<string, tls.SecureContext>();
+  const contextFor = (leaf: Leaf): tls.SecureContext => {
+    let ctx = contexts.get(leaf.cert);
+    if (!ctx) {
+      ctx = tls.createSecureContext({ cert: leaf.cert, key: leaf.key });
+      contexts.set(leaf.cert, ctx);
+    }
+    return ctx;
+  };
+
+  const warn = (message: string, err: unknown): void => {
+    // A client hanging up is not a fault, and this listener carries every
+    // browser in the VM: Chromium opens speculative connections and drops
+    // them, a fetch is abandoned when its test ends, and each of those
+    // arrives here as a reset. Logging them would bury the boot log a user
+    // reads with `spectest env logs` in noise that means nothing.
+    if (isRoutineDisconnect(err)) return;
+    if (opts.onWarning) opts.onWarning(message, err);
+    // eslint-disable-next-line no-console
+    else console.warn(`[ingress] ${message}:`, err);
+  };
+
+  /** upstream source port → the address the TLS client connected from. */
+  const clientIpByUpstreamPort = new Map<number, string>();
+  const live = new Set<net.Socket>();
+
+  // A default leaf is only ever used for a client that sends no SNI at
+  // all; `SNICallback` decides every other handshake. This mirrors what
+  // Bun did with the first entry of its `tls` array.
+  const fallback = opts.certFor(undefined);
+
+  const server = tls.createServer(
+    {
+      ...(fallback ? { cert: fallback.cert, key: fallback.key } : {}),
+      SNICallback: (serverName, cb) => {
+        const leaf = opts.certFor(serverName) ?? fallback;
+        // No certificate at all is a handshake we cannot complete. Answer
+        // with the fallback context rather than an error so the client
+        // gets a certificate-mismatch alert it can report, which is the
+        // same thing it saw when Bun served the first entry.
+        cb(null, leaf ? contextFor(leaf) : undefined);
+      },
+    },
+    (client) => {
+      // No idle timeout anywhere on the ingress path: it fronts app
+      // endpoints that legitimately take minutes (a deploy), and a
+      // timeout here would surface as a truncated response with no
+      // explanation.
+      client.setTimeout(0);
+      live.add(client);
+
+      // Nothing reads from `client` until `pipe` below, so a socket that
+      // is written to before the upstream is connected stays paused and
+      // its bytes are buffered rather than dropped — which is the normal
+      // case, since a client sends its first request the instant the
+      // handshake completes.
+      const upstream = net.connect(opts.upstreamPort, "127.0.0.1");
+      upstream.setTimeout(0);
+      upstream.setNoDelay(true);
+      client.setNoDelay(true);
+
+      let recordedPort: number | undefined;
+      upstream.on("connect", () => {
+        recordedPort = upstream.localPort;
+        if (recordedPort !== undefined && client.remoteAddress) {
+          clientIpByUpstreamPort.set(recordedPort, client.remoteAddress);
+        }
+        // `pipe` carries backpressure and the half-close in both
+        // directions, which is what a WebSocket bridge and a streamed
+        // upload each need.
+        client.pipe(upstream);
+        upstream.pipe(client);
+      });
+
+      const teardown = (): void => {
+        if (recordedPort !== undefined) clientIpByUpstreamPort.delete(recordedPort);
+        live.delete(client);
+        client.destroy();
+        upstream.destroy();
+      };
+      client.on("error", (err) => {
+        warn("tls client connection failed", err);
+        teardown();
+      });
+      upstream.on("error", (err) => {
+        warn(`ingress upstream :${opts.upstreamPort} failed`, err);
+        teardown();
+      });
+      client.on("close", teardown);
+      upstream.on("close", teardown);
+    },
+  );
+
+  // A client that aborts mid-handshake, or offers a protocol version we
+  // do not speak, emits this. It must be handled: an unhandled 'error' on
+  // a net server is thrown, and taking the daemon down over one bad
+  // handshake would be a far worse failure than the one being fixed.
+  server.on("tlsClientError", (err) => warn("tls handshake failed", err));
+  server.on("error", (err) => warn(`tls listener :${opts.port} failed`, err));
+
+  return new Promise<TlsTerminator>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(opts.port, opts.hostname, () => {
+      server.removeListener("error", reject);
+      resolve({
+        port: opts.port,
+        clientIpFor: (upstreamPort) =>
+          upstreamPort === undefined ? undefined : clientIpByUpstreamPort.get(upstreamPort),
+        connectionCount: () => live.size,
+        close: () => {
+          try {
+            server.close();
+          } catch (err) {
+            warn("closing the tls listener failed", err);
+          }
+          for (const sock of [...live]) sock.destroy();
+          live.clear();
+          clientIpByUpstreamPort.clear();
+        },
+      });
+    });
+  });
+}