@specific.dev/spectest 0.69.0 → 0.71.0

package/src/harness/hostmatch.ts

@@ -68,6 +68,41 @@ export function certCovers(serverNames: Iterable<string>, hostname: string): boo
   return names.some((n) => isWildcard(n) && wildcardCoversHost(n, hostname));
 }
 
+/**
+ * Pick the certificate server name that should serve `hostname`'s TLS
+ * handshake: the exact entry first, then the longest matching wildcard.
+ *
+ * This is the same precedence {@link matchRoute} applies to routes and the
+ * resolver applies to DNS — but over the *certificate* rule, so a wildcard
+ * covers exactly one label ({@link wildcardCoversHost}) rather than any
+ * depth.
+ *
+ * It exists because the SNI lookup is now ours to make. Until the :443
+ * listener stopped being rebuilt for every new certificate, Bun performed
+ * this match internally from the `tls` array it was served with; the
+ * terminator picks a certificate per handshake instead, so the rule has to
+ * be written down — and, being written down, it is testable, which Bun's
+ * internal version never was.
+ */
+export function selectCertName(
+  serverNames: Iterable<string>,
+  hostname: string,
+): string | undefined {
+  const names = [...serverNames];
+  if (names.includes(hostname)) return hostname;
+  let best: string | undefined;
+  let bestLen = -1;
+  for (const name of names) {
+    if (!isWildcard(name) || !wildcardCoversHost(name, hostname)) continue;
+    const len = wildcardSuffix(name).length;
+    if (len > bestLen) {
+      best = name;
+      bestLen = len;
+    }
+  }
+  return best;
+}
+
 /**
  * Pick the route for `host`: exact match first, then the longest matching
  * wildcard suffix.

package/src/harness/ingress-rebind.test.ts

@@ -1,17 +1,25 @@
 /**
- * The :443 rebind must not be collateral damage for traffic already on the
- * listener.
+ * What a listener swap does and does not cost.
  *
- * Adding an SNI certificate needs a second `Bun.serve` — `reload()` accepts
- * a new `tls` option and goes on serving the old certificates. The question
- * this file pins down is the *order*: bind the replacement first and drain
- * the old one, rather than stopping the old one and leaving a gap.
+ * **:443 no longer swaps.** Adding an SNI certificate used to need a
+ * second `Bun.serve` (`reload()` accepts a new `tls` option and goes on
+ * serving the old certificates — still true on Bun 1.4.0), and this file
+ * pinned down the *order*: bind the replacement first and drain the old
+ * one, rather than stopping first and leaving a gap. That ordering was
+ * necessary and insufficient — it saves a request already in flight and
+ * severs every idle kept-alive connection, which is the failure that kept
+ * reaching production. `tls-terminator.ts` chooses the certificate per
+ * handshake now, so the certificate path swaps nothing; `tls-terminator.
+ * test.ts` covers the invariant that replaced this one.
  *
- * These drive real sockets rather than a model of them, because the bug was
+ * These stay because the swap itself stays, on the remaining path: the
+ * `/load` teardown force-closes every listener and rebinds the next
+ * project's. That is what `reusePort` is still for, and the two properties
+ * below — a replacement can bind alongside the original, and the port is
+ * never refused across the swap — are what make it safe.
+ *
+ * They drive real sockets rather than a model of them, because the bug was
  * in socket behaviour: a model would have agreed with the broken code.
- * They use plain HTTP — the TLS part is Bun's SNI table, measured
- * separately; what is asserted here is that a listener swap costs no
- * connection, which is the part that was wrong.
  */
 import { afterEach, describe, expect, test } from "bun:test";
 

package/src/harness/ingress-table.test.ts

@@ -78,21 +78,36 @@ describe("route tables", () => {
 });
 
 describe("planBind", () => {
-  test("a brand-new hostname needs a cert and a rebind", () => {
+  test("a brand-new hostname needs a cert", () => {
     const t = emptyTables();
     expect(planBind(t, "new.test", { httpsListening: true })).toEqual({
       needsCert: true,
-      needsHttpsRebind: true,
+      needsHttpsListener: false,
     });
   });
 
-  test("the first bind rebinds even when a cert already covers the name", () => {
+  /**
+   * The rule the terminator bought, and the regression this file is the
+   * cheap half of: minting a certificate must NOT imply touching the
+   * listener. It used to (`needsCert || !httpsListening`), and every such
+   * restart severed the idle kept-alive connections :443 was carrying —
+   * failing an unrelated test with `SocketError: other side closed`.
+   */
+  test("a new certificate never asks for the listener to be restarted", () => {
+    const t = emptyTables();
+    t.certByHost.set("app.test", leaf("app"));
+    expect(planBind(t, "brand-new.test", { httpsListening: true })).toEqual({
+      needsCert: true,
+      needsHttpsListener: false,
+    });
+  });
+
+  test("the listener is asked for when nothing is serving :443 yet", () => {
     const t = emptyTables();
     t.certByHost.set("app.test", leaf("app"));
-    // Nothing is listening yet, so :443 has to be served regardless.
     expect(planBind(t, "app.test", { httpsListening: false })).toEqual({
       needsCert: false,
-      needsHttpsRebind: true,
+      needsHttpsListener: true,
     });
   });
 
@@ -101,19 +116,19 @@ describe("planBind", () => {
     t.certByHost.set("app.test", leaf("app"));
     expect(planBind(t, "app.test", { httpsListening: true })).toEqual({
       needsCert: false,
-      needsHttpsRebind: false,
+      needsHttpsListener: false,
     });
   });
 
   /** The case that makes runtime provisioning cheap: a component claims
    *  `*.db.test` up front, then a fake mints endpoints under it in a loop.
-   *  Rebinding per iteration would restart the listener each time. */
+   *  Each one is then a route entry and not even a keygen. */
   test("a hostname under an existing wildcard cert needs neither", () => {
     const t = emptyTables();
     t.certByHost.set("*.db.test", leaf("wild"));
     expect(planBind(t, "tenant-1.db.test", { httpsListening: true })).toEqual({
       needsCert: false,
-      needsHttpsRebind: false,
+      needsHttpsListener: false,
     });
   });
 

package/src/harness/ingress-table.ts

@@ -32,13 +32,14 @@
  * routes added afterwards go nowhere. {@link routesFor} is the only way to
  * reach a table so that this can't be done by accident.
  *
- * **A rebind is not free, so it must be earned.** Bun fixes a server's TLS
- * config at `Bun.serve` time — `reload` will not add an SNI entry — so a
- * genuinely new certificate means stopping and re-serving :443. It's cheap
- * (~1 ms) but it is a real interruption of live traffic, and a hostname
- * already covered by an existing exact or wildcard cert needs nothing but a
- * route entry. {@link planBind} is that judgement, separated from the
- * mutation so it can be tested without a network stack.
+ * **Adding a certificate no longer touches the listener.** :443 is
+ * terminated by `harness/tls-terminator.ts`, which chooses a leaf per TLS
+ * handshake, so a new certificate is a `Map` entry and nothing more. It
+ * used to mean stopping and re-serving :443, and that swap severed every
+ * idle kept-alive connection the listener was carrying — an unrelated
+ * test's request, failing with `SocketError: other side closed`. The only
+ * thing {@link planBind} still has to say is whether the listener exists
+ * yet.
  */
 
 import { certCovers, isWildcard, wildcardSuffix } from "./hostmatch";
@@ -102,19 +103,22 @@ export function routesFor<F>(tables: IngressTables<F>, port: number): Map<string
 export interface BindPlan {
   /** No existing cert covers the hostname, so a leaf must be minted. */
   needsCert: boolean;
-  /** The :443 listener must be stopped and re-served. */
-  needsHttpsRebind: boolean;
+  /** The :443 listener does not exist yet and must be started. */
+  needsHttpsListener: boolean;
 }
 
 /**
  * Decide what binding `hostname` for TLS costs.
  *
- * A rebind is needed when a new certificate is going into the SNI table,
- * or when :443 isn't listening yet. It is deliberately *not* needed for a
- * hostname an existing wildcard already covers — the common case when a
- * component claims a whole domain up front and services appear under it
- * later, and the reason a fake can mint endpoints in a loop without
- * restarting the listener once per iteration.
+ * Minting is still earned — a hostname an existing exact or wildcard cert
+ * already covers needs nothing but a route, which is why a component can
+ * claim a whole domain up front and have services appear under it for
+ * free.
+ *
+ * Starting the listener, on the other hand, is now a once-per-process
+ * event: it is needed when :443 is not up yet, and never again. A new
+ * certificate does **not** imply it. That is the whole point of the
+ * terminator — see the module header.
  */
 export function planBind<F>(
   tables: IngressTables<F>,
@@ -122,7 +126,7 @@ export function planBind<F>(
   opts: { httpsListening: boolean },
 ): BindPlan {
   const needsCert = !certCovers(tables.certByHost.keys(), hostname);
-  return { needsCert, needsHttpsRebind: needsCert || !opts.httpsListening };
+  return { needsCert, needsHttpsListener: !opts.httpsListening };
 }
 
 /**
@@ -141,15 +145,22 @@ export function bindRoute<F>(tables: IngressTables<F>, hostname: string, route:
  * Drop `hostname`'s routes, so it 404s.
  *
  * The certificate is deliberately left in the SNI table: it is harmless
- * without a route, and removing it would force an otherwise unnecessary
- * :443 rebind at exactly the moment a service is going away.
+ * without a route, and a leaf nobody asks for costs one unused `Map`
+ * entry. (Before the terminator, removing it would also have forced a
+ * :443 rebind at exactly the moment a service was going away.)
  */
 export function unbindRoute<F>(tables: IngressTables<F>, hostname: string): void {
   tables.routesByPort.get(INGRESS_HTTP_PORT)?.delete(hostname);
   tables.routesByPort.get(INGRESS_HTTPS_PORT)?.delete(hostname);
 }
 
-/** The SNI entries for `Bun.serve`'s TLS config. */
+/**
+ * The SNI table flattened to entries.
+ *
+ * This was `Bun.serve`'s `tls` array. The terminator selects per handshake
+ * instead ({@link import("./hostmatch").selectCertName}), so this survives
+ * as the readable projection of the table — for diagnostics and tests.
+ */
 export function certEntries<F>(
   tables: IngressTables<F>,
 ): Array<{ cert: string; key: string; serverName: string }> {

package/src/harness/tls-terminator.test.ts

@@ -0,0 +1,240 @@
+/**
+ * Adding a certificate must cost no connection.
+ *
+ * This is the file the previous shape of :443 could not have: when the
+ * listener was a `Bun.serve` whose `tls` array was fixed at serve time,
+ * every new certificate meant a swap, and the best a swap could do was
+ * save requests that were already in flight (`ingress-rebind.test.ts`).
+ * The failure that kept reaching production was the other half — an
+ * **idle kept-alive connection**, severed by `stop(false)` within 1-2 ms,
+ * surfacing in an unrelated test as `SocketError: other side closed`.
+ *
+ * So these drive real TLS sockets rather than a model of them: the whole
+ * bug was in socket behaviour, and a model would have agreed with the
+ * broken code. The client is Node's own `node:tls`/HTTP-over-socket so
+ * that keep-alive is under the test's control rather than a pool's.
+ */
+import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import net from "node:net";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import tls from "node:tls";
+
+import type { Leaf } from "./ingress-table";
+import { selectCertName } from "./hostmatch";
+import { startTlsTerminator, type TlsTerminator } from "./tls-terminator";
+
+// ── a throwaway CA and leaves, so the client can verify for real ──────────
+let dir: string;
+const leaves = new Map<string, Leaf>();
+let caPem: string;
+
+function openssl(args: string[], input?: string): void {
+  const r = spawnSync("openssl", args, { cwd: dir, input, encoding: "utf8" });
+  if (r.status !== 0) throw new Error(`openssl ${args.join(" ")}: ${r.stderr}`);
+}
+
+function mintLeaf(name: string, san: string): Leaf {
+  const file = name.replace("*", "wild");
+  openssl(["req", "-newkey", "rsa:2048", "-keyout", `${file}.key`, "-out", `${file}.csr`,
+    "-nodes", "-subj", `/CN=${name}`]);
+  openssl(["x509", "-req", "-in", `${file}.csr`, "-CA", "ca.crt", "-CAkey", "ca.key",
+    "-CAcreateserial", "-out", `${file}.crt`, "-days", "2", "-extfile", "ext.cnf"]);
+  // -extfile is rewritten per call; write it first.
+  return {
+    cert: readFileSync(join(dir, `${file}.crt`), "utf8"),
+    key: readFileSync(join(dir, `${file}.key`), "utf8"),
+  };
+}
+
+beforeAll(() => {
+  dir = mkdtempSync(join(tmpdir(), "spectest-tls-"));
+  openssl(["req", "-x509", "-newkey", "rsa:2048", "-keyout", "ca.key", "-out", "ca.crt",
+    "-days", "2", "-nodes", "-subj", "/CN=spectest-test-ca"]);
+  caPem = readFileSync(join(dir, "ca.crt"), "utf8");
+  for (const [name, san] of [
+    ["app.test", "DNS:app.test"],
+    ["late.test", "DNS:late.test"],
+    ["*.db.test", "DNS:*.db.test"],
+  ] as const) {
+    require("node:fs").writeFileSync(join(dir, "ext.cnf"), `subjectAltName=${san}\n`);
+    leaves.set(name, mintLeaf(name, san));
+  }
+});
+afterAll(() => rmSync(dir, { recursive: true, force: true }));
+
+// ── the ingress shape: a plaintext Bun.serve behind the terminator ────────
+const PORT = 19701;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+let plain: any;
+let term: TlsTerminator;
+/** The live SNI table, exactly as the daemon mutates `INGRESS.certByHost`. */
+let certByHost: Map<string, Leaf>;
+/** What the plaintext server reported as the client's address, per request. */
+let seenClientIp: string | undefined;
+
+beforeAll(async () => {
+  certByHost = new Map([["app.test", leaves.get("app.test")!]]);
+  plain = Bun.serve({
+    port: 0,
+    hostname: "127.0.0.1",
+    idleTimeout: 0,
+    fetch(req, server) {
+      const peer = server.requestIP(req);
+      seenClientIp = term.clientIpFor(peer?.port) ?? peer?.address;
+      return new Response(`served ${new URL(req.url).hostname}`);
+    },
+  });
+  term = await startTlsTerminator({
+    port: PORT,
+    hostname: "127.0.0.1",
+    upstreamPort: plain.port,
+    certFor: (serverName) => {
+      if (serverName === undefined) return certByHost.values().next().value;
+      const name = selectCertName(certByHost.keys(), serverName.toLowerCase());
+      return name === undefined ? undefined : certByHost.get(name);
+    },
+    onWarning: () => {
+      /* a client hanging up mid-handshake is a case under test */
+    },
+  });
+});
+afterAll(() => {
+  term?.close();
+  plain?.stop(true);
+});
+
+const sockets: tls.TLSSocket[] = [];
+afterEach(() => {
+  for (const s of sockets.splice(0)) s.destroy();
+});
+
+/** A kept-alive TLS connection, handshaken and left open. */
+async function connect(servername: string, localAddress?: string): Promise<tls.TLSSocket> {
+  const sock = tls.connect({
+    host: "127.0.0.1", port: PORT, servername, ca: caPem,
+    ...(localAddress ? { localAddress } : {}),
+  });
+  sockets.push(sock);
+  await new Promise<void>((resolve, reject) => {
+    sock.once("secureConnect", () => resolve());
+    sock.once("error", reject);
+  });
+  return sock;
+}
+
+/** One keep-alive request on an existing connection. Rejects if the peer
+ *  closed it instead of answering — which is exactly the reported bug. */
+function request(sock: tls.TLSSocket, host: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+    let buf = "";
+    const onData = (d: Buffer): void => {
+      buf += d.toString();
+      if (buf.includes("\r\n\r\n")) {
+        sock.off("data", onData);
+        resolve(buf);
+      }
+    };
+    sock.on("data", onData);
+    sock.once("close", () => reject(new Error("other side closed")));
+    sock.once("error", (err) => reject(err));
+    sock.write(`GET / HTTP/1.1\r\nHost: ${host}\r\nConnection: keep-alive\r\n\r\n`);
+    setTimeout(() => reject(new Error("timed out")), 4000);
+  });
+}
+
+describe("adding a certificate", () => {
+  /**
+   * THE regression. An idle pooled connection is what a client actually
+   * holds between polls, and a listener swap severed it: the next request
+   * written into that socket came back `SocketError: other side closed`,
+   * unretried, failing whichever test happened to be deploying. Adding a
+   * leaf to the live table must be invisible to it.
+   */
+  test("an idle kept-alive connection survives it", async () => {
+    const sock = await connect("app.test");
+    expect(await request(sock, "app.test")).toContain("served app.test");
+
+    // …the connection now sits idle in the client's pool, as a CLI polling
+    // a deploy would leave it, while an unrelated service is provisioned.
+    certByHost.set("late.test", leaves.get("late.test")!);
+    await Bun.sleep(50);
+
+    expect(await request(sock, "app.test")).toContain("served app.test");
+  });
+
+  test("many certificates later, the same connection is still good", async () => {
+    const sock = await connect("app.test");
+    await request(sock, "app.test");
+    for (let i = 0; i < 10; i++) {
+      certByHost.set(`svc-${i}.db.test`, leaves.get("*.db.test")!);
+      await Bun.sleep(5);
+    }
+    expect(await request(sock, "app.test")).toContain("served app.test");
+  });
+
+  /** The other side of the same coin: the point of adding a leaf is that
+   *  it serves the next handshake, with no restart to wait for. */
+  test("the new hostname is served on the next handshake", async () => {
+    certByHost.set("late.test", leaves.get("late.test")!);
+    const sock = await connect("late.test");
+    expect((sock.getPeerCertificate().subject as { CN: string }).CN).toBe("late.test");
+    expect(await request(sock, "late.test")).toContain("served late.test");
+  });
+
+  /** A wildcard leaf serves one label under it — the same RFC 6125 rule
+   *  `certCovers` plans against, now enforced at the handshake. */
+  test("a wildcard leaf serves a name one label under it", async () => {
+    certByHost.set("*.db.test", leaves.get("*.db.test")!);
+    const sock = await connect("tenant-7.db.test");
+    expect((sock.getPeerCertificate().subject as { CN: string }).CN).toBe("*.db.test");
+  });
+});
+
+describe("the pipe", () => {
+  /** `x-forwarded-for` is built from this. The plaintext server's peer is
+   *  the terminator, so without the port map every proxied request would
+   *  tell the app under test its caller was 127.0.0.1. */
+  test("reports the real client address, not the loopback hop", async () => {
+    seenClientIp = undefined;
+    const sock = await connect("app.test", "127.0.0.2");
+    await request(sock, "app.test");
+    expect(seenClientIp).toBe("127.0.0.2");
+  });
+
+  /** A handshake the terminator cannot complete must cost the process
+   *  nothing: an unhandled 'error' on a node server is thrown, and taking
+   *  the daemon down over one bad client would be worse than the bug. */
+  test("a client that speaks no TLS does not disturb the listener", async () => {
+    const raw = net.connect(PORT, "127.0.0.1");
+    await new Promise<void>((r) => raw.on("connect", () => r()));
+    raw.write("this is not a ClientHello\r\n\r\n");
+    await Bun.sleep(100);
+    raw.destroy();
+
+    const sock = await connect("app.test");
+    expect(await request(sock, "app.test")).toContain("served app.test");
+  });
+
+  /** A test VM is short-lived, but a fake that provisions in a loop opens
+   *  a connection per probe — the terminator must not accumulate them. */
+  test("connections are released when they close", async () => {
+    // Earlier tests' sockets are destroyed in afterEach; let those close
+    // events land so this measures only what it opens itself.
+    const settle = async (): Promise<void> => {
+      for (let i = 0; i < 100 && term.connectionCount() > 0; i++) await Bun.sleep(20);
+    };
+    await settle();
+    expect(term.connectionCount()).toBe(0);
+
+    const sock = await connect("app.test");
+    await request(sock, "app.test");
+    expect(term.connectionCount()).toBe(1);
+
+    sock.destroy();
+    await settle();
+    expect(term.connectionCount()).toBe(0);
+  });
+});