@specific.dev/spectest 0.69.0 → 0.71.0

package/src/harness/tls-terminator.ts

@@ -0,0 +1,244 @@
+/**
+ * The :443 TLS terminator — one listener, for the life of the harness.
+ *
+ * ## Why this exists
+ *
+ * Ingress certificates are minted *while tests run*: a fake that
+ * provisions a database hands the app a CA-trusted `https://<new-host>/`
+ * endpoint, and that hostname did not exist a second earlier. Bun fixes a
+ * server's TLS configuration at `Bun.serve` time — `reload({ tls })` is
+ * accepted and then ignored (re-measured on Bun 1.4.0: the added
+ * `serverName` still gets the first entry's leaf) — so for as long as
+ * :443 was a `Bun.serve`, every new certificate meant swapping the
+ * listener.
+ *
+ * **No listener swap can be made safe.** Binding the replacement first
+ * under SO_REUSEPORT and draining the original with `stop(false)` saves a
+ * request that is already *in flight*, and that is all it saves: measured
+ * on Bun 1.4.0, `stop(false)` sends FIN to an **idle kept-alive
+ * connection within 1–2 ms**. A client that writes a request into that
+ * socket in the same instant gets `SocketError: other side closed`
+ * (`UND_ERR_SOCKET` under undici), and Node's `fetch` does not retry it.
+ * That is a flake in whatever unrelated test happened to be talking
+ * through the ingress — never in the one that provisioned the service —
+ * and it is why the app under test would die far from any ingress code.
+ *
+ * So the listener must stop being swapped, which means the certificate
+ * must be chosen **per handshake** rather than baked into the socket.
+ * That is what every real server does, and under Bun it is reachable
+ * through `node:tls`'s `SNICallback` (verified working on Bun 1.4.0:
+ * adding an entry to the table is served on the next handshake, with no
+ * restart and no effect on any established connection).
+ *
+ * ## The shape, and why it is a byte pipe
+ *
+ * The terminator does TLS and *nothing else*: it accepts on :443, picks a
+ * leaf by SNI, and pipes the decrypted stream to a plaintext `Bun.serve`
+ * bound on loopback. Everything downstream — routing, fakes,
+ * interceptors, reverse-proxying, and `server.upgrade()` WebSocket
+ * bridging — is the same `Bun.serve` code that already serves :80, run
+ * unchanged. Terminating HTTP here instead (`node:https`, `(req, res)`)
+ * would have meant reimplementing the WebSocket bridge against a
+ * different API, which is exactly the kind of trade this change exists to
+ * avoid: one flake removed, another introduced somewhere subtler.
+ *
+ * Measured cost of the extra hop on this host (Bun 1.4.0, undici client,
+ * 50 MB bodies): ~11 % fewer small requests per second, download
+ * throughput unchanged, upload ~500 MB/s against ~800 MB/s direct. Ingress
+ * traffic in a test VM is nowhere near either bound.
+ *
+ * ## Recovering the client's address
+ *
+ * The plaintext server's peer is the terminator, so `server.requestIP()`
+ * reports loopback and `x-forwarded-for` would lose the caller. The
+ * upstream socket's *source port* is unique among live connections, so the
+ * terminator records `source port → real client address` for the life of
+ * each connection and {@link TlsTerminator.clientIpFor} hands it back.
+ * Verified end to end: a client connecting from 127.0.0.2 is reported as
+ * 127.0.0.2 by the plaintext server behind the pipe.
+ */
+import net from "node:net";
+import tls from "node:tls";
+
+import type { Leaf } from "./ingress-table";
+
+export interface TlsTerminatorOptions {
+  /** Public port to accept TLS on (443 in the harness). */
+  port: number;
+  /** Interface to accept on. */
+  hostname: string;
+  /** Loopback port of the plaintext ingress server to pipe into. */
+  upstreamPort: number;
+  /**
+   * The leaf to serve for an SNI server name, read live on every
+   * handshake — this is what lets a certificate be added with no restart.
+   * `undefined` server name means the client sent no SNI.
+   */
+  certFor(serverName: string | undefined): Leaf | undefined;
+  /** Non-fatal diagnostics (a handshake a client aborted, a pipe error). */
+  onWarning?(message: string, err: unknown): void;
+}
+
+export interface TlsTerminator {
+  /** The port actually bound. */
+  readonly port: number;
+  /**
+   * The real client address behind an upstream connection, keyed by the
+   * source port the plaintext server sees. `undefined` for a port this
+   * terminator did not open.
+   */
+  clientIpFor(upstreamPort: number | undefined): string | undefined;
+  /** Open TLS connections right now (diagnostics and tests). */
+  connectionCount(): number;
+  /** Stop accepting and force-close every connection. */
+  close(): void;
+}
+
+/**
+ * Start the terminator. Resolves once :443 is accepting, so a caller can
+ * register the hostname in DNS immediately afterwards and know the port is
+ * live.
+ */
+/**
+ * Is this the ordinary end of a connection rather than a fault? A peer
+ * that goes away mid-stream, or a half-finished handshake, is routine on a
+ * listener that fronts browsers.
+ */
+function isRoutineDisconnect(err: unknown): boolean {
+  const code = (err as { code?: string } | undefined)?.code;
+  return (
+    code === "ECONNRESET" ||
+    code === "EPIPE" ||
+    code === "ERR_STREAM_PREMATURE_CLOSE" ||
+    code === "ECONNABORTED"
+  );
+}
+
+export function startTlsTerminator(opts: TlsTerminatorOptions): Promise<TlsTerminator> {
+  // Parsing a PEM into a SecureContext is the expensive half of choosing a
+  // certificate, and the table is stable — cache by the certificate's own
+  // bytes so a handshake is a Map lookup. Keyed on the PEM rather than the
+  // hostname so a re-minted leaf for the same name cannot be served stale.
+  const contexts = new Map<string, tls.SecureContext>();
+  const contextFor = (leaf: Leaf): tls.SecureContext => {
+    let ctx = contexts.get(leaf.cert);
+    if (!ctx) {
+      ctx = tls.createSecureContext({ cert: leaf.cert, key: leaf.key });
+      contexts.set(leaf.cert, ctx);
+    }
+    return ctx;
+  };
+
+  const warn = (message: string, err: unknown): void => {
+    // A client hanging up is not a fault, and this listener carries every
+    // browser in the VM: Chromium opens speculative connections and drops
+    // them, a fetch is abandoned when its test ends, and each of those
+    // arrives here as a reset. Logging them would bury the boot log a user
+    // reads with `spectest env logs` in noise that means nothing.
+    if (isRoutineDisconnect(err)) return;
+    if (opts.onWarning) opts.onWarning(message, err);
+    // eslint-disable-next-line no-console
+    else console.warn(`[ingress] ${message}:`, err);
+  };
+
+  /** upstream source port → the address the TLS client connected from. */
+  const clientIpByUpstreamPort = new Map<number, string>();
+  const live = new Set<net.Socket>();
+
+  // A default leaf is only ever used for a client that sends no SNI at
+  // all; `SNICallback` decides every other handshake. This mirrors what
+  // Bun did with the first entry of its `tls` array.
+  const fallback = opts.certFor(undefined);
+
+  const server = tls.createServer(
+    {
+      ...(fallback ? { cert: fallback.cert, key: fallback.key } : {}),
+      SNICallback: (serverName, cb) => {
+        const leaf = opts.certFor(serverName) ?? fallback;
+        // No certificate at all is a handshake we cannot complete. Answer
+        // with the fallback context rather than an error so the client
+        // gets a certificate-mismatch alert it can report, which is the
+        // same thing it saw when Bun served the first entry.
+        cb(null, leaf ? contextFor(leaf) : undefined);
+      },
+    },
+    (client) => {
+      // No idle timeout anywhere on the ingress path: it fronts app
+      // endpoints that legitimately take minutes (a deploy), and a
+      // timeout here would surface as a truncated response with no
+      // explanation.
+      client.setTimeout(0);
+      live.add(client);
+
+      // Nothing reads from `client` until `pipe` below, so a socket that
+      // is written to before the upstream is connected stays paused and
+      // its bytes are buffered rather than dropped — which is the normal
+      // case, since a client sends its first request the instant the
+      // handshake completes.
+      const upstream = net.connect(opts.upstreamPort, "127.0.0.1");
+      upstream.setTimeout(0);
+      upstream.setNoDelay(true);
+      client.setNoDelay(true);
+
+      let recordedPort: number | undefined;
+      upstream.on("connect", () => {
+        recordedPort = upstream.localPort;
+        if (recordedPort !== undefined && client.remoteAddress) {
+          clientIpByUpstreamPort.set(recordedPort, client.remoteAddress);
+        }
+        // `pipe` carries backpressure and the half-close in both
+        // directions, which is what a WebSocket bridge and a streamed
+        // upload each need.
+        client.pipe(upstream);
+        upstream.pipe(client);
+      });
+
+      const teardown = (): void => {
+        if (recordedPort !== undefined) clientIpByUpstreamPort.delete(recordedPort);
+        live.delete(client);
+        client.destroy();
+        upstream.destroy();
+      };
+      client.on("error", (err) => {
+        warn("tls client connection failed", err);
+        teardown();
+      });
+      upstream.on("error", (err) => {
+        warn(`ingress upstream :${opts.upstreamPort} failed`, err);
+        teardown();
+      });
+      client.on("close", teardown);
+      upstream.on("close", teardown);
+    },
+  );
+
+  // A client that aborts mid-handshake, or offers a protocol version we
+  // do not speak, emits this. It must be handled: an unhandled 'error' on
+  // a net server is thrown, and taking the daemon down over one bad
+  // handshake would be a far worse failure than the one being fixed.
+  server.on("tlsClientError", (err) => warn("tls handshake failed", err));
+  server.on("error", (err) => warn(`tls listener :${opts.port} failed`, err));
+
+  return new Promise<TlsTerminator>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(opts.port, opts.hostname, () => {
+      server.removeListener("error", reject);
+      resolve({
+        port: opts.port,
+        clientIpFor: (upstreamPort) =>
+          upstreamPort === undefined ? undefined : clientIpByUpstreamPort.get(upstreamPort),
+        connectionCount: () => live.size,
+        close: () => {
+          try {
+            server.close();
+          } catch (err) {
+            warn("closing the tls listener failed", err);
+          }
+          for (const sock of [...live]) sock.destroy();
+          live.clear();
+          clientIpByUpstreamPort.clear();
+        },
+      });
+    });
+  });
+}

package/src/harness/volume-paths.test.ts

@@ -1,7 +1,6 @@
 import { describe, expect, test } from "bun:test";
 
 import {
-  CACHE_ROOT,
   resolveHostPath,
   sanitizeSegment,
   survivesTeardown,
@@ -68,35 +67,3 @@ describe("resolveHostPath", () => {
   });
 });
 
-describe("cache volumes and delta teardown", () => {
-  /** The whole point of the flag: teardown does `rm -rf /workspace`, so a
-   *  cache volume must be rooted outside it or it does not survive. */
-  test("a cache volume is rooted outside the workspace", () => {
-    const host = resolveHostPath("bun", { target: "/root/.bun/install/cache", cache: true }, WS);
-    expect(host.startsWith(CACHE_ROOT)).toBe(true);
-    expect(host.startsWith(WS)).toBe(false);
-  });
-
-  test("a non-cache volume is inside the workspace, so teardown wipes it", () => {
-    const host = resolveHostPath("db", { target: "/data" }, WS);
-    expect(host.startsWith(`${WS}/`)).toBe(true);
-  });
-
-  test("a shared volume honours the cache flag too", () => {
-    const cached = resolveHostPath("a", { name: "layers", target: "/x", cache: true }, WS);
-    const plain = resolveHostPath("a", { name: "layers", target: "/x" }, WS);
-    expect(cached.startsWith(CACHE_ROOT)).toBe(true);
-    expect(plain.startsWith(`${WS}/`)).toBe(true);
-    expect(cached).not.toBe(plain);
-  });
-
-  test("survivesTeardown agrees with where the path landed", () => {
-    expect(survivesTeardown({ target: "/data" }, "db", WS)).toBe(false);
-    expect(survivesTeardown({ target: "/data", cache: true }, "db", WS)).toBe(true);
-    // An absolute source outside /workspace also survives — worth knowing,
-    // because it is a way to keep state by accident.
-    expect(survivesTeardown({ source: "/mnt/big", target: "/d" }, "db", WS)).toBe(true);
-    // …but an absolute source inside /workspace does not.
-    expect(survivesTeardown({ source: "/workspace/keep", target: "/d" }, "db", WS)).toBe(false);
-  });
-});

package/src/harness/volume-paths.ts

@@ -6,27 +6,18 @@
  * delta-restore teardown**.
  *
  * Teardown wipes `/workspace` to give a restored environment fresh-state
- * semantics. Anything that must survive it therefore has to live outside
- * `/workspace` — and that is exactly what `cache: true` selects, by
- * rooting the directory under `/var/cache/spectest/volumes` instead.
- *
- * The flag is only ever correct for **content-addressed accelerator
- * data**: package stores, layer caches — data whose presence can change
- * how *fast* something runs but never *what* it does. It is wrong for any
- * real state, because a restored environment would then start with a
- * previous run's data and stop being reproducible. (Counter-example worth
- * remembering: `k3s()` deliberately does not cache its containerd store —
- * a fresh cluster over an un-cleanly-killed store wedged the apiserver.)
+ * semantics. Every volume lives inside it: the one cache spectest keeps
+ * across environments is the container store, which is not a volume at
+ * all (CONTAINER_STORE.md).
  */
 
 import path from "node:path";
 
+import { expandServiceToken } from "./file-mounts";
+
 /** Root of the per-environment state tree. Wiped by delta teardown. */
 export const DEFAULT_WORKSPACE = "/workspace";
 
-/** Root of the cache tree. Deliberately outside the workspace. */
-export const CACHE_ROOT = "/var/cache/spectest/volumes";
-
 /** Directory holding named shared volumes, under whichever root applies. */
 export const SHARED_DIR = "_shared";
 
@@ -40,8 +31,6 @@ export interface VolumeSpec {
   /** Path inside the container. Used to derive a directory when neither
    *  `name` nor `source` is given. */
   target: string;
-  /** Survive the delta-restore teardown. Content-addressed data only. */
-  cache?: boolean;
 }
 
 /**
@@ -70,6 +59,13 @@ export function sanitizeSegment(p: string): string {
  *  3. A relative `source`, or nothing at all — under the service's own
  *     directory, derived from `target` when `source` is absent.
  *
+ * `source` honours the `{{SPECTEST_SERVICE}}` token, for the same reason
+ * `files` does: a component cannot know the map key the user will give it,
+ * and an **absolute** source gets no automatic per-service directory. A
+ * component that needs one — a nested runtime keeping its store under
+ * {@link NESTED_STORE_ROOT}, where two of them sharing one directory would
+ * be two daemons on one metadata store — writes the token into the path.
+ *
  * `workspace` is a parameter rather than a module constant so the rule is
  * testable without touching the filesystem.
  */
@@ -79,19 +75,18 @@ export function resolveHostPath(
   workspace: string = DEFAULT_WORKSPACE,
 ): string {
   const stateRoot = [workspace, ".spectest", "volumes"];
+  const source = vol.source === undefined ? undefined : expandServiceToken(vol.source, service);
 
   if (vol.name) {
-    const root = vol.cache ? [CACHE_ROOT, SHARED_DIR] : [...stateRoot, SHARED_DIR];
-    return path.join(...root, sanitizeSegment(vol.name));
+    return path.join(...stateRoot, SHARED_DIR, sanitizeSegment(vol.name));
   }
 
-  // An absolute source is the project's own path; `cache` doesn't apply
-  // because the location was already chosen explicitly.
-  if (vol.source && vol.source.startsWith("/")) return vol.source;
+  // An absolute source is the project's own path, used as is.
+  if (source && source.startsWith("/")) return source;
 
-  const root = vol.cache ? [CACHE_ROOT, service] : [...stateRoot, service];
-  if (vol.source) {
-    return path.join(...root, vol.source.replace(/^\/+/, ""));
+  const root = [...stateRoot, service];
+  if (source) {
+    return path.join(...root, source.replace(/^\/+/, ""));
   }
   return path.join(...root, sanitizeSegment(vol.target));
 }
@@ -99,8 +94,9 @@ export function resolveHostPath(
 /**
  * Does this volume survive a delta-restore teardown?
  *
- * True for cache-flagged volumes and for absolute sources outside the
- * workspace — the two ways a directory ends up beyond `rm -rf /workspace`.
+ * True for a volume on a mounted cache disk, and for an absolute source
+ * outside the workspace — the two ways a directory ends up beyond
+ * `rm -rf /workspace`.
  */
 export function survivesTeardown(
   vol: VolumeSpec,

package/src/index.ts

@@ -1276,15 +1276,6 @@ export interface VolumeMount {
   /** Container path. */
   target: string;
   readOnly?: boolean;
-  /**
-   * Cache volume: the backing dir lives outside the per-env state tree and
-   * survives a delta-restore teardown (which recreates every container,
-   * volume, and the daemon for fresh-state semantics). Reserve this for
-   * content-addressed data whose presence is purely an accelerator — an
-   * image/layer store, a package cache — never for app state: anything in
-   * a cache volume is visible to the "fresh" environment.
-   */
-  cache?: boolean;
 }
 
 export interface FileMount {