@specific.dev/spectest 0.120.1 → 0.122.0

package/dist/components/k3s.js

@@ -1,4 +1,5 @@
 import { isIP } from "node:net";
+import { waitForK3sNodes } from "./k3s-nodes.js";
 import { installPodTrust } from "./k3s-trust.js";
 import { waitForAll } from "../harness/concurrency.js";
 import { CONTENT_BLOBS, REGISTRY_CONTENT, REGISTRY_DATA, REGISTRY_IMAGE, registryDirectory, seedRegistry } from "../harness/registry-seed.js";
@@ -10,7 +11,7 @@ import { existsSync } from "node:fs";
 import { readFile, unlink } from "node:fs/promises";
 import { AppsV1Api, BatchV1Api, CoreV1Api, KubeConfig, KubernetesObjectApi, PatchStrategy, ResponseContext, ServerConfiguration, createConfiguration, loadAllYaml, } from "@kubernetes/client-node";
 import { Observable } from "@kubernetes/client-node/dist/gen/rxjsStub.js";
-import { dnsName, provides, SELF_SERVICE_TOKEN } from "../index.js";
+import { dnsName, provides, serviceGroup, SELF_SERVICE_TOKEN } from "../index.js";
 import { stickyDiskPathSync } from "../harness/sticky-paths.js";
 import { registerStickyKubernetesStore, stickyKubernetesDirectory } from "../harness/sticky-kubernetes.js";
 import { prePullK3sImages } from "../harness/k3s-prepull.js";
@@ -19,16 +20,16 @@ import { deepUnwrap, readTag, wrap } from "../inspect.js";
 import { recorderAnnotate, recorderRemove } from "../recorder.js";
 /** Port the in-cluster registry listens on (plain HTTP). */
 const K3S_REGISTRY_PORT = 5000;
+const JOIN_DIRECTORY = "/etc/spectest-k3s-join";
 /** The cluster's own containerd root, inside the k3s container. */
 const CLUSTER_STORE_TARGET = "/var/lib/rancher/k3s/agent/containerd";
 /** The socket of the throwaway daemon that prepares the store. */
 const PREPARE_SOCKET = "/run/spectest-prepare.sock";
 /**
- * Written before the server starts and removed by the ready probe.
+ * Written before each node starts and removed after cluster setup succeeds.
  *
- * It survives in the store only when a boot never reached a live API
- * server, which is the one thing the sweep below cannot work out for
- * itself. Blacksmith's rule, from the other end: they refuse to commit a
+ * It survives in the store when a node or its image preparation never became
+ * ready, which is the one thing the sweep below cannot work out for itself. Blacksmith's rule, from the other end: they refuse to commit a
  * cache when the integrity check *did not run*, not only when it failed.
  */
 const BOOT_MARKER = ".spectest-boot-incomplete";
@@ -119,6 +120,9 @@ const OVERRIDDEN_EXTRA_ARGS = {
 };
 function assertUsableExtraArgs(extra) {
     for (const arg of extra) {
+        if (/^--(?:node-name|with-node-id|token|token-file|agent-token|agent-token-file|server|disable-agent)(?:[=\s]|$)/.test(arg.trim())) {
+            throw new Error(`k3s(): extraArgs cannot override cluster identity or joining: ${arg}`);
+        }
         const m = /^--disable[=\s]+(.+)$/.exec(arg.trim());
         const addon = m?.[1]?.trim();
         if (!addon)
@@ -521,7 +525,7 @@ const TRAEFIK_DYNAMIC_TLS = `tls:
  * then works for any routed host under the cluster's `ingressDomains`
  * with zero per-Ingress config; the :80 `web` entrypoint is unchanged.
  */
-function buildTraefikManifest(tls) {
+function buildTraefikManifest(tls, node) {
     const args = [
         "        - --entrypoints.web.address=:80",
         ...(tls
@@ -631,6 +635,8 @@ spec:
         app: traefik
     spec:
       serviceAccountName: traefik
+      nodeSelector:
+        kubernetes.io/hostname: ${JSON.stringify(node)}
       hostNetwork: true
       dnsPolicy: Default
       tolerations:
@@ -662,14 +668,10 @@ const HUB_MIRROR = { registry: "docker.io", mirror: "https://mirror.gcr.io", ups
  * `setup` hook. Two jobs:
  *   1. Mirror the cluster's Docker Hub pulls through `mirror.gcr.io`.
  *   2. Trust the in-cluster registry, addressed as `<key>.internal:5000`
- *      (the `{{SPECTEST_SERVICE}}` token is expanded to the cluster's
- *      service key when the file is written). Image *references* use
- *      that peer-reachable name, but containerd pulls via the loopback
- *      endpoint `http://127.0.0.1:5000` — the hostNetwork registry pod
- *      shares the node's netns, so this needs no in-container DNS and
- *      can't be broken by a clobbered `hostnames`.
+ *      using the final group key from service expansion. Every node pulls through
+ *      the server's stable .internal alias, including the server itself.
  */
-function buildRegistriesYaml(registryEnabled, host) {
+function buildRegistriesYaml(registryEnabled, cluster, host) {
     const lines = [
         "mirrors:",
         `  "${HUB_MIRROR.registry}":`,
@@ -678,11 +680,11 @@ function buildRegistriesYaml(registryEnabled, host) {
         `      - "${HUB_MIRROR.upstream}"`,
     ];
     if (registryEnabled) {
-        const host = `{{SPECTEST_SERVICE}}.internal:${K3S_REGISTRY_PORT}`;
-        lines.push(`  "${host}":`, `    endpoint:`, `      - "http://127.0.0.1:${K3S_REGISTRY_PORT}"`, "configs:", 
+        const host = `${cluster}.internal:${K3S_REGISTRY_PORT}`;
+        lines.push(`  "${host}":`, `    endpoint:`, `      - "http://${host}"`, "configs:", 
         // The endpoint is plain HTTP; the config (keyed by endpoint host)
         // makes that explicit and disables any TLS attempt against it.
-        `  "127.0.0.1:${K3S_REGISTRY_PORT}":`, `    tls:`, `      insecure_skip_verify: true`);
+        `  "${host}":`, `    tls:`, `      insecure_skip_verify: true`);
     }
     if (host) {
         // This names the verified HTTPS endpoint itself; do not mirror it to HTTP.
@@ -695,13 +697,13 @@ function buildRegistriesYaml(registryEnabled, host) {
  * In-cluster OCI registry (CNCF `distribution`). `hostNetwork: true`
  * binds the cluster container's `:5000` directly — the same trick
  * Traefik uses — so peer services reach it at `<cluster-key>.internal:5000`
- * (the cluster service's own alias) and the node's own containerd reaches
- * it at `127.0.0.1:5000`. Storage is an `emptyDir` by default. Declared
+ * (the cluster service's own alias). The pod is pinned to the server node
+ * so this address stays valid. Storage is an `emptyDir` by default. Declared
  * publications use a rootfs directory plus a read-only content mount; both
  * variants keep writable registry state outside persistent image caches and
  * capture it with the VM snapshot, isolated per test fork.
  */
-function registryManifest(seeded) {
+function registryManifest(seeded, node) {
     return `apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -719,6 +721,8 @@ spec:
       labels:
         app: spectest-registry
     spec:
+      nodeSelector:
+        kubernetes.io/hostname: ${JSON.stringify(node)}
       hostNetwork: true
       dnsPolicy: Default
       tolerations:
@@ -1164,53 +1168,52 @@ async function waitForJob(deps, name, opts) {
  * entrypoint serving it as the default cert. Those domains are then
  * reachable over HTTPS with a cert the test framework already trusts.
  */
-async function setupK3sCluster(name, helpers, opts) {
+function setupK3sCluster(name, helpers, opts) {
     // The registry can start while Traefik's certificates are being prepared.
     const wait = (deployment) => helpers.waitForRollout(deployment, {
         namespace: "kube-system",
         timeoutMs: 120_000,
         intervalMs: 250,
     });
-    await waitForAll([
-        (async () => {
-            if (!opts.traefik)
-                return;
-            const tlsEnabled = opts.ingressDomains.length > 0 && caPresent();
-            if (tlsEnabled) {
-                const { cert, key } = await issueIngressCert(opts.ingressDomains.map((d) => `*.${d}`));
-                // Apply the cert Secret + dynamic-config ConfigMap before the
-                // Deployment that mounts them. `stringData` lets us hand over plain
-                // PEM; the API server base64-encodes it.
-                await helpers.client.core.createNamespacedSecret({
-                    namespace: "kube-system",
-                    body: {
-                        metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
-                        type: "kubernetes.io/tls",
-                        stringData: { "tls.crt": cert, "tls.key": key },
-                    },
-                });
-                await helpers.client.core.createNamespacedConfigMap({
-                    namespace: "kube-system",
-                    body: {
-                        metadata: {
-                            name: TRAEFIK_DYNAMIC_CONFIGMAP,
-                            namespace: "kube-system",
-                        },
-                        data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
+    const ingress = (async () => {
+        if (!opts.traefik)
+            return;
+        const tlsEnabled = opts.ingressDomains.length > 0 && caPresent();
+        if (tlsEnabled) {
+            const { cert, key } = await issueIngressCert(opts.ingressDomains.map((d) => `*.${d}`));
+            // Apply the cert Secret + dynamic-config ConfigMap before the
+            // Deployment that mounts them. `stringData` lets us hand over plain
+            // PEM; the API server base64-encodes it.
+            await helpers.client.core.createNamespacedSecret({
+                namespace: "kube-system",
+                body: {
+                    metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
+                    type: "kubernetes.io/tls",
+                    stringData: { "tls.crt": cert, "tls.key": key },
+                },
+            });
+            await helpers.client.core.createNamespacedConfigMap({
+                namespace: "kube-system",
+                body: {
+                    metadata: {
+                        name: TRAEFIK_DYNAMIC_CONFIGMAP,
+                        namespace: "kube-system",
                     },
-                });
-            }
-            await helpers.apply(buildTraefikManifest(tlsEnabled));
-            await wait("traefik");
-        })(),
-        (async () => {
-            if (!opts.registry)
-                return;
-            await opts.seed?.();
-            await helpers.apply(registryManifest(!!opts.seed));
-            await wait("spectest-registry");
-        })(),
-    ]);
+                    data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
+                },
+            });
+        }
+        await helpers.apply(buildTraefikManifest(tlsEnabled, name));
+        await wait("traefik");
+    })();
+    const registry = (async () => {
+        if (!opts.registry)
+            return;
+        await opts.seed?.();
+        await helpers.apply(registryManifest(!!opts.seed, name));
+        await wait("spectest-registry");
+    })();
+    return { ingress, registry };
 }
 /**
  * Snapshot of a namespace's state, dumped whenever a wait times out: pod
@@ -1273,7 +1276,7 @@ async function collectDiagnostics(client, namespace) {
     return lines.join("\n");
 }
 /**
- * A ready-to-use single-node Kubernetes cluster (k3s). Drop into
+ * A ready-to-use Kubernetes cluster (k3s). Drop into
  * `environment.services`:
  *
  * ```ts
@@ -1305,8 +1308,8 @@ async function collectDiagnostics(client, namespace) {
  * path regardless, so it stays. It is GA in k8s 1.32, which is what
  * pins `DEFAULT_K3S_VERSION`.
  *
- * Flannel uses the `host-gw` backend: single-node clusters never route
- * pod traffic off the node, so VXLAN encapsulation is pure overhead.
+ * Flannel uses `host-gw`: all node containers share a layer-2 Docker bridge,
+ * so cross-node pod traffic uses direct routes without VXLAN encapsulation.
  */
 /**
  * Default k3s docker image tag.
@@ -1324,6 +1327,12 @@ async function collectDiagnostics(client, namespace) {
  */
 const DEFAULT_K3S_VERSION = "v1.33.5-k3s1";
 export function k3s(opts = {}) {
+    const count = opts.nodes === undefined ? 1 : opts.nodes;
+    if (!Number.isSafeInteger(count) || count < 1)
+        throw new Error("k3s(): nodes must be a positive integer");
+    return createK3sCluster(opts, count);
+}
+function createK3sCluster(opts, count) {
     const version = opts.version ?? DEFAULT_K3S_VERSION;
     const images = kubernetesImages(opts.images ?? []);
     const extra = opts.extraArgs ?? [];
@@ -1348,144 +1357,182 @@ export function k3s(opts = {}) {
     // `/etc/rancher/k3s/registries.yaml` (host-cache mirrors + trust for
     // the in-cluster registry). Seeded via `files` because k3s reads it
     // only at startup, before any setup hook could run.
-    const registriesYaml = buildRegistriesYaml(registryEnabled, registryHost);
     // Pre-pulls read content from the environment's containerd store.
     const sticky = stickyDiskPathSync();
     const contentDirectory = `${sticky ? `${sticky}/containerd` : "/var/lib/containerd"}/${CONTENT_BLOBS}`;
-    const serverArgs = [
-        "k3s",
-        "server",
-        // CoreDNS / pod DNS upstream. Without this, k3s sees only the
-        // loopback 127.0.0.11 (Docker's embedded DNS) in the container's
-        // /etc/resolv.conf, decides no usable nameserver exists, and writes a
-        // fallback `nameserver 8.8.8.8` that CoreDNS then forwards to — so
-        // pods reach the public internet but NOT peer services on
-        // spectest-net (`<svc>.internal`, fakes, service-TLS hosts all
-        // NXDOMAIN). We instead point k3s at the container's default gateway
-        // — the spectest-net bridge gateway, where spectest-resolver binds a
-        // second listener for exactly this. The file is written by the
-        // command wrapper below because the gateway IP is only known at
-        // container start.
-        "--resolv-conf=/run/spectest-resolv.conf",
-        // metrics-server isn't useful in a test cluster.
-        "--disable=metrics-server",
-        // The bundled traefik is disabled because we install our own with
-        // hostNetwork (see setupK3sCluster) — ours is wired to the in-VM CA
-        // for `ingressDomains` TLS and to a fixed IngressClass. Opt out of
-        // ours entirely with `k3s({ traefik: false })`; passing
-        // `--disable=traefik` via `extraArgs` does NOT do that (it only
-        // re-disables the bundled one, which is already off) and is rejected
-        // below rather than silently ignored.
-        "--disable=traefik",
-        // ServiceLB (klipper-lb) stays ENABLED, so `type: LoadBalancer`
-        // Services get an address and work. It was disabled for years
-        // because klipper-lb binds its ports with a CNI portmap hostPort,
-        // and portmap's iptables-nft rules need the `xt_comment` netfilter
-        // match — absent from the stock kernel we used to run on, which made
-        // every LoadBalancer hang at <pending> with no svclb DaemonSet. We
-        // build the guest kernel ourselves now and it carries
-        // CONFIG_NETFILTER_XT_MATCH_COMMENT (see
-        // scripts/local-vms-kernel-additions.config), so the constraint is
-        // gone. local-storage stays enabled too: a controller pod that binds
-        // no host ports.
-        ...(loadBalancerEnabled ? [] : ["--disable=servicelb"]),
-        // Pod CIDR MUST avoid 10.42.0.0/16: that's the spectest-br0 host
-        // bridge subnet, whose gateway 10.42.0.1 fronts the host image caches
-        // (zot :5000-5007, buildkitd :1234). k3s's *default* pod CIDR is also
-        // 10.42.0.0/16 — with --flannel-backend=host-gw, flannel programs that
-        // route into the node's own routing table and gives cni0 the subnet's
-        // .1 (10.42.0.1). That shadows the route to the host gateway, so once
-        // CNI comes up the node can no longer reach 10.42.0.1 and every
-        // subsequent registry pull dies with "connect: connection refused"
-        // (e.g. the in-cluster registry's `registry:2`, applied after the
-        // cluster is up — the airgap-bundled system images pull *before* CNI
-        // and so sneak through). Move pods to 10.44/service to 10.45.
-        "--cluster-cidr=10.44.0.0/16",
-        "--service-cidr=10.45.0.0/16",
-        "--cluster-dns=10.45.0.10",
-        "--flannel-backend=host-gw",
-        "--write-kubeconfig-mode=644",
-        // kube-proxy in nftables mode: native nftables rules, no
-        // xt_comment dependency. Pod→ClusterIP routing works, so
-        // CoreDNS / helm-install / anything-talking-to-the-API works.
-        // GA in k8s 1.32.
-        "--kube-proxy-arg=proxy-mode=nftables",
-        ...extra,
-    ].join(" ");
-    // The service `command` runs under `/bin/sh -c` (see runContainer in
-    // daemon.ts), so derive the bridge gateway from the container's default
-    // route at start time, write it as the k3s resolv-conf, then exec k3s
-    // (exec so it stays the container's main process and signals / the
-    // readyCheck behave exactly as before). `/run` is a tmpfs on this
-    // service, so the file is writable and never persisted into a snapshot.
-    const cmd = "GW=\"$(ip route 2>/dev/null | awk '/^default/{print $3; exit}')\"; " +
-        'if [ -n "$GW" ]; then ' +
-        "printf 'nameserver %s\\noptions ndots:0\\n' \"$GW\" > /run/spectest-resolv.conf; " +
-        "else echo 'spectest: no default gateway found; k3s pod DNS for peer services will not resolve' >&2; " +
-        ": > /run/spectest-resolv.conf; fi; " +
-        // Make the root mount shared. A CSI node plugin bind-mounts volumes
-        // under /var/lib/kubelet and needs that propagation to reach the
-        // kubelet and the workload pod; docker gives a container a private
-        // root, so without this every CSI node plugin fails to publish and
-        // any snapshot/PVC-backed storage test is impossible. `kind` does the
-        // same in its entrypoint for the same reason. Best-effort: on a
-        // kernel/runtime that refuses it, the cluster still boots — only CSI
-        // is affected.
-        "mount --make-rshared / 2>/dev/null || " +
-        "echo 'spectest: could not make / rshared; CSI node plugins may fail to publish volumes' >&2; " +
-        storePrepareSh() +
-        // Sticky and environment-local stores use overlayfs.
-        `exec ${serverArgs} --snapshotter=$SNAP`;
-    // Plain /readyz probe. On a warm zot cache the cluster's images are
-    // already local, so the first boot completes in seconds; the
-    // first-ever boot on a cold-cache host pulls through the mirror and
-    // can take a couple of minutes (covered by readyTimeoutSecs).
-    //
-    // Clearing the boot marker is part of the probe on purpose: "the API
-    // server answers" is exactly the condition that proves this cluster came
-    // up on the store it was given, and it is the only signal the sweep can
-    // read on the next boot. See STORE_SWEEP_SH.
-    const readyCmd = `kubectl get --raw=/readyz >/dev/null 2>&1 && rm -f ${CLUSTER_STORE_TARGET}/${BOOT_MARKER}`;
-    const def = {
-        image: { type: "registry", reference: `rancher/k3s:${version}` },
-        imageBuilds: images.builds,
-        tls: registryHost ? [{ hostname: registryHost, port: K3S_REGISTRY_PORT }] : [],
-        command: cmd,
-        privileged: true,
-        tmpfs: ["/run", "/var/run"],
-        cgroupns: "host",
-        // 80/443 are advisory — peer services and host code reach them via
-        // the k3s container's IP. The component's Traefik binds them directly
-        // (hostNetwork, so it shares the container's netns). 5000 is the
-        // in-cluster registry (hostNetwork pod bound to the container netns),
-        // reached by peers at the cluster's own `<key>.internal:5000` alias.
-        // A `type: LoadBalancer` Service's ports are bound by klipper-lb in
-        // the same netns and are reachable the same way, without appearing
-        // in this list (it's documentation, not a firewall).
-        ports: registryEnabled ? [80, 443, 6443, K3S_REGISTRY_PORT] : [80, 443, 6443],
-        // Sticky disks retain an independent overlayfs store per service/version.
-        volumes: [...(sticky
-                ? [{ source: `${stickyKubernetesDirectory(SELF_SERVICE_TOKEN, version)}/containerd`, target: CLUSTER_STORE_TARGET }]
-                : [{ target: CLUSTER_STORE_TARGET }]),
-            ...(publishBuilds.length ? [
-                { source: registryDirectory(SELF_SERVICE_TOKEN), target: REGISTRY_DATA },
-                { source: contentDirectory, target: REGISTRY_CONTENT, readOnly: true },
+    const nodeNames = (cluster) => Array.from({ length: count }, (_, i) => i === 0 ? cluster : `${cluster}-node-${i + 1}`);
+    const createNode = (cluster, server) => {
+        const registriesYaml = buildRegistriesYaml(registryEnabled, cluster, registryHost);
+        const nodeArgs = [
+            "k3s", server ? "server" : "agent",
+            "--node-name=$(hostname)",
+            // cgroupns=host is required by our nested runtime. Each kubelet must
+            // own a separate tree or its cleanup can remove another node's pods.
+            "--kubelet-arg=cgroup-root=/spectest-$(hostname)",
+            "--kubelet-arg=kubelet-cgroups=/spectest-$(hostname)/k3s",
+            "--kubelet-arg=runtime-cgroups=/spectest-$(hostname)/k3s",
+            `--token-file=${JOIN_DIRECTORY}/token`,
+            ...(server ? [] : [`--server=https://${cluster}.internal:6443`]),
+            // CoreDNS / pod DNS upstream. Without this, k3s sees only the
+            // loopback 127.0.0.11 (Docker's embedded DNS) in the container's
+            // /etc/resolv.conf, decides no usable nameserver exists, and writes a
+            // fallback `nameserver 8.8.8.8` that CoreDNS then forwards to — so
+            // pods reach the public internet but NOT peer services on
+            // spectest-net (`<svc>.internal`, fakes, service-TLS hosts all
+            // NXDOMAIN). We instead point k3s at the container's default gateway
+            // — the spectest-net bridge gateway, where spectest-resolver binds a
+            // second listener for exactly this. The file is written by the
+            // command wrapper below because the gateway IP is only known at
+            // container start.
+            "--resolv-conf=/run/spectest-resolv.conf",
+            // metrics-server isn't useful in a test cluster.
+            ...(server ? [
+                "--disable=metrics-server",
+                // The bundled traefik is disabled because we install our own with
+                // hostNetwork (see setupK3sCluster) — ours is wired to the in-VM CA
+                // for `ingressDomains` TLS and to a fixed IngressClass. Opt out of
+                // ours entirely with `k3s({ traefik: false })`; passing
+                // `--disable=traefik` via `extraArgs` does NOT do that (it only
+                // re-disables the bundled one, which is already off) and is rejected
+                // below rather than silently ignored.
+                "--disable=traefik",
+                // ServiceLB (klipper-lb) stays ENABLED, so `type: LoadBalancer`
+                // Services get an address and work. It was disabled for years
+                // because klipper-lb binds its ports with a CNI portmap hostPort,
+                // and portmap's iptables-nft rules need the `xt_comment` netfilter
+                // match — absent from the stock kernel we used to run on, which made
+                // every LoadBalancer hang at <pending> with no svclb DaemonSet. We
+                // build the guest kernel ourselves now and it carries
+                // CONFIG_NETFILTER_XT_MATCH_COMMENT (see
+                // scripts/local-vms-kernel-additions.config), so the constraint is
+                // gone. local-storage stays enabled too: a controller pod that binds
+                // no host ports.
+                ...(loadBalancerEnabled ? [] : ["--disable=servicelb"]),
+                // Pod CIDR MUST avoid 10.42.0.0/16: that's the spectest-br0 host
+                // bridge subnet, whose gateway 10.42.0.1 fronts the host image caches
+                // (zot :5000-5007, buildkitd :1234). k3s's *default* pod CIDR is also
+                // 10.42.0.0/16 — with --flannel-backend=host-gw, flannel programs that
+                // route into the node's own routing table and gives cni0 the subnet's
+                // .1 (10.42.0.1). That shadows the route to the host gateway, so once
+                // CNI comes up the node can no longer reach 10.42.0.1 and every
+                // subsequent registry pull dies with "connect: connection refused"
+                // (e.g. the in-cluster registry's `registry:2`, applied after the
+                // cluster is up — the airgap-bundled system images pull *before* CNI
+                // and so sneak through). Move pods to 10.44/service to 10.45.
+                "--cluster-cidr=10.44.0.0/16",
+                "--service-cidr=10.45.0.0/16",
+                "--cluster-dns=10.45.0.10",
+                "--flannel-backend=host-gw",
+                "--write-kubeconfig-mode=644",
+                `--tls-san=${cluster}.internal`,
             ] : []),
-        ],
-        files: [
-            ...(registriesYaml ? [{ path: "/etc/rancher/k3s/registries.yaml", content: registriesYaml }] : []),
-        ],
-        readyCheck: {
-            type: "exec",
-            command: readyCmd,
-            timeoutSecs: opts.readyTimeoutSecs ?? 120,
+            // kube-proxy in nftables mode: native nftables rules, no
+            // xt_comment dependency. Pod→ClusterIP routing works, so
+            // CoreDNS / helm-install / anything-talking-to-the-API works.
+            // GA in k8s 1.32.
+            "--kube-proxy-arg=proxy-mode=nftables",
+            ...(server ? extra : []),
+        ].join(" ");
+        // The service `command` runs under `/bin/sh -c` (see runContainer in
+        // daemon.ts), so derive the bridge gateway from the container's default
+        // route at start time, write it as the k3s resolv-conf, then exec k3s
+        // (exec so it stays the container's main process and signals / the
+        // readyCheck behave exactly as before). `/run` is a tmpfs on this
+        // service; its contents are captured with the VM's memory on snapshot.
+        const cmd = "GW=\"$(ip route 2>/dev/null | awk '/^default/{print $3; exit}')\"; " +
+            'if [ -n "$GW" ]; then ' +
+            "printf 'nameserver %s\\noptions ndots:0\\n' \"$GW\" > /run/spectest-resolv.conf; " +
+            "else echo 'spectest: no default gateway found; k3s pod DNS for peer services will not resolve' >&2; " +
+            ": > /run/spectest-resolv.conf; fi; " +
+            // Make the root mount shared. A CSI node plugin bind-mounts volumes
+            // under /var/lib/kubelet and needs that propagation to reach the
+            // kubelet and the workload pod; docker gives a container a private
+            // root, so without this every CSI node plugin fails to publish and
+            // any snapshot/PVC-backed storage test is impossible. `kind` does the
+            // same in its entrypoint for the same reason. Best-effort: on a
+            // kernel/runtime that refuses it, the cluster still boots — only CSI
+            // is affected.
+            "mount --make-rshared / 2>/dev/null || " +
+            "echo 'spectest: could not make / rshared; CSI node plugins may fail to publish volumes' >&2; " +
+            // Kubelet validates its cgroup root before it creates kubepods. Delegate
+            // the controllers on this empty parent, keeping k3s processes in a leaf.
+            'CG="spectest-$(hostname)"; ' +
+            'if [ -f /sys/fs/cgroup/cgroup.controllers ]; then ' +
+            'mkdir -p "/sys/fs/cgroup/$CG/k3s" || exit 1; ' +
+            'sed "s/[^ ][^ ]*/+&/g" "/sys/fs/cgroup/$CG/cgroup.controllers" > "/sys/fs/cgroup/$CG/cgroup.subtree_control" || exit 1; ' +
+            'else for C in /sys/fs/cgroup/*; do [ ! -d "$C" ] || mkdir -p "$C/$CG/k3s" || exit 1; done; fi; ' +
+            storePrepareSh() +
+            (server
+                ? `if [ ! -s ${JOIN_DIRECTORY}/token ]; then (umask 077; cat /proc/sys/kernel/random/uuid > ${JOIN_DIRECTORY}/token); fi; `
+                : `while [ ! -s ${JOIN_DIRECTORY}/token ]; do sleep 0.1; done; `) +
+            // Sticky and environment-local stores use overlayfs.
+            `exec ${nodeArgs} --snapshotter=$SNAP`;
+        // Plain /readyz probe. On a warm zot cache the cluster's images are
+        // already local, so the first boot completes in seconds; the
+        // first-ever boot on a cold-cache host pulls through the mirror and
+        // can take a couple of minutes (covered by readyTimeoutSecs).
+        //
+        // Per-node image-store boot markers are cleared after group setup has
+        // checked all Nodes and finished loading images, not at API readiness.
+        const readyCmd = "kubectl get --raw=/readyz >/dev/null 2>&1";
+        const def = {
+            image: { type: "registry", reference: `rancher/k3s:${version}` },
+            imageBuilds: server ? images.builds : [],
+            tls: server && registryHost ? [{ hostname: registryHost, port: K3S_REGISTRY_PORT }] : [],
+            command: cmd,
+            privileged: true,
+            tmpfs: ["/run", "/var/run"],
+            cgroupns: "host",
+            // 80/443 are advisory — peer services and host code reach them via
+            // the k3s container's IP. The component's Traefik binds them directly
+            // (hostNetwork, so it shares the container's netns). 5000 is the
+            // in-cluster registry (hostNetwork pod bound to the container netns),
+            // reached by peers at the cluster's own `<key>.internal:5000` alias.
+            // A `type: LoadBalancer` Service's ports are bound by klipper-lb in
+            // the same netns and are reachable the same way, without appearing
+            // in this list (it's documentation, not a firewall).
+            ports: server ? (registryEnabled ? [80, 443, 6443, K3S_REGISTRY_PORT] : [80, 443, 6443]) : [],
+            // Sticky disks retain an independent overlayfs store per service/version.
+            // Every node has its own writable containerd root.
+            volumes: [{ name: `spectest-k3s-${cluster}-join`, target: JOIN_DIRECTORY }, ...(sticky
+                    ? [{ source: `${stickyKubernetesDirectory(SELF_SERVICE_TOKEN, version)}/containerd`, target: CLUSTER_STORE_TARGET }]
+                    : [{ target: CLUSTER_STORE_TARGET }]),
+                ...(server && publishBuilds.length ? [
+                    { source: registryDirectory(SELF_SERVICE_TOKEN), target: REGISTRY_DATA },
+                    { source: contentDirectory, target: REGISTRY_CONTENT, readOnly: true },
+                ] : []),
+            ],
+            files: [
+                ...(registriesYaml ? [{ path: "/etc/rancher/k3s/registries.yaml", content: registriesYaml }] : []),
+            ],
+            // Agents start before the server and wait for its join token. The group
+            // setup below gates readiness on every actual Kubernetes Node, including
+            // the server, rather than treating container startup as cluster readiness.
+            readyCheck: server ? {
+                type: "exec",
+                command: readyCmd,
+                timeoutSecs: opts.readyTimeoutSecs ?? 120,
+            } : undefined,
+        };
+        return server && ingressDomains.length
+            ? provides(def, ingressDomains.map(domain => dnsName(`*.${domain}`, { service: SELF_SERVICE_TOKEN })))
+            : def;
+    };
+    return serviceGroup({
+        primary: "server",
+        services: g => {
+            const parts = { server: createNode(g.name, true) };
+            for (let i = 2; i <= count; i++)
+                parts[`node-${i}`] = createNode(g.name, false);
+            return parts;
         },
         setup: async (ctx) => {
             const { name, helpers, exec } = ctx;
             const runtime = componentRuntime(ctx);
+            const nodes = nodeNames(name);
             if (images.builds.length && !runtime)
                 throw new Error(`k3s(${name}): prepared image inputs are missing`);
-            const infrastructure = (async () => {
+            const admission = (async () => {
                 if (runtime?.caBundle && caPresent()) {
                     const { cert, key } = await issueIngressCert([runtime.gateway]);
                     await installPodTrust({ name, gateway: runtime.gateway, cert, key,
@@ -1526,31 +1573,35 @@ export function k3s(opts = {}) {
                         await new Promise(resolve => setTimeout(resolve, 100));
                     }
                 }
-                await setupK3sCluster(name, helpers, {
-                    registry: registryEnabled,
-                    seed: publishBuilds.length ? () => seedRegistry(name, contentDirectory, runtime.images.filter(image => isPublished(image.reference))) : undefined,
-                    ingressDomains,
-                    traefik: traefikEnabled,
-                });
             })();
+            const infrastructure = admission.then(() => setupK3sCluster(name, helpers, {
+                registry: registryEnabled,
+                seed: publishBuilds.length ? () => seedRegistry(name, contentDirectory, runtime.images.filter(image => isPublished(image.reference))) : undefined,
+                ingressDomains,
+                traefik: traefikEnabled,
+            }));
             await waitForAll([
-                runtime ? loadK3sBuildImages(name, runtime.images, runtime.gateway, exec) : Promise.resolve(),
-                prePullK3sImages(name, images.pulls.filter(ref => !isPublished(ref)), exec),
+                infrastructure.then(tasks => tasks.ingress),
+                waitForK3sNodes(name, nodes, exec, (opts.readyTimeoutSecs ?? 120) * 1000),
+                ...nodes.map(node => runtime ? loadK3sBuildImages(node, runtime.images, runtime.gateway, exec) : Promise.resolve()),
+                ...nodes.map(node => prePullK3sImages(node, images.pulls.filter(ref => !isPublished(ref) && !ref.startsWith(`${name}.internal:${K3S_REGISTRY_PORT}/`)), exec)),
                 (async () => {
                     // Registry-local references must not race registry startup/publication.
-                    await infrastructure;
-                    await prePullK3sImages(name, images.pulls.filter(isPublished), exec);
+                    await infrastructure.then(tasks => tasks.registry);
+                    await waitForAll(nodes.map(node => prePullK3sImages(node, images.pulls.filter(ref => isPublished(ref) || ref.startsWith(`${name}.internal:${K3S_REGISTRY_PORT}/`)), exec)));
                 })(),
             ]);
-            if (sticky)
-                await registerStickyKubernetesStore(name, version);
+            await waitForAll(nodes.map(async (node) => {
+                const cleared = await exec(node, ["rm", "-f", `${CLUSTER_STORE_TARGET}/${BOOT_MARKER}`]);
+                if (cleared.exitCode !== 0)
+                    throw new Error(`k3s(${node}): failed to mark image store ready: ${cleared.stderr}`);
+                if (sticky)
+                    await registerStickyKubernetesStore(node, version);
+            }));
         },
         helpers: async ({ name, exec, poll }) => {
-            // Read the cluster's kubeconfig and address the API server by its
-            // auto-assigned `<name>.internal` hostname on spectest-net. TLS
-            // verification is off (see the K3sHelpers docstring), so the
-            // server's cert SAN list doesn't need to include the .internal
-            // name.
+            // Read the server's kubeconfig and address its stable .internal alias.
+            // This endpoint is also used by the agents joining the cluster.
             const kcRead = await exec(name, ["cat", "/etc/rancher/k3s/k3s.yaml"]);
             if (kcRead.exitCode !== 0) {
                 throw new Error(`k3s(${name}): failed to read kubeconfig from container: ${kcRead.stderr.trim()}`);
@@ -1592,6 +1643,7 @@ export function k3s(opts = {}) {
                 poll,
             };
             return {
+                nodes: nodeNames(name),
                 kubeconfig,
                 client,
                 apply: (manifest, applyOpts) => applyManifest(deps, manifest, applyOpts),
@@ -1601,12 +1653,5 @@ export function k3s(opts = {}) {
                 list: (listOpts) => objects.list(listOpts.apiVersion, listOpts.kind, listOpts.namespace, undefined, undefined, undefined, listOpts.fieldSelector, listOpts.labelSelector, listOpts.limit),
             };
         },
-    };
-    // Wildcard ingress domains → a dnsName(`*.<domain>`, { service: self })
-    // each, attached via provides(). SELF_SERVICE_TOKEN resolves to this
-    // service's key at load time (the component can't know it here). The
-    // resolver then points every host under the domain at the cluster.
-    if (ingressDomains.length === 0)
-        return def;
-    return provides(def, ingressDomains.map((domain) => dnsName(`*.${domain}`, { service: SELF_SERVICE_TOKEN })));
+    });
 }