@specific.dev/spectest 0.120.1 → 0.122.0

package/src/components/k3s.ts

@@ -1,4 +1,5 @@
 import { isIP } from "node:net";
+import { waitForK3sNodes } from "./k3s-nodes.js";
 import { installPodTrust } from "./k3s-trust.js";
 import { waitForAll } from "../harness/concurrency.js";
 import { CONTENT_BLOBS, REGISTRY_CONTENT, REGISTRY_DATA, REGISTRY_IMAGE, registryDirectory, seedRegistry } from "../harness/registry-seed.js";
@@ -34,12 +35,12 @@ import {
 import { Observable } from "@kubernetes/client-node/dist/gen/rxjsStub.js";
 
 import type {
-  ServiceDefinition,
+  ServiceConfig,
   ServiceHelpersContext,
   ServiceSetupContext,
   SpectestContext,
 } from "../index.js";
-import { dnsName, provides, SELF_SERVICE_TOKEN } from "../index.js";
+import { dnsName, provides, serviceGroup, SELF_SERVICE_TOKEN } from "../index.js";
 import { stickyDiskPathSync } from "../harness/sticky-paths.js";
 import { registerStickyKubernetesStore, stickyKubernetesDirectory } from "../harness/sticky-kubernetes.js";
 import { prePullK3sImages } from "../harness/k3s-prepull.js";
@@ -66,6 +67,9 @@ export interface K3sRegistryOptions {
 /** Kubernetes pods receive read-only environment CA mounts during admission.
  * Image bytes and digests stay unchanged; explicit pod trust settings win. */
 export interface K3sOptions {
+  /** Total schedulable nodes, including the server node. Defaults to 1.
+   * Additional nodes run k3s agents in separate containers inside the same VM. */
+  nodes?: number;
   /** Images available to kubelet before project setup. Use registry strings,
    * or Dockerfile builds with a full `reference` matching your manifests. Builds
    * support path/content, context, exclude, target and buildArgs, and run in
@@ -184,6 +188,7 @@ export interface K3sOptions {
 
 /** Port the in-cluster registry listens on (plain HTTP). */
 const K3S_REGISTRY_PORT = 5000;
+const JOIN_DIRECTORY = "/etc/spectest-k3s-join";
 
 /** The cluster's own containerd root, inside the k3s container. */
 const CLUSTER_STORE_TARGET = "/var/lib/rancher/k3s/agent/containerd";
@@ -191,11 +196,10 @@ const CLUSTER_STORE_TARGET = "/var/lib/rancher/k3s/agent/containerd";
 const PREPARE_SOCKET = "/run/spectest-prepare.sock";
 
 /**
- * Written before the server starts and removed by the ready probe.
+ * Written before each node starts and removed after cluster setup succeeds.
  *
- * It survives in the store only when a boot never reached a live API
- * server, which is the one thing the sweep below cannot work out for
- * itself. Blacksmith's rule, from the other end: they refuse to commit a
+ * It survives in the store when a node or its image preparation never became
+ * ready, which is the one thing the sweep below cannot work out for itself. Blacksmith's rule, from the other end: they refuse to commit a
  * cache when the integrity check *did not run*, not only when it failed.
  */
 const BOOT_MARKER = ".spectest-boot-incomplete";
@@ -295,6 +299,9 @@ const OVERRIDDEN_EXTRA_ARGS: Record<string, string> = {
 
 function assertUsableExtraArgs(extra: readonly string[]): void {
   for (const arg of extra) {
+    if (/^--(?:node-name|with-node-id|token|token-file|agent-token|agent-token-file|server|disable-agent)(?:[=\s]|$)/.test(arg.trim())) {
+      throw new Error(`k3s(): extraArgs cannot override cluster identity or joining: ${arg}`);
+    }
     const m = /^--disable[=\s]+(.+)$/.exec(arg.trim());
     const addon = m?.[1]?.trim();
     if (!addon) continue;
@@ -475,6 +482,8 @@ export interface LogsOptions {
 
 /** Helpers a `k3s(...)` service exposes on `ctx.svc.<name>`. */
 export interface K3sHelpers {
+  /** Kubernetes node names, also their Spectest container/service names. */
+  nodes: readonly string[];
   /**
    * Fully-loaded `KubeConfig`. The cluster server URL is rewritten to
    * `https://<service-name>.internal:6443`, the auto-assigned DNS name
@@ -1023,7 +1032,7 @@ const TRAEFIK_DYNAMIC_TLS = `tls:
  * then works for any routed host under the cluster's `ingressDomains`
  * with zero per-Ingress config; the :80 `web` entrypoint is unchanged.
  */
-function buildTraefikManifest(tls: boolean): string {
+function buildTraefikManifest(tls: boolean, node: string): string {
   const args = [
     "        - --entrypoints.web.address=:80",
     ...(tls
@@ -1133,6 +1142,8 @@ spec:
         app: traefik
     spec:
       serviceAccountName: traefik
+      nodeSelector:
+        kubernetes.io/hostname: ${JSON.stringify(node)}
       hostNetwork: true
       dnsPolicy: Default
       tolerations:
@@ -1166,14 +1177,10 @@ const HUB_MIRROR = { registry: "docker.io", mirror: "https://mirror.gcr.io", ups
  * `setup` hook. Two jobs:
  *   1. Mirror the cluster's Docker Hub pulls through `mirror.gcr.io`.
  *   2. Trust the in-cluster registry, addressed as `<key>.internal:5000`
- *      (the `{{SPECTEST_SERVICE}}` token is expanded to the cluster's
- *      service key when the file is written). Image *references* use
- *      that peer-reachable name, but containerd pulls via the loopback
- *      endpoint `http://127.0.0.1:5000` — the hostNetwork registry pod
- *      shares the node's netns, so this needs no in-container DNS and
- *      can't be broken by a clobbered `hostnames`.
+ *      using the final group key from service expansion. Every node pulls through
+ *      the server's stable .internal alias, including the server itself.
  */
-function buildRegistriesYaml(registryEnabled: boolean, host?: string): string | null {
+function buildRegistriesYaml(registryEnabled: boolean, cluster: string, host?: string): string | null {
   const lines: string[] = [
     "mirrors:",
     `  "${HUB_MIRROR.registry}":`,
@@ -1182,15 +1189,15 @@ function buildRegistriesYaml(registryEnabled: boolean, host?: string): string |
     `      - "${HUB_MIRROR.upstream}"`,
   ];
   if (registryEnabled) {
-    const host = `{{SPECTEST_SERVICE}}.internal:${K3S_REGISTRY_PORT}`;
+    const host = `${cluster}.internal:${K3S_REGISTRY_PORT}`;
     lines.push(
       `  "${host}":`,
       `    endpoint:`,
-      `      - "http://127.0.0.1:${K3S_REGISTRY_PORT}"`,
+      `      - "http://${host}"`,
       "configs:",
       // The endpoint is plain HTTP; the config (keyed by endpoint host)
       // makes that explicit and disables any TLS attempt against it.
-      `  "127.0.0.1:${K3S_REGISTRY_PORT}":`,
+      `  "${host}":`,
       `    tls:`,
       `      insecure_skip_verify: true`,
     );
@@ -1207,13 +1214,13 @@ function buildRegistriesYaml(registryEnabled: boolean, host?: string): string |
  * In-cluster OCI registry (CNCF `distribution`). `hostNetwork: true`
  * binds the cluster container's `:5000` directly — the same trick
  * Traefik uses — so peer services reach it at `<cluster-key>.internal:5000`
- * (the cluster service's own alias) and the node's own containerd reaches
- * it at `127.0.0.1:5000`. Storage is an `emptyDir` by default. Declared
+ * (the cluster service's own alias). The pod is pinned to the server node
+ * so this address stays valid. Storage is an `emptyDir` by default. Declared
  * publications use a rootfs directory plus a read-only content mount; both
  * variants keep writable registry state outside persistent image caches and
  * capture it with the VM snapshot, isolated per test fork.
  */
-function registryManifest(seeded: boolean): string {
+function registryManifest(seeded: boolean, node: string): string {
   return `apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -1231,6 +1238,8 @@ spec:
       labels:
         app: spectest-registry
     spec:
+      nodeSelector:
+        kubernetes.io/hostname: ${JSON.stringify(node)}
       hostNetwork: true
       dnsPolicy: Default
       tolerations:
@@ -1782,11 +1791,11 @@ async function waitForJob(
  * entrypoint serving it as the default cert. Those domains are then
  * reachable over HTTPS with a cert the test framework already trusts.
  */
-async function setupK3sCluster(
+function setupK3sCluster(
   name: string,
   helpers: K3sHelpers,
   opts: { registry: boolean; ingressDomains: string[]; traefik: boolean; seed?: () => Promise<void> },
-): Promise<void> {
+): { ingress: Promise<void>; registry: Promise<void> } {
   // The registry can start while Traefik's certificates are being prepared.
   const wait = (deployment: string): Promise<void> =>
     helpers.waitForRollout(deployment, {
@@ -1794,46 +1803,45 @@ async function setupK3sCluster(
       timeoutMs: 120_000,
       intervalMs: 250,
     });
-  await waitForAll([
-    (async () => {
-      if (!opts.traefik) return;
-      const tlsEnabled = opts.ingressDomains.length > 0 && caPresent();
-      if (tlsEnabled) {
-        const { cert, key } = await issueIngressCert(
-          opts.ingressDomains.map((d) => `*.${d}`),
-        );
-        // Apply the cert Secret + dynamic-config ConfigMap before the
-        // Deployment that mounts them. `stringData` lets us hand over plain
-        // PEM; the API server base64-encodes it.
-        await helpers.client.core.createNamespacedSecret({
-          namespace: "kube-system",
-          body: {
-            metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
-            type: "kubernetes.io/tls",
-            stringData: { "tls.crt": cert, "tls.key": key },
-          },
-        });
-        await helpers.client.core.createNamespacedConfigMap({
-          namespace: "kube-system",
-          body: {
-            metadata: {
-              name: TRAEFIK_DYNAMIC_CONFIGMAP,
-              namespace: "kube-system",
-            },
-            data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
+  const ingress = (async () => {
+    if (!opts.traefik) return;
+    const tlsEnabled = opts.ingressDomains.length > 0 && caPresent();
+    if (tlsEnabled) {
+      const { cert, key } = await issueIngressCert(
+        opts.ingressDomains.map((d) => `*.${d}`),
+      );
+      // Apply the cert Secret + dynamic-config ConfigMap before the
+      // Deployment that mounts them. `stringData` lets us hand over plain
+      // PEM; the API server base64-encodes it.
+      await helpers.client.core.createNamespacedSecret({
+        namespace: "kube-system",
+        body: {
+          metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
+          type: "kubernetes.io/tls",
+          stringData: { "tls.crt": cert, "tls.key": key },
+        },
+      });
+      await helpers.client.core.createNamespacedConfigMap({
+        namespace: "kube-system",
+        body: {
+          metadata: {
+            name: TRAEFIK_DYNAMIC_CONFIGMAP,
+            namespace: "kube-system",
           },
-        });
-      }
-      await helpers.apply(buildTraefikManifest(tlsEnabled));
-      await wait("traefik");
-    })(),
-    (async () => {
-      if (!opts.registry) return;
-      await opts.seed?.();
-      await helpers.apply(registryManifest(!!opts.seed));
-      await wait("spectest-registry");
-    })(),
-  ]);
+          data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
+        },
+      });
+    }
+    await helpers.apply(buildTraefikManifest(tlsEnabled, name));
+    await wait("traefik");
+  })();
+  const registry = (async () => {
+    if (!opts.registry) return;
+    await opts.seed?.();
+    await helpers.apply(registryManifest(!!opts.seed, name));
+    await wait("spectest-registry");
+  })();
+  return { ingress, registry };
 }
 
 /**
@@ -1901,7 +1909,7 @@ async function collectDiagnostics(
 }
 
 /**
- * A ready-to-use single-node Kubernetes cluster (k3s). Drop into
+ * A ready-to-use Kubernetes cluster (k3s). Drop into
  * `environment.services`:
  *
  * ```ts
@@ -1933,8 +1941,8 @@ async function collectDiagnostics(
  * path regardless, so it stays. It is GA in k8s 1.32, which is what
  * pins `DEFAULT_K3S_VERSION`.
  *
- * Flannel uses the `host-gw` backend: single-node clusters never route
- * pod traffic off the node, so VXLAN encapsulation is pure overhead.
+ * Flannel uses `host-gw`: all node containers share a layer-2 Docker bridge,
+ * so cross-node pod traffic uses direct routes without VXLAN encapsulation.
  */
 /**
  * Default k3s docker image tag.
@@ -1953,6 +1961,12 @@ async function collectDiagnostics(
 const DEFAULT_K3S_VERSION = "v1.33.5-k3s1";
 
 export function k3s(opts: K3sOptions = {}) {
+  const count = opts.nodes === undefined ? 1 : opts.nodes;
+  if (!Number.isSafeInteger(count) || count < 1) throw new Error("k3s(): nodes must be a positive integer");
+  return createK3sCluster(opts, count);
+}
+
+function createK3sCluster(opts: K3sOptions, count: number) {
   const version = opts.version ?? DEFAULT_K3S_VERSION;
   const images = kubernetesImages(opts.images ?? []);
   const extra = opts.extraArgs ?? [];
@@ -1977,144 +1991,182 @@ export function k3s(opts: K3sOptions = {}) {
   // `/etc/rancher/k3s/registries.yaml` (host-cache mirrors + trust for
   // the in-cluster registry). Seeded via `files` because k3s reads it
   // only at startup, before any setup hook could run.
-  const registriesYaml = buildRegistriesYaml(registryEnabled, registryHost);
   // Pre-pulls read content from the environment's containerd store.
   const sticky = stickyDiskPathSync();
   const contentDirectory = `${sticky ? `${sticky}/containerd` : "/var/lib/containerd"}/${CONTENT_BLOBS}`;
-  const serverArgs = [
-    "k3s",
-    "server",
-    // CoreDNS / pod DNS upstream. Without this, k3s sees only the
-    // loopback 127.0.0.11 (Docker's embedded DNS) in the container's
-    // /etc/resolv.conf, decides no usable nameserver exists, and writes a
-    // fallback `nameserver 8.8.8.8` that CoreDNS then forwards to — so
-    // pods reach the public internet but NOT peer services on
-    // spectest-net (`<svc>.internal`, fakes, service-TLS hosts all
-    // NXDOMAIN). We instead point k3s at the container's default gateway
-    // — the spectest-net bridge gateway, where spectest-resolver binds a
-    // second listener for exactly this. The file is written by the
-    // command wrapper below because the gateway IP is only known at
-    // container start.
-    "--resolv-conf=/run/spectest-resolv.conf",
-    // metrics-server isn't useful in a test cluster.
-    "--disable=metrics-server",
-    // The bundled traefik is disabled because we install our own with
-    // hostNetwork (see setupK3sCluster) — ours is wired to the in-VM CA
-    // for `ingressDomains` TLS and to a fixed IngressClass. Opt out of
-    // ours entirely with `k3s({ traefik: false })`; passing
-    // `--disable=traefik` via `extraArgs` does NOT do that (it only
-    // re-disables the bundled one, which is already off) and is rejected
-    // below rather than silently ignored.
-    "--disable=traefik",
-    // ServiceLB (klipper-lb) stays ENABLED, so `type: LoadBalancer`
-    // Services get an address and work. It was disabled for years
-    // because klipper-lb binds its ports with a CNI portmap hostPort,
-    // and portmap's iptables-nft rules need the `xt_comment` netfilter
-    // match — absent from the stock kernel we used to run on, which made
-    // every LoadBalancer hang at <pending> with no svclb DaemonSet. We
-    // build the guest kernel ourselves now and it carries
-    // CONFIG_NETFILTER_XT_MATCH_COMMENT (see
-    // scripts/local-vms-kernel-additions.config), so the constraint is
-    // gone. local-storage stays enabled too: a controller pod that binds
-    // no host ports.
-    ...(loadBalancerEnabled ? [] : ["--disable=servicelb"]),
-    // Pod CIDR MUST avoid 10.42.0.0/16: that's the spectest-br0 host
-    // bridge subnet, whose gateway 10.42.0.1 fronts the host image caches
-    // (zot :5000-5007, buildkitd :1234). k3s's *default* pod CIDR is also
-    // 10.42.0.0/16 — with --flannel-backend=host-gw, flannel programs that
-    // route into the node's own routing table and gives cni0 the subnet's
-    // .1 (10.42.0.1). That shadows the route to the host gateway, so once
-    // CNI comes up the node can no longer reach 10.42.0.1 and every
-    // subsequent registry pull dies with "connect: connection refused"
-    // (e.g. the in-cluster registry's `registry:2`, applied after the
-    // cluster is up — the airgap-bundled system images pull *before* CNI
-    // and so sneak through). Move pods to 10.44/service to 10.45.
-    "--cluster-cidr=10.44.0.0/16",
-    "--service-cidr=10.45.0.0/16",
-    "--cluster-dns=10.45.0.10",
-    "--flannel-backend=host-gw",
-    "--write-kubeconfig-mode=644",
-    // kube-proxy in nftables mode: native nftables rules, no
-    // xt_comment dependency. Pod→ClusterIP routing works, so
-    // CoreDNS / helm-install / anything-talking-to-the-API works.
-    // GA in k8s 1.32.
-    "--kube-proxy-arg=proxy-mode=nftables",
-    ...extra,
-  ].join(" ");
-  // The service `command` runs under `/bin/sh -c` (see runContainer in
-  // daemon.ts), so derive the bridge gateway from the container's default
-  // route at start time, write it as the k3s resolv-conf, then exec k3s
-  // (exec so it stays the container's main process and signals / the
-  // readyCheck behave exactly as before). `/run` is a tmpfs on this
-  // service, so the file is writable and never persisted into a snapshot.
-  const cmd =
-    "GW=\"$(ip route 2>/dev/null | awk '/^default/{print $3; exit}')\"; " +
-    'if [ -n "$GW" ]; then ' +
-    "printf 'nameserver %s\\noptions ndots:0\\n' \"$GW\" > /run/spectest-resolv.conf; " +
-    "else echo 'spectest: no default gateway found; k3s pod DNS for peer services will not resolve' >&2; " +
-    ": > /run/spectest-resolv.conf; fi; " +
-    // Make the root mount shared. A CSI node plugin bind-mounts volumes
-    // under /var/lib/kubelet and needs that propagation to reach the
-    // kubelet and the workload pod; docker gives a container a private
-    // root, so without this every CSI node plugin fails to publish and
-    // any snapshot/PVC-backed storage test is impossible. `kind` does the
-    // same in its entrypoint for the same reason. Best-effort: on a
-    // kernel/runtime that refuses it, the cluster still boots — only CSI
-    // is affected.
-    "mount --make-rshared / 2>/dev/null || " +
-    "echo 'spectest: could not make / rshared; CSI node plugins may fail to publish volumes' >&2; " +
-    storePrepareSh() +
-    // Sticky and environment-local stores use overlayfs.
-    `exec ${serverArgs} --snapshotter=$SNAP`;
-  // Plain /readyz probe. On a warm zot cache the cluster's images are
-  // already local, so the first boot completes in seconds; the
-  // first-ever boot on a cold-cache host pulls through the mirror and
-  // can take a couple of minutes (covered by readyTimeoutSecs).
-  //
-  // Clearing the boot marker is part of the probe on purpose: "the API
-  // server answers" is exactly the condition that proves this cluster came
-  // up on the store it was given, and it is the only signal the sweep can
-  // read on the next boot. See STORE_SWEEP_SH.
-  const readyCmd = `kubectl get --raw=/readyz >/dev/null 2>&1 && rm -f ${CLUSTER_STORE_TARGET}/${BOOT_MARKER}`;
-  const def = {
-    image: { type: "registry", reference: `rancher/k3s:${version}` },
-    imageBuilds: images.builds,
-    tls: registryHost ? [{ hostname: registryHost, port: K3S_REGISTRY_PORT }] : [],
-    command: cmd,
-    privileged: true,
-    tmpfs: ["/run", "/var/run"],
-    cgroupns: "host",
-    // 80/443 are advisory — peer services and host code reach them via
-    // the k3s container's IP. The component's Traefik binds them directly
-    // (hostNetwork, so it shares the container's netns). 5000 is the
-    // in-cluster registry (hostNetwork pod bound to the container netns),
-    // reached by peers at the cluster's own `<key>.internal:5000` alias.
-    // A `type: LoadBalancer` Service's ports are bound by klipper-lb in
-    // the same netns and are reachable the same way, without appearing
-    // in this list (it's documentation, not a firewall).
-    ports: registryEnabled ? [80, 443, 6443, K3S_REGISTRY_PORT] : [80, 443, 6443],
-    // Sticky disks retain an independent overlayfs store per service/version.
-    volumes: [...(sticky
-      ? [{ source: `${stickyKubernetesDirectory(SELF_SERVICE_TOKEN, version)}/containerd`, target: CLUSTER_STORE_TARGET }]
-      : [{ target: CLUSTER_STORE_TARGET }]),
-      ...(publishBuilds.length ? [
-        { source: registryDirectory(SELF_SERVICE_TOKEN), target: REGISTRY_DATA },
-        { source: contentDirectory, target: REGISTRY_CONTENT, readOnly: true },
+  const nodeNames = (cluster: string) => Array.from({ length: count }, (_, i) => i === 0 ? cluster : `${cluster}-node-${i + 1}`);
+  const createNode = (cluster: string, server: boolean): ServiceConfig => {
+    const registriesYaml = buildRegistriesYaml(registryEnabled, cluster, registryHost);
+    const nodeArgs = [
+      "k3s", server ? "server" : "agent",
+      "--node-name=$(hostname)",
+      // cgroupns=host is required by our nested runtime. Each kubelet must
+      // own a separate tree or its cleanup can remove another node's pods.
+      "--kubelet-arg=cgroup-root=/spectest-$(hostname)",
+      "--kubelet-arg=kubelet-cgroups=/spectest-$(hostname)/k3s",
+      "--kubelet-arg=runtime-cgroups=/spectest-$(hostname)/k3s",
+      `--token-file=${JOIN_DIRECTORY}/token`,
+      ...(server ? [] : [`--server=https://${cluster}.internal:6443`]),
+      // CoreDNS / pod DNS upstream. Without this, k3s sees only the
+      // loopback 127.0.0.11 (Docker's embedded DNS) in the container's
+      // /etc/resolv.conf, decides no usable nameserver exists, and writes a
+      // fallback `nameserver 8.8.8.8` that CoreDNS then forwards to — so
+      // pods reach the public internet but NOT peer services on
+      // spectest-net (`<svc>.internal`, fakes, service-TLS hosts all
+      // NXDOMAIN). We instead point k3s at the container's default gateway
+      // — the spectest-net bridge gateway, where spectest-resolver binds a
+      // second listener for exactly this. The file is written by the
+      // command wrapper below because the gateway IP is only known at
+      // container start.
+      "--resolv-conf=/run/spectest-resolv.conf",
+      // metrics-server isn't useful in a test cluster.
+      ...(server ? [
+        "--disable=metrics-server",
+        // The bundled traefik is disabled because we install our own with
+        // hostNetwork (see setupK3sCluster) — ours is wired to the in-VM CA
+        // for `ingressDomains` TLS and to a fixed IngressClass. Opt out of
+        // ours entirely with `k3s({ traefik: false })`; passing
+        // `--disable=traefik` via `extraArgs` does NOT do that (it only
+        // re-disables the bundled one, which is already off) and is rejected
+        // below rather than silently ignored.
+        "--disable=traefik",
+        // ServiceLB (klipper-lb) stays ENABLED, so `type: LoadBalancer`
+        // Services get an address and work. It was disabled for years
+        // because klipper-lb binds its ports with a CNI portmap hostPort,
+        // and portmap's iptables-nft rules need the `xt_comment` netfilter
+        // match — absent from the stock kernel we used to run on, which made
+        // every LoadBalancer hang at <pending> with no svclb DaemonSet. We
+        // build the guest kernel ourselves now and it carries
+        // CONFIG_NETFILTER_XT_MATCH_COMMENT (see
+        // scripts/local-vms-kernel-additions.config), so the constraint is
+        // gone. local-storage stays enabled too: a controller pod that binds
+        // no host ports.
+        ...(loadBalancerEnabled ? [] : ["--disable=servicelb"]),
+        // Pod CIDR MUST avoid 10.42.0.0/16: that's the spectest-br0 host
+        // bridge subnet, whose gateway 10.42.0.1 fronts the host image caches
+        // (zot :5000-5007, buildkitd :1234). k3s's *default* pod CIDR is also
+        // 10.42.0.0/16 — with --flannel-backend=host-gw, flannel programs that
+        // route into the node's own routing table and gives cni0 the subnet's
+        // .1 (10.42.0.1). That shadows the route to the host gateway, so once
+        // CNI comes up the node can no longer reach 10.42.0.1 and every
+        // subsequent registry pull dies with "connect: connection refused"
+        // (e.g. the in-cluster registry's `registry:2`, applied after the
+        // cluster is up — the airgap-bundled system images pull *before* CNI
+        // and so sneak through). Move pods to 10.44/service to 10.45.
+        "--cluster-cidr=10.44.0.0/16",
+        "--service-cidr=10.45.0.0/16",
+        "--cluster-dns=10.45.0.10",
+        "--flannel-backend=host-gw",
+        "--write-kubeconfig-mode=644",
+        `--tls-san=${cluster}.internal`,
       ] : []),
-    ],
-    files: [
-      ...(registriesYaml ? [{ path: "/etc/rancher/k3s/registries.yaml", content: registriesYaml }] : []),
-    ],
-    readyCheck: {
-      type: "exec" as const,
-      command: readyCmd,
-      timeoutSecs: opts.readyTimeoutSecs ?? 120,
+      // kube-proxy in nftables mode: native nftables rules, no
+      // xt_comment dependency. Pod→ClusterIP routing works, so
+      // CoreDNS / helm-install / anything-talking-to-the-API works.
+      // GA in k8s 1.32.
+      "--kube-proxy-arg=proxy-mode=nftables",
+      ...(server ? extra : []),
+    ].join(" ");
+    // The service `command` runs under `/bin/sh -c` (see runContainer in
+    // daemon.ts), so derive the bridge gateway from the container's default
+    // route at start time, write it as the k3s resolv-conf, then exec k3s
+    // (exec so it stays the container's main process and signals / the
+    // readyCheck behave exactly as before). `/run` is a tmpfs on this
+    // service; its contents are captured with the VM's memory on snapshot.
+    const cmd =
+      "GW=\"$(ip route 2>/dev/null | awk '/^default/{print $3; exit}')\"; " +
+      'if [ -n "$GW" ]; then ' +
+      "printf 'nameserver %s\\noptions ndots:0\\n' \"$GW\" > /run/spectest-resolv.conf; " +
+      "else echo 'spectest: no default gateway found; k3s pod DNS for peer services will not resolve' >&2; " +
+      ": > /run/spectest-resolv.conf; fi; " +
+      // Make the root mount shared. A CSI node plugin bind-mounts volumes
+      // under /var/lib/kubelet and needs that propagation to reach the
+      // kubelet and the workload pod; docker gives a container a private
+      // root, so without this every CSI node plugin fails to publish and
+      // any snapshot/PVC-backed storage test is impossible. `kind` does the
+      // same in its entrypoint for the same reason. Best-effort: on a
+      // kernel/runtime that refuses it, the cluster still boots — only CSI
+      // is affected.
+      "mount --make-rshared / 2>/dev/null || " +
+      "echo 'spectest: could not make / rshared; CSI node plugins may fail to publish volumes' >&2; " +
+      // Kubelet validates its cgroup root before it creates kubepods. Delegate
+      // the controllers on this empty parent, keeping k3s processes in a leaf.
+      'CG="spectest-$(hostname)"; ' +
+      'if [ -f /sys/fs/cgroup/cgroup.controllers ]; then ' +
+      'mkdir -p "/sys/fs/cgroup/$CG/k3s" || exit 1; ' +
+      'sed "s/[^ ][^ ]*/+&/g" "/sys/fs/cgroup/$CG/cgroup.controllers" > "/sys/fs/cgroup/$CG/cgroup.subtree_control" || exit 1; ' +
+      'else for C in /sys/fs/cgroup/*; do [ ! -d "$C" ] || mkdir -p "$C/$CG/k3s" || exit 1; done; fi; ' +
+      storePrepareSh() +
+      (server
+        ? `if [ ! -s ${JOIN_DIRECTORY}/token ]; then (umask 077; cat /proc/sys/kernel/random/uuid > ${JOIN_DIRECTORY}/token); fi; `
+        : `while [ ! -s ${JOIN_DIRECTORY}/token ]; do sleep 0.1; done; `) +
+      // Sticky and environment-local stores use overlayfs.
+      `exec ${nodeArgs} --snapshotter=$SNAP`;
+    // Plain /readyz probe. On a warm zot cache the cluster's images are
+    // already local, so the first boot completes in seconds; the
+    // first-ever boot on a cold-cache host pulls through the mirror and
+    // can take a couple of minutes (covered by readyTimeoutSecs).
+    //
+    // Per-node image-store boot markers are cleared after group setup has
+    // checked all Nodes and finished loading images, not at API readiness.
+    const readyCmd = "kubectl get --raw=/readyz >/dev/null 2>&1";
+    const def = {
+      image: { type: "registry", reference: `rancher/k3s:${version}` },
+      imageBuilds: server ? images.builds : [],
+      tls: server && registryHost ? [{ hostname: registryHost, port: K3S_REGISTRY_PORT }] : [],
+      command: cmd,
+      privileged: true,
+      tmpfs: ["/run", "/var/run"],
+      cgroupns: "host",
+      // 80/443 are advisory — peer services and host code reach them via
+      // the k3s container's IP. The component's Traefik binds them directly
+      // (hostNetwork, so it shares the container's netns). 5000 is the
+      // in-cluster registry (hostNetwork pod bound to the container netns),
+      // reached by peers at the cluster's own `<key>.internal:5000` alias.
+      // A `type: LoadBalancer` Service's ports are bound by klipper-lb in
+      // the same netns and are reachable the same way, without appearing
+      // in this list (it's documentation, not a firewall).
+      ports: server ? (registryEnabled ? [80, 443, 6443, K3S_REGISTRY_PORT] : [80, 443, 6443]) : [],
+      // Sticky disks retain an independent overlayfs store per service/version.
+      // Every node has its own writable containerd root.
+      volumes: [{ name: `spectest-k3s-${cluster}-join`, target: JOIN_DIRECTORY }, ...(sticky
+        ? [{ source: `${stickyKubernetesDirectory(SELF_SERVICE_TOKEN, version)}/containerd`, target: CLUSTER_STORE_TARGET }]
+        : [{ target: CLUSTER_STORE_TARGET }]),
+        ...(server && publishBuilds.length ? [
+          { source: registryDirectory(SELF_SERVICE_TOKEN), target: REGISTRY_DATA },
+          { source: contentDirectory, target: REGISTRY_CONTENT, readOnly: true },
+        ] : []),
+      ],
+      files: [
+        ...(registriesYaml ? [{ path: "/etc/rancher/k3s/registries.yaml", content: registriesYaml }] : []),
+      ],
+      // Agents start before the server and wait for its join token. The group
+      // setup below gates readiness on every actual Kubernetes Node, including
+      // the server, rather than treating container startup as cluster readiness.
+      readyCheck: server ? {
+        type: "exec" as const,
+        command: readyCmd,
+        timeoutSecs: opts.readyTimeoutSecs ?? 120,
+      } : undefined,
+    } satisfies ServiceConfig;
+    return server && ingressDomains.length
+      ? provides(def, ingressDomains.map(domain => dnsName(`*.${domain}`, { service: SELF_SERVICE_TOKEN })))
+      : def;
+  };
+
+  return serviceGroup({
+    primary: "server",
+    services: g => {
+      const parts: Record<string, ServiceConfig> = { server: createNode(g.name, true) };
+      for (let i = 2; i <= count; i++) parts[`node-${i}`] = createNode(g.name, false);
+      return parts;
     },
     setup: async (ctx: ServiceSetupContext<K3sHelpers>) => {
       const { name, helpers, exec } = ctx;
       const runtime = componentRuntime(ctx);
+      const nodes = nodeNames(name);
       if (images.builds.length && !runtime) throw new Error(`k3s(${name}): prepared image inputs are missing`);
-      const infrastructure = (async () => {
+      const admission = (async () => {
         if (runtime?.caBundle && caPresent()) {
           const { cert, key } = await issueIngressCert([runtime.gateway]);
           await installPodTrust({ name, gateway: runtime.gateway, cert, key,
@@ -2149,31 +2201,35 @@ export function k3s(opts: K3sOptions = {}) {
             await new Promise(resolve => setTimeout(resolve, 100));
           }
         }
-        await setupK3sCluster(name, helpers, {
-          registry: registryEnabled,
-          seed: publishBuilds.length ? () => seedRegistry(name, contentDirectory,
-            runtime!.images.filter(image => isPublished(image.reference))) : undefined,
-          ingressDomains,
-          traefik: traefikEnabled,
-        });
       })();
+      const infrastructure = admission.then(() => setupK3sCluster(name, helpers, {
+        registry: registryEnabled,
+        seed: publishBuilds.length ? () => seedRegistry(name, contentDirectory,
+          runtime!.images.filter(image => isPublished(image.reference))) : undefined,
+        ingressDomains,
+        traefik: traefikEnabled,
+      }));
       await waitForAll([
-        runtime ? loadK3sBuildImages(name, runtime.images, runtime.gateway, exec) : Promise.resolve(),
-        prePullK3sImages(name, images.pulls.filter(ref => !isPublished(ref)), exec),
+        infrastructure.then(tasks => tasks.ingress),
+        waitForK3sNodes(name, nodes, exec, (opts.readyTimeoutSecs ?? 120) * 1000),
+        ...nodes.map(node => runtime ? loadK3sBuildImages(node, runtime.images, runtime.gateway, exec) : Promise.resolve()),
+        ...nodes.map(node => prePullK3sImages(node, images.pulls.filter(ref => !isPublished(ref) && !ref.startsWith(`${name}.internal:${K3S_REGISTRY_PORT}/`)), exec)),
         (async () => {
           // Registry-local references must not race registry startup/publication.
-          await infrastructure;
-          await prePullK3sImages(name, images.pulls.filter(isPublished), exec);
+          await infrastructure.then(tasks => tasks.registry);
+          await waitForAll(nodes.map(node => prePullK3sImages(node,
+            images.pulls.filter(ref => isPublished(ref) || ref.startsWith(`${name}.internal:${K3S_REGISTRY_PORT}/`)), exec)));
         })(),
       ]);
-      if (sticky) await registerStickyKubernetesStore(name, version);
+      await waitForAll(nodes.map(async node => {
+        const cleared = await exec(node, ["rm", "-f", `${CLUSTER_STORE_TARGET}/${BOOT_MARKER}`]);
+        if (cleared.exitCode !== 0) throw new Error(`k3s(${node}): failed to mark image store ready: ${cleared.stderr}`);
+        if (sticky) await registerStickyKubernetesStore(node, version);
+      }));
     },
     helpers: async ({ name, exec, poll }: ServiceHelpersContext): Promise<K3sHelpers> => {
-      // Read the cluster's kubeconfig and address the API server by its
-      // auto-assigned `<name>.internal` hostname on spectest-net. TLS
-      // verification is off (see the K3sHelpers docstring), so the
-      // server's cert SAN list doesn't need to include the .internal
-      // name.
+      // Read the server's kubeconfig and address its stable .internal alias.
+      // This endpoint is also used by the agents joining the cluster.
       const kcRead = await exec(name, ["cat", "/etc/rancher/k3s/k3s.yaml"]);
       if (kcRead.exitCode !== 0) {
         throw new Error(
@@ -2223,6 +2279,7 @@ export function k3s(opts: K3sOptions = {}) {
       };
 
       return {
+        nodes: nodeNames(name),
         kubeconfig,
         client,
         apply: (manifest, applyOpts) => applyManifest(deps, manifest, applyOpts),
@@ -2251,17 +2308,5 @@ export function k3s(opts: K3sOptions = {}) {
           ),
       };
     },
-  } satisfies ServiceDefinition<K3sHelpers>;
-
-  // Wildcard ingress domains → a dnsName(`*.<domain>`, { service: self })
-  // each, attached via provides(). SELF_SERVICE_TOKEN resolves to this
-  // service's key at load time (the component can't know it here). The
-  // resolver then points every host under the domain at the cluster.
-  if (ingressDomains.length === 0) return def;
-  return provides(
-    def,
-    ingressDomains.map((domain) =>
-      dnsName(`*.${domain}`, { service: SELF_SERVICE_TOKEN }),
-    ),
-  );
+  });
 }