@specific.dev/spectest 0.4.0

package/src/components/k3s.ts

@@ -0,0 +1,1324 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { spawn as nodeSpawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { existsSync, readFileSync } from "node:fs";
+import { readFile, unlink } from "node:fs/promises";
+
+import {
+  AppsV1Api,
+  CoreV1Api,
+  KubeConfig,
+  KubernetesObjectApi,
+  ResponseContext,
+  ServerConfiguration,
+  createConfiguration,
+  loadAllYaml,
+  type KubernetesObject,
+  type RequestContext,
+} from "@kubernetes/client-node";
+import { Observable } from "@kubernetes/client-node/dist/gen/rxjsStub.js";
+
+import type { ServiceDefinition } from "../index.js";
+import { dnsName, provides, SELF_SERVICE_TOKEN } from "../index.js";
+import { readRaw, readTag, wrap } from "../inspect.js";
+import type { Wrapped } from "../inspect.js";
+import { recorderAnnotate, recorderRemove } from "../recorder.js";
+
+export interface K3sOptions {
+  /** Image tag for the official `rancher/k3s` image. Default `"v1.30.6-k3s1"`. */
+  version?: string;
+  /**
+   * Extra arguments appended to `k3s server`. Useful for `--tls-san=...`,
+   * additional `--disable=<addon>`, custom CIDRs, etc.
+   */
+  extraArgs?: string[];
+  /**
+   * Readiness probe timeout in seconds. k3s on a warm image is ready in
+   * a few seconds; the first cold start of an env (image pull + cluster
+   * bootstrap) can take 30–60s. Default `120`.
+   */
+  readyTimeoutSecs?: number;
+  /**
+   * Run an in-cluster OCI registry (CNCF `distribution` / `registry:2`)
+   * that the cluster's own containerd trusts. This is the hermetic
+   * stand-in for a cloud registry (ECR/GCR/GHCR): a peer service builds
+   * an image, pushes it here over plain HTTP, references
+   * `<cluster-key>.internal:5000/...` from a Deployment, and the kubelet
+   * pulls it straight back. Lets you test a real
+   * `build → push → deploy → pull` pipeline with no external registry
+   * and no image pre-baking.
+   *
+   * **The registry's push/pull address is `<cluster-key>.internal:5000`**,
+   * where `<cluster-key>` is the key you give this service in the
+   * `services` map (e.g. a cluster at `services.cluster` is reachable at
+   * `cluster.internal:5000`). That's the cluster service's own
+   * unconditional `.internal` alias — peer-reachable and never clobbered
+   * by any `hostnames` you set — so it resolves identically from peer
+   * containers (push) and the cluster's own containerd (pull). Wire it
+   * into your platform, e.g. `env: { REGISTRY_URL: "cluster.internal:5000" }`.
+   * Plain HTTP, so configure your push client for an insecure registry.
+   *
+   * On by default. Set `false` for clusters that only ever run public
+   * images — that skips the extra pod.
+   */
+  registry?: boolean;
+  /**
+   * Domains to route into this cluster's ingress via **wildcard DNS**. For
+   * each `"example.com"`, spectest-resolver answers any `*.example.com`
+   * query with the cluster container's IP, where Traefik dispatches by Host
+   * to the matching Ingress. This lets a test `kubectl apply` an Ingress for
+   * any host under the domain and reach it immediately — no need to
+   * pre-declare each hostname in `hostnames`.
+   *
+   * ```ts
+   * services: { k8s: k3s({ ingressDomains: ["example.com"] }) }
+   * // a test then applies an Ingress for foo.example.com and fetches it.
+   * ```
+   *
+   * For one-off hosts not under a declared domain, a test can also register
+   * dynamically with `ctx.dnsName(host, { service: "k8s" })`.
+   *
+   * **TLS.** Setting `ingressDomains` also makes those domains reachable
+   * over **HTTPS**: Traefik gains a `:443` entrypoint and serves a default
+   * certificate minted from the in-VM root CA with SANs `*.<domain>` for
+   * each declared domain. The CA is already trusted by the test framework
+   * (Node `fetch`, `ctx.browser()`, Python, the system store), so
+   * `ctx.fetch("https://foo.example.com")` gets a clean handshake — no
+   * per-Ingress `spec.tls` and no `--insecure` needed. Only hosts **under**
+   * a declared domain are covered by the cert; static `hostnames` not under
+   * one (and one-off `ctx.dnsName` hosts) remain HTTP-only.
+   */
+  ingressDomains?: string[];
+}
+
+/** Port the in-cluster registry listens on (plain HTTP). */
+const K3S_REGISTRY_PORT = 5000;
+
+/**
+ * Rewrites a `@kubernetes/client-node` API class so each method's resolved
+ * value comes back **inspect-wrapped** ({@link Wrapped}) — exactly what
+ * `withTagging` does at runtime. The method *signatures* (argument types) are
+ * untouched; only the `Promise<R>` result becomes `Promise<Wrapped<R>>`, so
+ * `expect(pod.status.phase)` links to the API call with no cast and you
+ * `.unwrap()` before using a value as raw data. Non-method
+ * members pass through unchanged. Mirrors {@link RecordingSqlClient} on the
+ * postgres side.
+ */
+type Tagged<T> = {
+  [K in keyof T]: T[K] extends (...args: infer A) => Promise<infer R>
+    ? (...args: A) => Promise<Wrapped<R>>
+    : T[K];
+};
+
+/**
+ * Pre-instantiated `@kubernetes/client-node` API clients sharing the
+ * same recording HTTP transport. Every method call lands on the test
+ * event log as an HTTP event alongside `fetch` calls, and its result is
+ * inspect-wrapped (see {@link Tagged}) so assertions on it stay linked.
+ */
+export interface K3sClient {
+  core: Tagged<CoreV1Api>;
+  apps: Tagged<AppsV1Api>;
+  /** Generic object API — `create()`, `read()`, `patch()`, `delete()`
+   *  against any Kubernetes resource (custom resources included). */
+  objects: Tagged<KubernetesObjectApi>;
+}
+
+/** Helpers a `k3s(...)` service exposes on `ctx.svc.<name>`. */
+export interface K3sHelpers {
+  /**
+   * Fully-loaded `KubeConfig`. The cluster server URL is rewritten to
+   * `https://<service-name>.internal:6443`, the auto-assigned DNS name
+   * for this service on `spectest-net`. TLS verification is off because
+   * Bun's fetch doesn't honor an https.Agent's CA option (the client
+   * cert from the kubeconfig still flows through for auth), so the
+   * server's cert SAN list doesn't need to include the .internal name.
+   */
+  kubeconfig: KubeConfig;
+  /** Pre-built API clients. */
+  client: K3sClient;
+  /**
+   * Apply a (multi-document) YAML manifest. Each parsed document is
+   * created via `KubernetesObjectApi.create`. Returns the API server's
+   * response objects in input order — each element inspect-wrapped (the
+   * array container itself is plain), so `expect(created[0]!.metadata.uid)`
+   * links to its create call.
+   */
+  apply: (manifest: string) => Promise<Wrapped<KubernetesObject>[]>;
+}
+
+interface DockerExecResult {
+  stdout: string;
+  stderr: string;
+  code: number;
+}
+
+function runProcess(
+  cmd: string,
+  args: string[],
+  timeoutMs = 30_000,
+): Promise<DockerExecResult> {
+  return new Promise((resolve, reject) => {
+    const cp = nodeSpawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"] });
+    const out: Buffer[] = [];
+    const err: Buffer[] = [];
+    cp.stdout!.on("data", (c) => out.push(c));
+    cp.stderr!.on("data", (c) => err.push(c));
+    const t = setTimeout(() => cp.kill("SIGKILL"), timeoutMs);
+    cp.on("error", (e) => {
+      clearTimeout(t);
+      reject(e);
+    });
+    cp.on("close", (code) => {
+      clearTimeout(t);
+      resolve({
+        stdout: Buffer.concat(out).toString("utf8"),
+        stderr: Buffer.concat(err).toString("utf8"),
+        code: code ?? -1,
+      });
+    });
+  });
+}
+
+function runDocker(
+  args: string[],
+  timeoutMs = 30_000,
+): Promise<DockerExecResult> {
+  return runProcess("docker", args, timeoutMs);
+}
+
+// In-VM root CA, generated once into the base snapshot (see
+// control-plane `base.rs`). Trusted everywhere the test framework runs —
+// Node (`NODE_EXTRA_CA_CERTS`), Chromium (NSS DB), Python, the system
+// store — so a leaf signed by it gives `ctx.fetch`/`ctx.browser()` a
+// clean HTTPS handshake. The k3s `setup` hook runs inside the daemon's
+// Bun process (root in the VM), so it can read the CA key and mint
+// directly. These constants are intentionally redeclared here rather than
+// imported from the daemon: the SDK ships to end users and must not
+// depend on daemon internals.
+const CA_PATH = process.env.SPECTEST_CA_PATH ?? "/etc/spectest/ca.crt";
+const CA_KEY_PATH = process.env.SPECTEST_CA_KEY_PATH ?? "/etc/spectest/ca.key";
+
+function caPresent(): boolean {
+  return existsSync(CA_PATH) && existsSync(CA_KEY_PATH);
+}
+
+/**
+ * Mint a leaf certificate from the in-VM root CA covering `hostnames`
+ * (used here as the SANs of the cluster's wildcard ingress domains).
+ * Returns the cert + key as PEM strings. Self-contained openssl shell-out
+ * — deliberately not shared with the daemon's own cert minting to keep
+ * the distributed SDK decoupled from daemon code.
+ */
+async function issueIngressCert(
+  hostnames: string[],
+): Promise<{ cert: string; key: string }> {
+  const id = `spectest-k3s-ingress-${randomUUID().slice(0, 8)}`;
+  const keyPath = `/tmp/${id}.key`;
+  const crtPath = `/tmp/${id}.crt`;
+  const sans = hostnames.map((h) => `DNS:${h}`).join(",");
+  const r = await runProcess(
+    "openssl",
+    [
+      "req",
+      "-newkey",
+      "rsa:2048",
+      "-nodes",
+      "-keyout",
+      keyPath,
+      "-out",
+      crtPath,
+      "-x509",
+      "-CA",
+      CA_PATH,
+      "-CAkey",
+      CA_KEY_PATH,
+      "-days",
+      "3650",
+      "-subj",
+      "/CN=spectest-k3s-ingress",
+      "-addext",
+      `subjectAltName=${sans}`,
+      "-addext",
+      "basicConstraints=CA:FALSE",
+      "-addext",
+      "extendedKeyUsage=serverAuth",
+      "-addext",
+      "keyUsage=digitalSignature,keyEncipherment",
+    ],
+    30_000,
+  );
+  if (r.code !== 0) {
+    throw new Error(
+      `k3s ingress cert minting failed (openssl rc=${r.code}): ${
+        r.stderr.trim() || r.stdout.trim()
+      }`,
+    );
+  }
+  try {
+    const [cert, key] = await Promise.all([
+      readFile(crtPath, "utf8"),
+      readFile(keyPath, "utf8"),
+    ]);
+    return { cert, key };
+  } finally {
+    await Promise.all([
+      unlink(keyPath).catch(() => {}),
+      unlink(crtPath).catch(() => {}),
+    ]);
+  }
+}
+
+// Holds the inspector `sourceSeq` for the most recent HTTP call inside
+// a single API-method invocation. Filled in by `doFetch` after each
+// request, read by the `withTagging` proxy when the method's promise
+// resolves so the returned parsed object carries the back-reference.
+// AsyncLocalStorage is the right scope here — every call to an Api
+// method runs in its own holder and concurrent calls don't race.
+interface CallSlot {
+  seq?: number;
+}
+const callContext = new AsyncLocalStorage<CallSlot>();
+
+// HTTP transport for `@kubernetes/client-node` that routes through
+// `globalThis.fetch`. Two reasons:
+//   1. The daemon's per-test `installFetchWrapper` already records every
+//      `globalThis.fetch` call as an HTTP event — using fetch here gets
+//      k8s API calls recorded for free, no library-specific wiring.
+//   2. Bun's fetch needs Bun-shaped TLS options (`tls: { ... }`) for
+//      mTLS; node-fetch's `agent` parameter — which the library's
+//      default transport relies on — is silently ignored under Bun.
+//      We honor the lib's auth flow (`KubeConfig.applySecurityAuthentication`
+//      sets an Agent on the request) by extracting cert/key off that
+//      agent and passing them via the Bun-shaped option.
+class FetchHttpLibrary {
+  send(request: RequestContext): Observable<ResponseContext> {
+    const promise = doFetch(request);
+    return new Observable(promise);
+  }
+}
+
+/**
+ * Parsed Kubernetes semantics of a single API request, derived purely
+ * from the HTTP method + request path. Fed to `recorderAnnotate` to
+ * reclassify the generic `http` event the fetch wrapper recorded into a
+ * Kubernetes-specific `kube` event.
+ */
+interface KubeRequestMeta {
+  verb: string;
+  group?: string;
+  apiVersion?: string;
+  resource?: string;
+  subresource?: string;
+  name?: string;
+  namespace?: string;
+}
+
+/**
+ * Map a Kubernetes API request to its `(verb, group/version, resource,
+ * namespace, name, subresource)` from the URL + HTTP method alone.
+ *
+ * Path grammar (the two API roots):
+ *   - core group: `/api/<version>/...`
+ *   - named group: `/apis/<group>/<version>/...`
+ * after which the remainder is either a cluster-scoped resource
+ * (`nodes`, `namespaces`, …) or `namespaces/<ns>/<resource>...`. The
+ * trailing `<resource>[/<name>[/<subresource>]]` shape plus the method
+ * (and `?watch=`) yields the verb.
+ *
+ * Returns `null` for non-resource paths — discovery (`/api`, `/apis`,
+ * `/apis/<group>/<version>`), `/version`, `/healthz`, `/openapi/...` —
+ * so those stay rendered as plain `http`.
+ */
+function describeKubeRequest(
+  method: string,
+  rawUrl: string,
+): KubeRequestMeta | null {
+  let path: string;
+  let query: URLSearchParams;
+  try {
+    const u = new URL(rawUrl);
+    path = u.pathname;
+    query = u.searchParams;
+  } catch {
+    const q = rawUrl.indexOf("?");
+    path = q === -1 ? rawUrl : rawUrl.slice(0, q);
+    query = new URLSearchParams(q === -1 ? "" : rawUrl.slice(q + 1));
+  }
+
+  const segs = path.split("/").filter((s) => s.length > 0);
+  if (segs.length === 0) return null;
+
+  let group: string | undefined;
+  let apiVersion: string | undefined;
+  let rest: string[];
+  if (segs[0] === "api") {
+    group = "";
+    apiVersion = segs[1];
+    rest = segs.slice(2);
+  } else if (segs[0] === "apis") {
+    group = segs[1];
+    apiVersion = segs[2];
+    rest = segs.slice(3);
+  } else {
+    return null; // /version, /healthz, /openapi, …
+  }
+  if (!apiVersion) return null; // discovery root (/api, /apis/<group>)
+
+  // `namespaces/<ns>/<resource>...` is namespaced; everything else
+  // (including `namespaces` and `namespaces/<name>` themselves, and
+  // cluster-scoped resources like `nodes`) is taken as-is.
+  let namespace: string | undefined;
+  let resourcePath = rest;
+  if (rest[0] === "namespaces" && rest.length >= 3) {
+    namespace = rest[1];
+    resourcePath = rest.slice(2);
+  }
+  if (resourcePath.length === 0) return null; // APIResourceList discovery
+
+  const resource = resourcePath[0];
+  const name = resourcePath.length >= 2 ? resourcePath[1] : undefined;
+  const subresource = resourcePath.length >= 3 ? resourcePath[2] : undefined;
+
+  const watchParam = query.get("watch");
+  const watch = watchParam === "true" || watchParam === "1";
+  const hasName = name !== undefined;
+  let verb: string;
+  switch (method.toUpperCase()) {
+    case "GET":
+    case "HEAD":
+      verb = hasName ? "get" : watch ? "watch" : "list";
+      break;
+    case "POST":
+      verb = "create";
+      break;
+    case "PUT":
+      verb = "update";
+      break;
+    case "PATCH":
+      verb = "patch";
+      break;
+    case "DELETE":
+      verb = hasName ? "delete" : "deletecollection";
+      break;
+    default:
+      verb = method.toLowerCase();
+  }
+
+  return { verb, group, apiVersion, resource, subresource, name, namespace };
+}
+
+/**
+ * True for Kubernetes API *discovery* paths — the version/group/resource
+ * enumeration endpoints (`/api`, `/api/<version>`, `/apis`, `/apis/<group>`,
+ * `/apis/<group>/<version>`) the dynamic client hits to resolve a kind to
+ * its resource path. They carry no resource operation (so
+ * `describeKubeRequest` returns null), and `doFetch` retracts their events
+ * from the timeline. Non-resource paths that are NOT discovery (`/healthz`,
+ * `/version`, `/openapi`, …) are deliberately not matched — they stay as
+ * `http`.
+ */
+function isKubeDiscoveryPath(rawUrl: string): boolean {
+  let path: string;
+  try {
+    path = new URL(rawUrl).pathname;
+  } catch {
+    const q = rawUrl.indexOf("?");
+    path = q === -1 ? rawUrl : rawUrl.slice(0, q);
+  }
+  const segs = path.split("/").filter((s) => s.length > 0);
+  if (segs.length === 0) return false;
+  // `/api` + `/api/<version>`; `/apis` + `/apis/<group>` + `/apis/<group>/<version>`.
+  // Anything longer carries a resource segment and is handled as `kube`.
+  if (segs[0] === "api") return segs.length <= 2;
+  if (segs[0] === "apis") return segs.length <= 3;
+  return false;
+}
+
+async function doFetch(request: RequestContext): Promise<ResponseContext> {
+  const url = request.getUrl();
+  const method = String(request.getHttpMethod());
+  const body = request.getBody();
+  const reqHeaders: Record<string, string> = {};
+  for (const [k, v] of Object.entries(request.getHeaders())) {
+    reqHeaders[k] = String(v);
+  }
+
+  // The library's auth flow puts client cert/key on an https.Agent
+  // attached to the request. Pull them out so we can hand them to Bun's
+  // fetch via its `tls` option.
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const agent = request.getAgent() as any;
+  const agentOpts = agent?.options ?? {};
+  const tlsOpts: Record<string, unknown> = { rejectUnauthorized: false };
+  if (agentOpts.cert) tlsOpts.cert = agentOpts.cert;
+  if (agentOpts.key) tlsOpts.key = agentOpts.key;
+
+  const wrapped = await fetch(url, {
+    method,
+    headers: reqHeaders,
+    body: body as BodyInit | undefined,
+    signal: request.getSignal(),
+    // Bun-specific TLS shape; under Node this option is ignored.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    tls: tlsOpts,
+  } as RequestInit);
+
+  // Before unwrapping, capture the inspector tag the daemon's fetch
+  // wrapper installed on the Response. We feed the seq back through
+  // AsyncLocalStorage so the API method's eventual return value can
+  // re-acquire it — otherwise the chain `core.listNode() →
+  // expect(result)` would record assertions with no back-reference.
+  const tag = readTag(wrapped);
+  const slot = callContext.getStore();
+  if (slot && tag) slot.seq = tag.sourceSeq;
+
+  // Reclassify the `http` event the fetch wrapper just recorded into a
+  // Kubernetes-specific `kube` event (verb/resource/namespace/name), so
+  // the timeline reads `list pods · default` rather than the raw API URL.
+  // A `tag` is only present when recording was active for this call, so
+  // this is a no-op outside instrumented test runs.
+  if (tag && tag.sourceSeq !== undefined) {
+    const meta = describeKubeRequest(method, url);
+    if (meta) {
+      recorderAnnotate(tag.sourceSeq, { kind: "kube", ...meta });
+    } else if (isKubeDiscoveryPath(url)) {
+      // The dynamic client (`objects`, KubernetesObjectApi) can't know a
+      // kind's resource path ahead of time, so before the real request it
+      // GETs the group's resource list (`/apis/<group>/<version>` →
+      // APIResourceList) to map e.g. Ingress → `ingresses`/namespaced, then
+      // caches it (apiVersionResourceCache). That discovery GET is library
+      // plumbing the test author never wrote, and because the cache is
+      // per-daemon-process it surfaces non-deterministically across forks
+      // (first access pays it; `dependsOn` children inheriting the warm
+      // cache don't). Retract it rather than leave a bare `http` row — the
+      // real list/read that follows is recorded and reclassified as usual.
+      recorderRemove(tag.sourceSeq);
+    }
+    // Other non-resource paths (/healthz, /version, /openapi, …) fall
+    // through and stay rendered as plain `http`.
+  }
+
+  // The daemon's fetch wrapper proxies `Response.status` and similar
+  // primitives as carrier objects (so test assertions can fold under
+  // the originating HTTP event). The kubernetes/client-node lib calls
+  // `httpStatusCode.toString()` which would then return
+  // "[object Object]" and the status-code dispatch falls through to
+  // "Unknown API Status Code!". Pull out the raw Response.
+  const response =
+    (wrapped as { unwrap?: () => Response }).unwrap?.() ?? wrapped;
+
+  const resHeaders: Record<string, string> = {};
+  response.headers.forEach((v, k) => {
+    resHeaders[k] = v;
+  });
+  const buf = Buffer.from(await response.arrayBuffer());
+  return new ResponseContext(response.status, resHeaders, {
+    text: async () => buf.toString("utf8"),
+    binary: async () => buf,
+  });
+}
+
+/**
+ * Recursively strip the inspector's carrier/proxy wrappers from a
+ * value. Needed for arguments flowing into the kubernetes/client-node
+ * API methods — if a wrapped pod's `metadata.name` (a primitive-carrier
+ * object) reaches a URL template, the lib stringifies it to
+ * `"[object Object]"` and the request 404s.
+ */
+function deepUnwrap(value: unknown): unknown {
+  if (value === null || value === undefined) return value;
+  const raw = readRaw(value);
+  if (raw !== value) return deepUnwrap(raw);
+  if (typeof value !== "object") return value;
+  if (Array.isArray(value)) return value.map(deepUnwrap);
+  const out: Record<string, unknown> = {};
+  for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+    out[k] = deepUnwrap(v);
+  }
+  return out;
+}
+
+/**
+ * Wrap a `@kubernetes/client-node` Api instance so each method call
+ * runs in its own AsyncLocalStorage slot — `doFetch` writes the HTTP
+ * event's `sourceSeq` into the slot, and after the lib parses the
+ * response we re-attach the seq to the returned object. Downstream
+ * `expect(result.items[0].status…)` assertions then fold under that
+ * HTTP event in the test event log, the same way `expect(res.status)`
+ * does for plain `fetch` calls.
+ *
+ * Method arguments are deep-unwrapped on the way in so values pulled
+ * from a previous API response (still carrying the inspector wrappers)
+ * can be passed straight back into another call.
+ *
+ * Non-function properties pass through untagged.
+ */
+function withTagging<T extends object>(api: T): Tagged<T> {
+  return new Proxy(api, {
+    get(target, prop, receiver) {
+      const value = Reflect.get(target, prop, receiver);
+      if (typeof value !== "function") return value;
+      // Bind the original method to `target` so the lib's internal
+      // `this.configuration` accesses keep working.
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const method = (value as any).bind(target);
+      return (...args: unknown[]): unknown => {
+        const unwrappedArgs = args.map(deepUnwrap);
+        const slot: CallSlot = {};
+        const result = callContext.run(slot, () =>
+          method(...unwrappedArgs),
+        );
+        // Wrap unconditionally — `slot.seq` is undefined when no event was
+        // recorded (setup/eval, no active recorder), but the result's type is
+        // wrapped, so the value must be wrapped at runtime too (just without a
+        // provenance link). Keeps `.unwrap()` available in every context.
+        if (result && typeof (result as Promise<unknown>).then === "function") {
+          return (result as Promise<unknown>).then((v) => wrap(v, slot.seq));
+        }
+        return wrap(result, slot.seq);
+      };
+    },
+  }) as unknown as Tagged<T>;
+}
+
+/**
+ * Image tag for the Traefik we install. Pulled on first cluster boot
+ * through the host `zot` mirror (`docker.io` → cache) configured in
+ * `registries.yaml`, then captured into the warm-template snapshot so
+ * warm starts never pull.
+ */
+const TRAEFIK_IMAGE = "rancher/mirrored-library-traefik:3.3.2";
+
+/**
+ * Names of the in-cluster resources that carry the CA-signed default
+ * ingress certificate (created in `setupK3sCluster` when TLS is enabled).
+ * The Secret holds the leaf cert+key; the ConfigMap holds the Traefik
+ * file-provider snippet that points the `default` TLS store at it.
+ */
+const TRAEFIK_TLS_SECRET = "traefik-default-tls";
+const TRAEFIK_DYNAMIC_CONFIGMAP = "traefik-dynamic";
+
+/**
+ * Traefik file-provider dynamic config: make the in-VM-CA leaf the
+ * `default` store certificate, so every router on the `websecure`
+ * entrypoint (which we force to TLS) serves it with no per-Ingress
+ * `spec.tls` needed.
+ */
+const TRAEFIK_DYNAMIC_TLS = `tls:
+  stores:
+    default:
+      defaultCertificate:
+        certFile: /certs/tls.crt
+        keyFile: /certs/tls.key
+`;
+
+/**
+ * Traefik manifest applied during setup(). `hostNetwork: true` puts
+ * Traefik in the k3s container's netns, so it binds the container's :80
+ * (and, with `tls`, :443) directly — no CNI portmap involved (that path
+ * still trips on the kernel's missing xt_comment match).
+ *
+ * When `tls` is set we add a `websecure` :443 entrypoint with TLS forced
+ * on (served from the `default` store, i.e. the in-VM-CA leaf mounted
+ * from the `traefik-default-tls` Secret via the file provider). HTTPS
+ * then works for any routed host under the cluster's `ingressDomains`
+ * with zero per-Ingress config; the :80 `web` entrypoint is unchanged.
+ */
+function buildTraefikManifest(tls: boolean): string {
+  const args = [
+    "        - --entrypoints.web.address=:80",
+    ...(tls
+      ? [
+          "        - --entrypoints.websecure.address=:443",
+          "        - --entrypoints.websecure.http.tls=true",
+        ]
+      : []),
+    "        - --providers.kubernetesingress=true",
+    "        - --providers.kubernetesingress.ingressclass=traefik",
+    ...(tls
+      ? [
+          "        - --providers.file.directory=/dynamic",
+          "        - --providers.file.watch=true",
+        ]
+      : []),
+    "        - --log.level=INFO",
+  ].join("\n");
+  const ports = [
+    "        - name: web",
+    "          containerPort: 80",
+    ...(tls
+      ? ["        - name: websecure", "          containerPort: 443"]
+      : []),
+  ].join("\n");
+  const volumeMounts = tls
+    ? `
+        volumeMounts:
+        - name: default-cert
+          mountPath: /certs
+          readOnly: true
+        - name: dynamic
+          mountPath: /dynamic
+          readOnly: true`
+    : "";
+  const volumes = tls
+    ? `
+      volumes:
+      - name: default-cert
+        secret:
+          secretName: ${TRAEFIK_TLS_SECRET}
+      - name: dynamic
+        configMap:
+          name: ${TRAEFIK_DYNAMIC_CONFIGMAP}`
+    : "";
+  return `apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: traefik
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: traefik
+rules:
+- apiGroups: [""]
+  resources: ["services", "endpoints", "secrets", "nodes"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["discovery.k8s.io"]
+  resources: ["endpointslices"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["networking.k8s.io"]
+  resources: ["ingresses", "ingressclasses"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["networking.k8s.io"]
+  resources: ["ingresses/status"]
+  verbs: ["update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: traefik
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: traefik
+subjects:
+- kind: ServiceAccount
+  name: traefik
+  namespace: kube-system
+---
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  name: traefik
+  annotations:
+    ingressclass.kubernetes.io/is-default-class: "true"
+spec:
+  controller: traefik.io/ingress-controller
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: traefik
+  namespace: kube-system
+  labels:
+    app: traefik
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: traefik
+  template:
+    metadata:
+      labels:
+        app: traefik
+    spec:
+      serviceAccountName: traefik
+      hostNetwork: true
+      dnsPolicy: Default
+      tolerations:
+      - operator: Exists
+      containers:
+      - name: traefik
+        image: ${TRAEFIK_IMAGE}
+        imagePullPolicy: IfNotPresent
+        args:
+${args}
+        ports:
+${ports}${volumeMounts}${volumes}
+`;
+}
+
+/**
+ * Host-side `zot` pull-through cache layout (local Firecracker provider
+ * only). One zot instance per upstream registry, all bound to the
+ * `spectest-br0` gateway `10.42.0.1` on the ports below — **kept in sync
+ * with `scripts/install-zot.sh`**. We mirror the cluster's containerd
+ * through these so every image pull reuses the shared host cache instead
+ * of hitting the public registry, and we list the canonical upstream as
+ * a fallback endpoint so a missing/cold mirror only ever slows a pull,
+ * never breaks it.
+ */
+const ZOT_MIRRORS: Array<{ registry: string; port: number; upstream: string }> = [
+  { registry: "docker.io", port: 5000, upstream: "https://registry-1.docker.io" },
+  { registry: "ghcr.io", port: 5001, upstream: "https://ghcr.io" },
+  { registry: "quay.io", port: 5002, upstream: "https://quay.io" },
+  { registry: "registry.k8s.io", port: 5003, upstream: "https://registry.k8s.io" },
+  { registry: "public.ecr.aws", port: 5004, upstream: "https://public.ecr.aws" },
+  { registry: "gcr.io", port: 5005, upstream: "https://gcr.io" },
+  { registry: "mcr.microsoft.com", port: 5006, upstream: "https://mcr.microsoft.com" },
+];
+
+/**
+ * Discover the host-side image cache gateway by reading the same
+ * `registry-mirrors` entry the in-VM dockerd already uses (baked into
+ * the local provider's golden `/etc/docker/daemon.json`). Returns the
+ * gateway host (`"10.42.0.1"`) when present, or `null` when there's no
+ * host cache — e.g. on Freestyle, where the cluster then pulls every
+ * image direct. Runs inside the daemon (VM) at `index.ts` load time, so
+ * the result is stable per host and never poisons the warm-template
+ * cache.
+ */
+function detectHostMirrorGateway(): string | null {
+  try {
+    const cfg = JSON.parse(
+      readFileSync("/etc/docker/daemon.json", "utf8"),
+    ) as { "registry-mirrors"?: string[] };
+    const first = cfg["registry-mirrors"]?.[0];
+    return first ? new URL(first).hostname || null : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Build `/etc/rancher/k3s/registries.yaml`. k3s reads this **once, at
+ * startup**, to configure its embedded containerd — which is why it has
+ * to be seeded via `files` (a pre-start bind mount) rather than a
+ * `setup` hook. Two jobs:
+ *   1. Mirror the cluster's image pulls through the host `zot` cache
+ *      (local provider only; omitted when there's no host cache).
+ *   2. Trust the in-cluster registry, addressed as `<key>.internal:5000`
+ *      (the `{{SPECTEST_SERVICE}}` token is expanded to the cluster's
+ *      service key when the file is written). Image *references* use
+ *      that peer-reachable name, but containerd pulls via the loopback
+ *      endpoint `http://127.0.0.1:5000` — the hostNetwork registry pod
+ *      shares the node's netns, so this needs no in-container DNS and
+ *      can't be broken by a clobbered `hostnames`.
+ * Returns `null` when there's nothing to configure (no host cache and
+ * `registry` disabled), in which case no file is injected.
+ */
+function buildRegistriesYaml(registryEnabled: boolean): string | null {
+  const gateway = detectHostMirrorGateway();
+  if (!gateway && !registryEnabled) return null;
+
+  const lines: string[] = ["mirrors:"];
+  if (gateway) {
+    for (const { registry, port, upstream } of ZOT_MIRRORS) {
+      lines.push(
+        `  "${registry}":`,
+        `    endpoint:`,
+        `      - "http://${gateway}:${port}"`,
+        `      - "${upstream}"`,
+      );
+    }
+  }
+  if (registryEnabled) {
+    const host = `{{SPECTEST_SERVICE}}.internal:${K3S_REGISTRY_PORT}`;
+    lines.push(
+      `  "${host}":`,
+      `    endpoint:`,
+      `      - "http://127.0.0.1:${K3S_REGISTRY_PORT}"`,
+      "configs:",
+      // The endpoint is plain HTTP; the config (keyed by endpoint host)
+      // makes that explicit and disables any TLS attempt against it.
+      `  "127.0.0.1:${K3S_REGISTRY_PORT}":`,
+      `    tls:`,
+      `      insecure_skip_verify: true`,
+    );
+  }
+  return lines.join("\n") + "\n";
+}
+
+/**
+ * In-cluster OCI registry (CNCF `distribution`). `hostNetwork: true`
+ * binds the cluster container's `:5000` directly — the same trick
+ * Traefik uses — so peer services reach it at `<cluster-key>.internal:5000`
+ * (the cluster service's own alias) and the node's own containerd reaches
+ * it at `127.0.0.1:5000`. Storage is an `emptyDir`, so pushed images live
+ * in the cluster and are captured by snapshot / isolated per test fork
+ * like all other in-VM state.
+ */
+const REGISTRY_MANIFEST = `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: spectest-registry
+  namespace: kube-system
+  labels:
+    app: spectest-registry
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: spectest-registry
+  template:
+    metadata:
+      labels:
+        app: spectest-registry
+    spec:
+      hostNetwork: true
+      dnsPolicy: Default
+      tolerations:
+      - operator: Exists
+      containers:
+      - name: registry
+        image: registry:2
+        imagePullPolicy: IfNotPresent
+        env:
+        - name: REGISTRY_HTTP_ADDR
+          value: ":${K3S_REGISTRY_PORT}"
+        - name: REGISTRY_STORAGE_DELETE_ENABLED
+          value: "true"
+        ports:
+        - name: registry
+          containerPort: ${K3S_REGISTRY_PORT}
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/registry
+      volumes:
+      - name: data
+        emptyDir: {}
+`;
+
+/**
+ * Wait for a Deployment to reach its desired ready-replica count,
+ * polling once a second up to `timeoutMs`. Throws with the last-seen
+ * status (plus kube-system pod diagnostics) on timeout.
+ */
+async function waitForDeployment(
+  clusterName: string,
+  helpers: K3sHelpers,
+  deployment: string,
+  timeoutMs: number,
+): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  let lastErr: string | undefined;
+  while (Date.now() < deadline) {
+    try {
+      // `.unwrap()` recovers the plain object — the client wraps its result in
+      // every context now (provenance-free here, since this internal poll runs
+      // during setup with no active recorder). We're reading for control flow,
+      // not asserting, so go straight to raw.
+      const dep = (
+        await helpers.client.apps.readNamespacedDeployment({
+          name: deployment,
+          namespace: "kube-system",
+        })
+      ).unwrap();
+      const ready = dep.status?.readyReplicas ?? 0;
+      const want = dep.spec?.replicas ?? 1;
+      if (ready >= want && want > 0) return;
+      lastErr = `${deployment} Deployment exists but only ${ready}/${want} replicas Ready`;
+    } catch (err) {
+      const msg = (err as Error)?.message ?? String(err);
+      lastErr = /not found|404/i.test(msg)
+        ? `${deployment} Deployment does not exist yet`
+        : msg;
+    }
+    // 250ms: the two sequential rollout waits in setup sit on the cold
+    // start's critical path, and a 1s poll wasted up to ~2s of it.
+    await new Promise((r) => setTimeout(r, 250));
+  }
+  const diag = await collectTraefikDiagnostics(helpers);
+  throw new Error(
+    `k3s(${clusterName}): ${deployment} did not reach Ready within ${
+      timeoutMs / 1000
+    }s. ${lastErr ?? ""}\n${diag}`,
+  );
+}
+
+/**
+ * Post-Ready setup. Apply the Traefik manifest (hostNetwork) and, when
+ * enabled, the in-cluster registry; wait for each Deployment to come
+ * Ready. Captured by the warm-template snapshot, so warm starts pay none
+ * of this cost.
+ *
+ * When the cluster declares `ingressDomains` (and the in-VM CA is
+ * present), TLS is enabled: we mint a CA-signed leaf covering `*.<domain>`
+ * for each domain, stash it in the `traefik-default-tls` Secret + a
+ * file-provider ConfigMap, and bring Traefik up with a `websecure` :443
+ * entrypoint serving it as the default cert. Those domains are then
+ * reachable over HTTPS with a cert the test framework already trusts.
+ */
+async function setupK3sCluster(
+  name: string,
+  helpers: K3sHelpers,
+  opts: { registry: boolean; ingressDomains: string[] },
+): Promise<void> {
+  const tlsEnabled = opts.ingressDomains.length > 0 && caPresent();
+  if (tlsEnabled) {
+    const { cert, key } = await issueIngressCert(
+      opts.ingressDomains.map((d) => `*.${d}`),
+    );
+    // Apply the cert Secret + dynamic-config ConfigMap before the
+    // Deployment that mounts them. `stringData` lets us hand over plain
+    // PEM; the API server base64-encodes it.
+    await helpers.client.core.createNamespacedSecret({
+      namespace: "kube-system",
+      body: {
+        metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
+        type: "kubernetes.io/tls",
+        stringData: { "tls.crt": cert, "tls.key": key },
+      },
+    });
+    await helpers.client.core.createNamespacedConfigMap({
+      namespace: "kube-system",
+      body: {
+        metadata: {
+          name: TRAEFIK_DYNAMIC_CONFIGMAP,
+          namespace: "kube-system",
+        },
+        data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
+      },
+    });
+  }
+  await helpers.apply(buildTraefikManifest(tlsEnabled));
+  if (opts.registry) await helpers.apply(REGISTRY_MANIFEST);
+  // Both rollouts proceed independently inside the cluster — wait on them
+  // concurrently (they used to serialize, wasting up to a rollout's tail).
+  const waits = [waitForDeployment(name, helpers, "traefik", 120_000)];
+  if (opts.registry) {
+    waits.push(waitForDeployment(name, helpers, "spectest-registry", 120_000));
+  }
+  await Promise.all(waits);
+}
+
+/**
+ * Snapshot of kube-system state, dumped on traefik-wait timeout. With
+ * the static install, the failure surface is just "did our Deployment
+ * schedule and become Ready?" — pod listing covers that.
+ */
+async function collectTraefikDiagnostics(helpers: K3sHelpers): Promise<string> {
+  const lines: string[] = [];
+  try {
+    const pods = await helpers.client.core.listNamespacedPod({
+      namespace: "kube-system",
+    });
+    lines.push(`kube-system pods (${pods.items.length}):`);
+    for (const p of pods.items) {
+      const phase = p.status?.phase ?? "?";
+      const cs = p.status?.containerStatuses ?? [];
+      const reasons = cs
+        .map((c) => c.state?.waiting?.reason ?? c.state?.terminated?.reason ?? "")
+        .filter((s) => s)
+        .join(",");
+      lines.push(
+        `  ${p.metadata?.name ?? "?"}: phase=${phase}${reasons ? ` reasons=${reasons}` : ""}`,
+      );
+      // The waiting `message` carries containerd's actual error — e.g. the
+      // failing endpoint, an upstream `429 Too Many Requests`, or a
+      // `connection refused`. The `reason` alone (`ErrImagePull`) hides all
+      // of that, which is exactly what we need when a pull won't settle.
+      for (const c of cs) {
+        const msg =
+          c.state?.waiting?.message ?? c.state?.terminated?.message ?? "";
+        if (msg) lines.push(`      ${c.name}: ${msg.replace(/\s+/g, " ").trim()}`);
+      }
+    }
+  } catch (err) {
+    lines.push(`(listing pods failed: ${(err as Error)?.message ?? String(err)})`);
+  }
+  // Recent Warning events surface pull failures the kubelet emits before a
+  // container status even settles (FailedPull / Failed / BackOff), with the
+  // raw containerd message attached. Best-effort: never let diagnostics throw.
+  try {
+    const events = await helpers.client.core.listNamespacedEvent({
+      namespace: "kube-system",
+    });
+    const warnings = (events.items ?? [])
+      .filter((e) => e.type === "Warning")
+      .map((e) => ({
+        obj: e.involvedObject?.name ?? "?",
+        reason: e.reason ?? "?",
+        message: (e.message ?? "").replace(/\s+/g, " ").trim(),
+      }))
+      .filter((e) => e.message);
+    if (warnings.length) {
+      lines.push(`kube-system Warning events (${warnings.length}):`);
+      // Keep the tail — newest events are appended last by the API.
+      for (const w of warnings.slice(-12)) {
+        lines.push(`  ${w.obj} [${w.reason}] ${w.message}`);
+      }
+    }
+  } catch (err) {
+    lines.push(`(listing events failed: ${(err as Error)?.message ?? String(err)})`);
+  }
+  return lines.join("\n");
+}
+
+/**
+ * A ready-to-use single-node Kubernetes cluster (k3s). Drop into
+ * `environment.services`:
+ *
+ * ```ts
+ * services: { k8s: k3s() }
+ * ```
+ *
+ * Tests get `@kubernetes/client-node` API objects pre-wired to this
+ * cluster at `ctx.svc.<key>.client` — `core`, `apps`, and a generic
+ * `objects` (`KubernetesObjectApi`). Every API call is recorded on the
+ * test event log alongside `fetch` calls. There's also `apply(yaml)`
+ * sugar for piping a multi-document manifest in.
+ *
+ * **Ingress.** We deploy Traefik ourselves in `hostNetwork` mode
+ * during `setup()`. Traefik binds the cluster container's :80 directly
+ * (no ServiceLB / klipper-lb needed), watches Ingress objects via the
+ * API, and routes incoming requests to pod Endpoints. Any `hostnames`
+ * declared on this service in env.ts therefore route through Traefik:
+ * a peer doing `fetch("http://app.example.com")` resolves the host to
+ * the k3s container's IP (via systest-resolver), lands on Traefik,
+ * and gets dispatched to the matching Ingress rule's backend pods.
+ *
+ * **Workarounds for Freestyle's kernel** (Linux 6.1.0-x-freestyle).
+ * The stock kernel is missing the `xt_comment` netfilter match
+ * extension. Two consequences, each handled below:
+ *
+ *   1. *kube-proxy* in default iptables mode generates rules with
+ *      `-m comment --comment "..."`, which the kernel rejects —
+ *      breaking pod→ClusterIP routing and every pod that talks to the
+ *      in-cluster API (helm-install Jobs, CoreDNS, …). Fixed by
+ *      `--kube-proxy-arg=proxy-mode=nftables`: kube-proxy emits
+ *      native nftables rules where comments are a first-class
+ *      construct, no xt_comment dependency. nftables proxy mode is
+ *      GA in k8s 1.32, which is why we pin that.
+ *
+ *   2. *CNI portmap plugin* (used by klipper-lb's hostPort to expose
+ *      LoadBalancer ports on the host) still uses iptables-nft and
+ *      hits the same xt_comment failure — there's no equivalent
+ *      flag to switch it to native nftables. Workaround: disable the
+ *      bundled traefik + ServiceLB and run Traefik with
+ *      `hostNetwork: true` ourselves. hostNetwork pods don't go
+ *      through portmap at all (they share the node's netns directly),
+ *      so the broken plugin is never invoked.
+ *
+ * Flannel uses the `host-gw` backend because Freestyle's stock kernel
+ * lacks the VXLAN module — fine for single-node clusters.
+ */
+/**
+ * Default k3s docker image tag.
+ *
+ * The cluster's system images (the `rancher/k3s` image itself, plus
+ * coredns / local-path-provisioner / pause and the Traefik we deploy)
+ * are pulled on first boot through the host `zot` pull-through cache —
+ * `registries.yaml` (seeded via `files`) mirrors `docker.io` and
+ * `registry.k8s.io` at it. The first-ever cluster boot on a cold-cache
+ * host pays the upstream pull once; thereafter zot serves the blobs
+ * host-wide and the warm-template snapshot captures the booted cluster,
+ * so neither cold-cache nor warm starts re-pull. Any `opts.version`
+ * works — there's no base-snapshot release to keep in sync with.
+ *
+ * **Why v1.32.x:** kube-proxy's `nftables` proxy mode is GA in k8s 1.32
+ * (beta in 1.31, alpha-gated in 1.30). The component runs kube-proxy in
+ * this mode to sidestep Freestyle's missing `xt_comment` netfilter
+ * extension; dropping below 1.31 reintroduces the broken iptables path.
+ */
+const DEFAULT_K3S_VERSION = "v1.32.1-k3s1";
+
+export function k3s(opts: K3sOptions = {}) {
+  const version = opts.version ?? DEFAULT_K3S_VERSION;
+  const extra = opts.extraArgs ?? [];
+  const registryEnabled = opts.registry !== false;
+  // Wildcard ingress domains. Drives both the `provides(... dnsName)`
+  // wiring below and (when non-empty) the CA-signed TLS default cert that
+  // setupK3sCluster mints so these domains are reachable over HTTPS.
+  const ingressDomains = opts.ingressDomains ?? [];
+  // `/etc/rancher/k3s/registries.yaml` (host-cache mirrors + trust for
+  // the in-cluster registry). Seeded via `files` because k3s reads it
+  // only at startup, before any setup hook could run.
+  const registriesYaml = buildRegistriesYaml(registryEnabled);
+  const serverArgs = [
+    "k3s",
+    "server",
+    // CoreDNS / pod DNS upstream. Without this, k3s sees only the
+    // loopback 127.0.0.11 (Docker's embedded DNS) in the container's
+    // /etc/resolv.conf, decides no usable nameserver exists, and writes a
+    // fallback `nameserver 8.8.8.8` that CoreDNS then forwards to — so
+    // pods reach the public internet but NOT peer services on
+    // spectest-net (`<svc>.internal`, fakes, service-TLS hosts all
+    // NXDOMAIN). We instead point k3s at the container's default gateway
+    // — the spectest-net bridge gateway, where spectest-resolver binds a
+    // second listener for exactly this. The file is written by the
+    // command wrapper below because the gateway IP is only known at
+    // container start.
+    "--resolv-conf=/run/spectest-resolv.conf",
+    // metrics-server isn't useful in a test cluster.
+    "--disable=metrics-server",
+    // traefik + servicelb disabled: their klipper-lb DaemonSet uses
+    // CNI portmap to bind host port 80, which still needs xt_comment
+    // (the iptables compat path that the kernel can't satisfy).
+    // We install Traefik with hostNetwork in setup() — same effect,
+    // no portmap involved. local-storage stays enabled: it's a
+    // controller pod that doesn't bind host ports.
+    "--disable=traefik",
+    "--disable=servicelb",
+    // Pod CIDR MUST avoid 10.42.0.0/16: that's the spectest-br0 host
+    // bridge subnet, whose gateway 10.42.0.1 fronts the host image caches
+    // (zot :5000-5007, buildkitd :1234). k3s's *default* pod CIDR is also
+    // 10.42.0.0/16 — with --flannel-backend=host-gw, flannel programs that
+    // route into the node's own routing table and gives cni0 the subnet's
+    // .1 (10.42.0.1). That shadows the route to the host gateway, so once
+    // CNI comes up the node can no longer reach 10.42.0.1 and every
+    // subsequent registry pull dies with "connect: connection refused"
+    // (e.g. the in-cluster registry's `registry:2`, applied after the
+    // cluster is up — the airgap-bundled system images pull *before* CNI
+    // and so sneak through). Move pods to 10.44/service to 10.45.
+    "--cluster-cidr=10.44.0.0/16",
+    "--service-cidr=10.45.0.0/16",
+    "--cluster-dns=10.45.0.10",
+    "--flannel-backend=host-gw",
+    "--write-kubeconfig-mode=644",
+    // kube-proxy in nftables mode: native nftables rules, no
+    // xt_comment dependency. Pod→ClusterIP routing works, so
+    // CoreDNS / helm-install / anything-talking-to-the-API works.
+    // GA in k8s 1.32.
+    "--kube-proxy-arg=proxy-mode=nftables",
+    ...extra,
+  ].join(" ");
+  // The service `command` runs under `/bin/sh -c` (see runContainer in
+  // daemon.ts), so derive the bridge gateway from the container's default
+  // route at start time, write it as the k3s resolv-conf, then exec k3s
+  // (exec so it stays the container's main process and signals / the
+  // readyCheck behave exactly as before). `/run` is a tmpfs on this
+  // service, so the file is writable and never persisted into a snapshot.
+  const cmd =
+    "GW=\"$(ip route 2>/dev/null | awk '/^default/{print $3; exit}')\"; " +
+    'if [ -n "$GW" ]; then ' +
+    "printf 'nameserver %s\\noptions ndots:0\\n' \"$GW\" > /run/spectest-resolv.conf; " +
+    "else echo 'spectest: no default gateway found; k3s pod DNS for peer services will not resolve' >&2; " +
+    ": > /run/spectest-resolv.conf; fi; " +
+    `exec ${serverArgs}`;
+  // Plain /readyz probe. On a warm zot cache the cluster's images are
+  // already local, so the first boot completes in seconds; the
+  // first-ever boot on a cold-cache host pulls through the mirror and
+  // can take a couple of minutes (covered by readyTimeoutSecs).
+  const readyCmd = "kubectl get --raw=/readyz >/dev/null 2>&1";
+  const def = {
+    image: { type: "registry", reference: `rancher/k3s:${version}` },
+    command: cmd,
+    privileged: true,
+    tmpfs: ["/run", "/var/run"],
+    cgroupns: "host",
+    // 80/443 are advisory — peer services and host code reach them via
+    // the k3s container's IP. ServiceLB (klipper-lb) binds them inside
+    // the container's netns and forwards to the traefik pod. 5000 is the
+    // in-cluster registry (hostNetwork pod bound to the container netns),
+    // reached by peers at the cluster's own `<key>.internal:5000` alias.
+    ports: registryEnabled ? [80, 443, 6443, K3S_REGISTRY_PORT] : [80, 443, 6443],
+    // NOTE: do NOT mount /var/lib/rancher/k3s/agent/containerd as a cache
+    // volume. It was tried (to spare a recreated cluster re-pulling its
+    // system images on delta restores) and a fresh k3s server against the
+    // previous container's containerd store — killed un-cleanly by the
+    // teardown's `docker rm -f` — wedged the apiserver minutes in
+    // (rollouts never settled, pod listing started failing). The zot
+    // mirror already makes those re-pulls cheap; the residual win wasn't
+    // worth the recovery semantics of a crash-state store under a fresh
+    // cluster db.
+    ...(registriesYaml
+      ? {
+          files: [
+            { path: "/etc/rancher/k3s/registries.yaml", content: registriesYaml },
+          ],
+        }
+      : {}),
+    readyCheck: {
+      type: "exec" as const,
+      command: readyCmd,
+      timeoutSecs: opts.readyTimeoutSecs ?? 120,
+    },
+    setup: async ({ name, helpers }: { name: string; helpers: K3sHelpers }) => {
+      await setupK3sCluster(name, helpers, {
+        registry: registryEnabled,
+        ingressDomains,
+      });
+    },
+    helpers: async ({ name }: { name: string }): Promise<K3sHelpers> => {
+      // Read the cluster's kubeconfig and address the API server by its
+      // auto-assigned `<name>.internal` hostname on spectest-net. TLS
+      // verification is off (see the K3sHelpers docstring), so the
+      // server's cert SAN list doesn't need to include the .internal
+      // name.
+      const kcRead = await runDocker([
+        "exec",
+        name,
+        "cat",
+        "/etc/rancher/k3s/k3s.yaml",
+      ]);
+      if (kcRead.code !== 0) {
+        throw new Error(
+          `k3s(${name}): failed to read kubeconfig from container: ${kcRead.stderr.trim()}`,
+        );
+      }
+
+      const kubeconfig = new KubeConfig();
+      kubeconfig.loadFromString(kcRead.stdout);
+
+      const server = `https://${name}.internal:6443`;
+      // Update kc.clusters so any code that reads kubeconfig sees the
+      // right server URL, but the actual request server comes from the
+      // Configuration we build below. `Cluster.server` is typed `readonly`
+      // by @kubernetes/client-node, but the loaded object is a plain mutable
+      // record — write through a mutable view rather than rebuild the config.
+      for (const cluster of kubeconfig.clusters) {
+        (cluster as { -readonly [K in keyof typeof cluster]: typeof cluster[K] }).server =
+          server;
+      }
+
+      const httpApi = new FetchHttpLibrary();
+      const baseServer = new ServerConfiguration(server, {});
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const config = createConfiguration({
+        baseServer,
+        authMethods: { default: kubeconfig as any },
+        httpApi: httpApi as any,
+      });
+
+      const core = withTagging(new CoreV1Api(config));
+      const apps = withTagging(new AppsV1Api(config));
+      const objects = withTagging(new KubernetesObjectApi(config));
+
+      const apply = async (
+        manifest: string,
+      ): Promise<Wrapped<KubernetesObject>[]> => {
+        const docs = loadAllYaml(manifest) as KubernetesObject[];
+        const out: Wrapped<KubernetesObject>[] = [];
+        for (const doc of docs) {
+          if (!doc || typeof doc !== "object" || !("kind" in doc)) continue;
+          // `objects.create` is wrapped by `withTagging`, so each returned
+          // object already carries the back-reference to its create call.
+          const created = await objects.create(doc);
+          out.push(created as unknown as Wrapped<KubernetesObject>);
+        }
+        return out;
+      };
+
+      return {
+        kubeconfig,
+        client: { core, apps, objects },
+        apply,
+      };
+    },
+  } satisfies ServiceDefinition<K3sHelpers>;
+
+  // Wildcard ingress domains → a dnsName(`*.<domain>`, { service: self })
+  // each, attached via provides(). SELF_SERVICE_TOKEN resolves to this
+  // service's key at load time (the component can't know it here). The
+  // resolver then points every host under the domain at the cluster.
+  if (ingressDomains.length === 0) return def;
+  return provides(
+    def,
+    ingressDomains.map((domain) =>
+      dnsName(`*.${domain}`, { service: SELF_SERVICE_TOKEN }),
+    ),
+  );
+}