@specific.dev/spectest 0.39.0 → 0.43.0

package/src/components/k3s.ts

@@ -0,0 +1,2061 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { spawn as nodeSpawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { existsSync, readFileSync } from "node:fs";
+import { readFile, unlink } from "node:fs/promises";
+
+import {
+  AppsV1Api,
+  BatchV1Api,
+  CoreV1Api,
+  KubeConfig,
+  KubernetesObjectApi,
+  PatchStrategy,
+  ResponseContext,
+  ServerConfiguration,
+  createConfiguration,
+  loadAllYaml,
+  type KubernetesListObject,
+  type KubernetesObject,
+  type RequestContext,
+  type V1APIResource,
+  type V1DeleteOptions,
+  type V1DaemonSetStatus,
+  type V1DeploymentStatus,
+  type V1Job,
+  type V1StatefulSetStatus,
+  type V1Status,
+} from "@kubernetes/client-node";
+import { Observable } from "@kubernetes/client-node/dist/gen/rxjsStub.js";
+
+import type {
+  ServiceDefinition,
+  ServiceHelpersContext,
+  SpectestContext,
+} from "../index.js";
+import { dnsName, provides, SELF_SERVICE_TOKEN } from "../index.js";
+import { deepUnwrap, readTag, wrap } from "../inspect.js";
+import type { Wrapped } from "../inspect.js";
+import { recorderAnnotate, recorderRemove } from "../recorder.js";
+
+export type {
+  KubernetesObject,
+  KubernetesListObject,
+} from "@kubernetes/client-node";
+
+export interface K3sOptions {
+  /** Image tag for the official `rancher/k3s` image. Default `"v1.30.6-k3s1"`. */
+  version?: string;
+  /**
+   * Extra arguments appended to `k3s server`. Useful for `--tls-san=...`,
+   * additional `--disable=<addon>`, custom CIDRs, etc.
+   *
+   * Two values are **rejected** rather than silently ignored, because the
+   * component owns those decisions and passing them here has no effect:
+   * `--disable=traefik` (use `traefik: false`) and `--disable=servicelb`
+   * (use `loadBalancer: false`).
+   */
+  extraArgs?: string[];
+  /**
+   * Install the component's own Traefik ingress controller (hostNetwork,
+   * default IngressClass, wired to the in-VM CA when `ingressDomains` is
+   * set). Default `true`.
+   *
+   * Set `false` to run a cluster with no ingress controller at all — for
+   * a project that installs its own (via Helm, an operator, or a raw
+   * manifest) and wants the port to itself. The bundled k3s Traefik is
+   * disabled either way, so `false` really does mean none.
+   */
+  traefik?: boolean;
+  /**
+   * Run k3s's ServiceLB (klipper-lb) so `type: LoadBalancer` Services are
+   * assigned an address instead of sitting at `<pending>` forever.
+   * Default `true`.
+   *
+   * The assigned address is the **node IP** — the k3s container's own IP
+   * on `spectest-net` — so a LoadBalancer on port 8080 is reachable from
+   * a test or a peer service at `<cluster-key>.internal:8080`. Because
+   * klipper-lb binds the port on the node, a LoadBalancer that asks for
+   * port 80 or 443 collides with the component's hostNetwork Traefik;
+   * either pick another port or pass `traefik: false`.
+   */
+  loadBalancer?: boolean;
+  /**
+   * Readiness probe timeout in seconds. k3s on a warm image is ready in
+   * a few seconds; the first cold start of an env (image pull + cluster
+   * bootstrap) can take 30–60s. Default `120`.
+   */
+  readyTimeoutSecs?: number;
+  /**
+   * Run an in-cluster OCI registry (CNCF `distribution` / `registry:2`)
+   * that the cluster's own containerd trusts. This is the hermetic
+   * stand-in for a cloud registry (ECR/GCR/GHCR): a peer service builds
+   * an image, pushes it here over plain HTTP, references
+   * `<cluster-key>.internal:5000/...` from a Deployment, and the kubelet
+   * pulls it straight back. Lets you test a real
+   * `build → push → deploy → pull` pipeline with no external registry
+   * and no image pre-baking.
+   *
+   * **The registry's push/pull address is `<cluster-key>.internal:5000`**,
+   * where `<cluster-key>` is the key you give this service in the
+   * `services` map (e.g. a cluster at `services.cluster` is reachable at
+   * `cluster.internal:5000`). That's the cluster service's own
+   * unconditional `.internal` alias — peer-reachable and never clobbered
+   * by any `hostnames` you set — so it resolves identically from peer
+   * containers (push) and the cluster's own containerd (pull). Wire it
+   * into your platform, e.g. `env: { REGISTRY_URL: "cluster.internal:5000" }`.
+   * Plain HTTP, so configure your push client for an insecure registry.
+   *
+   * On by default. Set `false` for clusters that only ever run public
+   * images — that skips the extra pod.
+   */
+  registry?: boolean;
+  /**
+   * Domains to route into this cluster's ingress via **wildcard DNS**. For
+   * each `"example.com"`, spectest-resolver answers any `*.example.com`
+   * query with the cluster container's IP, where Traefik dispatches by Host
+   * to the matching Ingress. This lets a test `kubectl apply` an Ingress for
+   * any host under the domain and reach it immediately — no need to
+   * pre-declare each hostname in `hostnames`.
+   *
+   * ```ts
+   * services: { k8s: k3s({ ingressDomains: ["example.com"] }) }
+   * // a test then applies an Ingress for foo.example.com and fetches it.
+   * ```
+   *
+   * For one-off hosts not under a declared domain, a test can also register
+   * dynamically with `ctx.dnsName(host, { service: "k8s" })`.
+   *
+   * **TLS.** Setting `ingressDomains` also makes those domains reachable
+   * over **HTTPS**: Traefik gains a `:443` entrypoint and serves a default
+   * certificate minted from the in-VM root CA with SANs `*.<domain>` for
+   * each declared domain. The CA is already trusted by the test framework
+   * (Node `fetch`, `ctx.browser()`, Python, the system store), so
+   * `ctx.fetch("https://foo.example.com")` gets a clean handshake — no
+   * per-Ingress `spec.tls` and no `--insecure` needed. Only hosts **under**
+   * a declared domain are covered by the cert; static `hostnames` not under
+   * one (and one-off `ctx.dnsName` hosts) remain HTTP-only.
+   */
+  ingressDomains?: string[];
+}
+
+/** Port the in-cluster registry listens on (plain HTTP). */
+const K3S_REGISTRY_PORT = 5000;
+
+/**
+ * `extraArgs` entries the component overrides anyway. Passing one of
+ * these looks like it works — the flag really is appended to the k3s
+ * command line — but the component's own behaviour is what decides the
+ * outcome, so the cluster comes up contradicting the argument. Failing
+ * at load time costs a second; discovering it costs a boot cycle.
+ */
+const OVERRIDDEN_EXTRA_ARGS: Record<string, string> = {
+  traefik:
+    "the component installs its own Traefik (hostNetwork) after the cluster is up — " +
+    "pass `traefik: false` to have no ingress controller at all",
+  servicelb:
+    "ServiceLB is controlled by the `loadBalancer` option — pass `loadBalancer: false` to disable it",
+};
+
+function assertUsableExtraArgs(extra: readonly string[]): void {
+  for (const arg of extra) {
+    const m = /^--disable[=\s]+(.+)$/.exec(arg.trim());
+    const addon = m?.[1]?.trim();
+    if (!addon) continue;
+    const why = OVERRIDDEN_EXTRA_ARGS[addon];
+    if (why) {
+      throw new Error(
+        `k3s(): extraArgs cannot control "${addon}" — ${why}. ` +
+          `(Remove ${JSON.stringify(arg)} from extraArgs.)`,
+      );
+    }
+  }
+}
+
+/**
+ * Rewrites a `@kubernetes/client-node` API class so each method's resolved
+ * value comes back **inspect-wrapped** ({@link Wrapped}) — exactly what
+ * `withTagging` does at runtime. The method *signatures* (argument types) are
+ * untouched; only the `Promise<R>` result becomes `Promise<Wrapped<R>>`, so
+ * `expect(pod.status.phase)` links to the API call with no cast and you
+ * `.unwrap()` before using a value as raw data. Non-method
+ * members pass through unchanged. Mirrors {@link RecordingSqlClient} on the
+ * postgres side.
+ */
+type Tagged<T> = {
+  [K in keyof T]: T[K] extends (...args: infer A) => Promise<infer R>
+    ? (...args: A) => Promise<Wrapped<R>>
+    : T[K];
+};
+
+/**
+ * `{ apiVersion, kind, metadata: { name } }` — enough to address an
+ * existing object. Redeclared here because `@kubernetes/client-node`
+ * keeps its equivalent (`KubernetesObjectHeader`) module-private.
+ */
+export type KubernetesObjectRef<T extends KubernetesObject = KubernetesObject> =
+  Pick<T, "apiVersion" | "kind"> & {
+    metadata: { name: string; namespace?: string };
+  };
+
+/**
+ * The dynamic object API, hand-declared rather than derived through
+ * {@link Tagged}.
+ *
+ * `Tagged<T>` infers each method from its *erased* signature, and a mapped
+ * type can't reintroduce a type parameter (TypeScript has no higher-kinded
+ * types). For `CoreV1Api`/`AppsV1Api` that's invisible — their methods
+ * return concrete types — but `KubernetesObjectApi.read<T>`/`list<T>` are
+ * generic, so mapping them collapsed `read<MyCustomResource>(…)` to
+ * `Promise<Wrapped<KubernetesObject>>`: `Expected 0 type arguments, but got
+ * 1`, then `Property 'spec' does not exist on type 'KubernetesObject'` —
+ * precisely on the custom resources this API exists to reach. Worse, the
+ * degraded value made `expect(...)` resolve to the *locator* overload, so
+ * the reported error was a baffling `Property 'toBe' does not exist on type
+ * 'LocatorAssertion'` three lines further down.
+ *
+ * So the generic methods are written out. Arguments match the library's
+ * (positional, as it declares them); only the resolved value becomes
+ * {@link Wrapped}. See {@link K3sHelpers.list} for an options-object
+ * wrapper over the positional `list`.
+ */
+export interface TaggedObjectApi {
+  create<T extends KubernetesObject>(
+    spec: T,
+    pretty?: string,
+    dryRun?: string,
+    fieldManager?: string,
+  ): Promise<Wrapped<T>>;
+  patch<T extends KubernetesObject>(
+    spec: T,
+    pretty?: string,
+    dryRun?: string,
+    fieldManager?: string,
+    force?: boolean,
+    patchStrategy?: PatchStrategy,
+  ): Promise<Wrapped<T>>;
+  /** Read one object. **Throws** on 404 — to assert something is *gone*,
+   *  prefer {@link K3sHelpers.list} with a `metadata.name` field selector
+   *  and assert the returned `items` are empty (a throw carries no value
+   *  to assert on, and no provenance link). */
+  read<T extends KubernetesObject>(
+    spec: KubernetesObjectRef<T>,
+    pretty?: string,
+    exact?: boolean,
+    exportt?: boolean,
+  ): Promise<Wrapped<T>>;
+  replace<T extends KubernetesObject>(
+    spec: T,
+    pretty?: string,
+    dryRun?: string,
+    fieldManager?: string,
+  ): Promise<Wrapped<T>>;
+  delete(
+    spec: KubernetesObject,
+    pretty?: string,
+    dryRun?: string,
+    gracePeriodSeconds?: number,
+    orphanDependents?: boolean,
+    propagationPolicy?: string,
+    body?: V1DeleteOptions,
+  ): Promise<Wrapped<V1Status>>;
+  list<T extends KubernetesObject>(
+    apiVersion: string,
+    kind: string,
+    namespace?: string,
+    pretty?: string,
+    exact?: boolean,
+    exportt?: boolean,
+    fieldSelector?: string,
+    labelSelector?: string,
+    limit?: number,
+    continueToken?: string,
+  ): Promise<Wrapped<KubernetesListObject<T>>>;
+}
+
+/**
+ * Pre-instantiated `@kubernetes/client-node` API clients sharing the
+ * same recording HTTP transport. Every method call lands on the test
+ * event log as an HTTP event alongside `fetch` calls, and its result is
+ * inspect-wrapped (see {@link Tagged}) so assertions on it stay linked.
+ */
+export interface K3sClient {
+  core: Tagged<CoreV1Api>;
+  apps: Tagged<AppsV1Api>;
+  batch: Tagged<BatchV1Api>;
+  /** Generic object API — `create()`, `read()`, `patch()`, `delete()`,
+   *  `list()` against any Kubernetes resource, custom resources included.
+   *  Generic in the resource type: `objects.read<MyCr>({ … })` resolves to
+   *  `Wrapped<MyCr>`. */
+  objects: TaggedObjectApi;
+}
+
+/** Options for {@link K3sHelpers.apply}. */
+export interface ApplyOptions {
+  /** Namespace for documents that don't declare one (cluster-scoped kinds
+   *  are unaffected). Defaults to `default`, like `kubectl`. */
+  namespace?: string;
+  /** Server-side-apply field manager. Default `"spectest"`. Keep it stable
+   *  across runs — that's what makes re-applying an *update* rather than a
+   *  conflict. */
+  fieldManager?: string;
+  /** Take ownership of fields another manager owns, i.e. `kubectl apply
+   *  --force-conflicts`. Default `true`: a test environment has one owner
+   *  and a conflict is noise. */
+  force?: boolean;
+  /** Total budget for converging the whole manifest set, including retries
+   *  of documents whose CRD/webhook isn't up yet. Default 120 s. */
+  timeoutMs?: number;
+}
+
+/** A workload to wait on: `"web"`, `"deployment/web"`,
+ *  `"statefulset/db"`, `"daemonset/agent"`, or an object with
+ *  `kind`/`metadata` (e.g. an element of {@link K3sHelpers.apply}'s
+ *  result). Bare names are Deployments. */
+export type RolloutTarget = string | KubernetesObject | KubernetesObjectRef;
+
+/** Options for the wait helpers. */
+export interface WaitOptions {
+  /** Namespace of the object. Default `default`, or the object's own. */
+  namespace?: string;
+  /** Give up after this long. Default 120 s. */
+  timeoutMs?: number;
+  /** Poll interval. Default 500 ms. */
+  intervalMs?: number;
+}
+
+/** Which pods to read logs from — see {@link K3sHelpers.logs}. */
+export interface LogsOptions {
+  /** A single pod by name. */
+  pod?: string;
+  /** All pods matching a label selector (`"app=web"`). */
+  selector?: string;
+  namespace?: string;
+  /** Container within the pod (required only for multi-container pods). */
+  container?: string;
+  /** Tail this many lines per pod. Default 200. */
+  tailLines?: number;
+}
+
+/** Helpers a `k3s(...)` service exposes on `ctx.svc.<name>`. */
+export interface K3sHelpers {
+  /**
+   * Fully-loaded `KubeConfig`. The cluster server URL is rewritten to
+   * `https://<service-name>.internal:6443`, the auto-assigned DNS name
+   * for this service on `spectest-net`. TLS verification is off because
+   * Bun's fetch doesn't honor an https.Agent's CA option (the client
+   * cert from the kubeconfig still flows through for auth), so the
+   * server's cert SAN list doesn't need to include the .internal name.
+   */
+  kubeconfig: KubeConfig;
+  /** Pre-built API clients. */
+  client: K3sClient;
+  /**
+   * Apply a (multi-document) YAML manifest with **server-side apply**
+   * semantics — the API equivalent of `kubectl apply --server-side
+   * --force-conflicts`. Each document is `PATCH`ed with
+   * `application/apply-patch+yaml`, so applying is create-or-update:
+   * re-applying an edited manifest converges instead of failing
+   * `AlreadyExists`, which is what makes it usable for real manifest sets
+   * (and for a `dependsOn` child re-applying over its parent's state).
+   *
+   * Documents that can't land *yet* are retried until `opts.timeoutMs`:
+   * a CR whose CRD is in the same manifest, or a resource an admission
+   * webhook rejects while its own pod is still coming up, both resolve on
+   * a later round instead of failing the run. A document that keeps
+   * failing throws with the API server's own message.
+   *
+   * Returns the API server's response objects in input order — each
+   * element inspect-wrapped (the array container itself is plain), so
+   * `expect(applied[0]!.metadata.uid)` links to its apply call.
+   *
+   * Nothing here shells out to `kubectl`; the cluster is driven over its
+   * API from the daemon.
+   */
+  apply(
+    manifest: string,
+    opts?: ApplyOptions,
+  ): Promise<Wrapped<KubernetesObject>[]>;
+  /**
+   * Wait until a Deployment / StatefulSet / DaemonSet has actually rolled
+   * out: the controller has observed the current generation, every replica
+   * is updated, and the desired number are Ready (the checks `kubectl
+   * rollout status` makes). Throws on timeout with the last-seen status
+   * plus pod phases, container reasons and recent Warning events from the
+   * namespace, so a stuck image pull or CrashLoop is readable from the
+   * failure alone.
+   *
+   * ```ts
+   * await ctx.svc.k8s.apply(manifest);
+   * await ctx.svc.k8s.waitForRollout("web");
+   * await ctx.svc.k8s.waitForRollout("statefulset/db", { namespace: "data" });
+   * ```
+   */
+  waitForRollout(target: RolloutTarget, opts?: WaitOptions): Promise<void>;
+  /**
+   * Wait for a Job to finish. Resolves with the completed Job (Complete
+   * condition, or `succeeded` at the requested completion count) and
+   * throws if it fails — with the failed pods' logs attached, which is
+   * the thing you actually want when a migration or seed Job dies.
+   *
+   * ```ts
+   * await ctx.svc.k8s.apply(migrateJob);
+   * const job = await ctx.svc.k8s.waitForJob("migrate");
+   * expect(job.status?.succeeded).toBe(1);
+   * ```
+   */
+  waitForJob(name: string, opts?: WaitOptions): Promise<Wrapped<V1Job>>;
+  /**
+   * Read pod logs — one pod by `pod`, or every pod matching `selector`
+   * (concatenated, each preceded by a `==> <pod> <==` header). The
+   * diagnostic companion to the wait helpers.
+   */
+  logs(opts: LogsOptions): Promise<Wrapped<string>>;
+  /**
+   * List objects of any kind by options object rather than the library's
+   * ten positional parameters (`list(apiVersion, kind, namespace, pretty,
+   * exact, exportt, fieldSelector, …)`). Generic in the resource type, so
+   * custom resources keep their shape:
+   *
+   * ```ts
+   * // "the Branch is gone" — without a 404 throw to catch
+   * const left = await ctx.svc.k8s.list<Branch>({
+   *   apiVersion: "xata.io/v1", kind: "Branch",
+   *   fieldSelector: `metadata.name=${name}`,
+   * });
+   * expect(left.items).toHaveLength(0);
+   * ```
+   */
+  list<T extends KubernetesObject = KubernetesObject>(opts: {
+    apiVersion: string;
+    kind: string;
+    namespace?: string;
+    fieldSelector?: string;
+    labelSelector?: string;
+    limit?: number;
+  }): Promise<Wrapped<KubernetesListObject<T>>>;
+}
+
+interface DockerExecResult {
+  stdout: string;
+  stderr: string;
+  code: number;
+}
+
+function runProcess(
+  cmd: string,
+  args: string[],
+  timeoutMs = 30_000,
+): Promise<DockerExecResult> {
+  return new Promise((resolve, reject) => {
+    const cp = nodeSpawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"] });
+    const out: Buffer[] = [];
+    const err: Buffer[] = [];
+    cp.stdout!.on("data", (c) => out.push(c));
+    cp.stderr!.on("data", (c) => err.push(c));
+    const t = setTimeout(() => cp.kill("SIGKILL"), timeoutMs);
+    cp.on("error", (e) => {
+      clearTimeout(t);
+      reject(e);
+    });
+    cp.on("close", (code) => {
+      clearTimeout(t);
+      resolve({
+        stdout: Buffer.concat(out).toString("utf8"),
+        stderr: Buffer.concat(err).toString("utf8"),
+        code: code ?? -1,
+      });
+    });
+  });
+}
+
+// In-VM root CA, generated once into the base snapshot (see
+// control-plane `base.rs`). Trusted everywhere the test framework runs —
+// Node (`NODE_EXTRA_CA_CERTS`), Chromium (NSS DB), Python, the system
+// store — so a leaf signed by it gives `ctx.fetch`/`ctx.browser()` a
+// clean HTTPS handshake. The k3s `setup` hook runs inside the daemon's
+// Bun process (root in the VM), so it can read the CA key and mint
+// directly. These constants are intentionally redeclared here rather than
+// imported from the daemon: the SDK ships to end users and must not
+// depend on daemon internals.
+const CA_PATH = process.env.SPECTEST_CA_PATH ?? "/etc/spectest/ca.crt";
+const CA_KEY_PATH = process.env.SPECTEST_CA_KEY_PATH ?? "/etc/spectest/ca.key";
+
+function caPresent(): boolean {
+  return existsSync(CA_PATH) && existsSync(CA_KEY_PATH);
+}
+
+/**
+ * Mint a leaf certificate from the in-VM root CA covering `hostnames`
+ * (used here as the SANs of the cluster's wildcard ingress domains).
+ * Returns the cert + key as PEM strings. Self-contained openssl shell-out
+ * — deliberately not shared with the daemon's own cert minting to keep
+ * the distributed SDK decoupled from daemon code.
+ */
+async function issueIngressCert(
+  hostnames: string[],
+): Promise<{ cert: string; key: string }> {
+  const id = `spectest-k3s-ingress-${randomUUID().slice(0, 8)}`;
+  const keyPath = `/tmp/${id}.key`;
+  const crtPath = `/tmp/${id}.crt`;
+  const sans = hostnames.map((h) => `DNS:${h}`).join(",");
+  const r = await runProcess(
+    "openssl",
+    [
+      "req",
+      "-newkey",
+      "rsa:2048",
+      "-nodes",
+      "-keyout",
+      keyPath,
+      "-out",
+      crtPath,
+      "-x509",
+      "-CA",
+      CA_PATH,
+      "-CAkey",
+      CA_KEY_PATH,
+      "-days",
+      "3650",
+      "-subj",
+      "/CN=spectest-k3s-ingress",
+      "-addext",
+      `subjectAltName=${sans}`,
+      "-addext",
+      "basicConstraints=CA:FALSE",
+      "-addext",
+      "extendedKeyUsage=serverAuth",
+      "-addext",
+      "keyUsage=digitalSignature,keyEncipherment",
+    ],
+    30_000,
+  );
+  if (r.code !== 0) {
+    throw new Error(
+      `k3s ingress cert minting failed (openssl rc=${r.code}): ${
+        r.stderr.trim() || r.stdout.trim()
+      }`,
+    );
+  }
+  try {
+    const [cert, key] = await Promise.all([
+      readFile(crtPath, "utf8"),
+      readFile(keyPath, "utf8"),
+    ]);
+    return { cert, key };
+  } finally {
+    await Promise.all([
+      unlink(keyPath).catch(() => {}),
+      unlink(crtPath).catch(() => {}),
+    ]);
+  }
+}
+
+// Holds the inspector `sourceSeq` for the most recent HTTP call inside
+// a single API-method invocation. Filled in by `doFetch` after each
+// request, read by the `withTagging` proxy when the method's promise
+// resolves so the returned parsed object carries the back-reference.
+// AsyncLocalStorage is the right scope here — every call to an Api
+// method runs in its own holder and concurrent calls don't race.
+interface CallSlot {
+  seq?: number;
+}
+const callContext = new AsyncLocalStorage<CallSlot>();
+
+// HTTP transport for `@kubernetes/client-node` that routes through
+// `globalThis.fetch`. Two reasons:
+//   1. The daemon's per-test `installFetchWrapper` already records every
+//      `globalThis.fetch` call as an HTTP event — using fetch here gets
+//      k8s API calls recorded for free, no library-specific wiring.
+//   2. Bun's fetch needs Bun-shaped TLS options (`tls: { ... }`) for
+//      mTLS; node-fetch's `agent` parameter — which the library's
+//      default transport relies on — is silently ignored under Bun.
+//      We honor the lib's auth flow (`KubeConfig.applySecurityAuthentication`
+//      sets an Agent on the request) by extracting cert/key off that
+//      agent and passing them via the Bun-shaped option.
+class FetchHttpLibrary {
+  send(request: RequestContext): Observable<ResponseContext> {
+    const promise = doFetch(request);
+    return new Observable(promise);
+  }
+}
+
+/**
+ * Parsed Kubernetes semantics of a single API request, derived purely
+ * from the HTTP method + request path. Fed to `recorderAnnotate` to
+ * reclassify the generic `http` event the fetch wrapper recorded into a
+ * Kubernetes-specific `kube` event.
+ */
+interface KubeRequestMeta {
+  verb: string;
+  group?: string;
+  apiVersion?: string;
+  resource?: string;
+  subresource?: string;
+  name?: string;
+  namespace?: string;
+}
+
+/**
+ * Map a Kubernetes API request to its `(verb, group/version, resource,
+ * namespace, name, subresource)` from the URL + HTTP method alone.
+ *
+ * Path grammar (the two API roots):
+ *   - core group: `/api/<version>/...`
+ *   - named group: `/apis/<group>/<version>/...`
+ * after which the remainder is either a cluster-scoped resource
+ * (`nodes`, `namespaces`, …) or `namespaces/<ns>/<resource>...`. The
+ * trailing `<resource>[/<name>[/<subresource>]]` shape plus the method
+ * (and `?watch=`) yields the verb.
+ *
+ * Returns `null` for non-resource paths — discovery (`/api`, `/apis`,
+ * `/apis/<group>/<version>`), `/version`, `/healthz`, `/openapi/...` —
+ * so those stay rendered as plain `http`.
+ */
+function describeKubeRequest(
+  method: string,
+  rawUrl: string,
+): KubeRequestMeta | null {
+  let path: string;
+  let query: URLSearchParams;
+  try {
+    const u = new URL(rawUrl);
+    path = u.pathname;
+    query = u.searchParams;
+  } catch {
+    const q = rawUrl.indexOf("?");
+    path = q === -1 ? rawUrl : rawUrl.slice(0, q);
+    query = new URLSearchParams(q === -1 ? "" : rawUrl.slice(q + 1));
+  }
+
+  const segs = path.split("/").filter((s) => s.length > 0);
+  if (segs.length === 0) return null;
+
+  let group: string | undefined;
+  let apiVersion: string | undefined;
+  let rest: string[];
+  if (segs[0] === "api") {
+    group = "";
+    apiVersion = segs[1];
+    rest = segs.slice(2);
+  } else if (segs[0] === "apis") {
+    group = segs[1];
+    apiVersion = segs[2];
+    rest = segs.slice(3);
+  } else {
+    return null; // /version, /healthz, /openapi, …
+  }
+  if (!apiVersion) return null; // discovery root (/api, /apis/<group>)
+
+  // `namespaces/<ns>/<resource>...` is namespaced; everything else
+  // (including `namespaces` and `namespaces/<name>` themselves, and
+  // cluster-scoped resources like `nodes`) is taken as-is.
+  let namespace: string | undefined;
+  let resourcePath = rest;
+  if (rest[0] === "namespaces" && rest.length >= 3) {
+    namespace = rest[1];
+    resourcePath = rest.slice(2);
+  }
+  if (resourcePath.length === 0) return null; // APIResourceList discovery
+
+  const resource = resourcePath[0];
+  const name = resourcePath.length >= 2 ? resourcePath[1] : undefined;
+  const subresource = resourcePath.length >= 3 ? resourcePath[2] : undefined;
+
+  const watchParam = query.get("watch");
+  const watch = watchParam === "true" || watchParam === "1";
+  const hasName = name !== undefined;
+  let verb: string;
+  switch (method.toUpperCase()) {
+    case "GET":
+    case "HEAD":
+      verb = hasName ? "get" : watch ? "watch" : "list";
+      break;
+    case "POST":
+      verb = "create";
+      break;
+    case "PUT":
+      verb = "update";
+      break;
+    case "PATCH":
+      verb = "patch";
+      break;
+    case "DELETE":
+      verb = hasName ? "delete" : "deletecollection";
+      break;
+    default:
+      verb = method.toLowerCase();
+  }
+
+  return { verb, group, apiVersion, resource, subresource, name, namespace };
+}
+
+/**
+ * True for Kubernetes API *discovery* paths — the version/group/resource
+ * enumeration endpoints (`/api`, `/api/<version>`, `/apis`, `/apis/<group>`,
+ * `/apis/<group>/<version>`) the dynamic client hits to resolve a kind to
+ * its resource path. They carry no resource operation (so
+ * `describeKubeRequest` returns null), and `doFetch` retracts their events
+ * from the timeline. Non-resource paths that are NOT discovery (`/healthz`,
+ * `/version`, `/openapi`, …) are deliberately not matched — they stay as
+ * `http`.
+ */
+function isKubeDiscoveryPath(rawUrl: string): boolean {
+  let path: string;
+  try {
+    path = new URL(rawUrl).pathname;
+  } catch {
+    const q = rawUrl.indexOf("?");
+    path = q === -1 ? rawUrl : rawUrl.slice(0, q);
+  }
+  const segs = path.split("/").filter((s) => s.length > 0);
+  if (segs.length === 0) return false;
+  // `/api` + `/api/<version>`; `/apis` + `/apis/<group>` + `/apis/<group>/<version>`.
+  // Anything longer carries a resource segment and is handled as `kube`.
+  if (segs[0] === "api") return segs.length <= 2;
+  if (segs[0] === "apis") return segs.length <= 3;
+  return false;
+}
+
+async function doFetch(request: RequestContext): Promise<ResponseContext> {
+  const url = request.getUrl();
+  const method = String(request.getHttpMethod());
+  const body = request.getBody();
+  const reqHeaders: Record<string, string> = {};
+  for (const [k, v] of Object.entries(request.getHeaders())) {
+    reqHeaders[k] = String(v);
+  }
+
+  // The library's auth flow puts client cert/key on an https.Agent
+  // attached to the request. Pull them out so we can hand them to Bun's
+  // fetch via its `tls` option.
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const agent = request.getAgent() as any;
+  const agentOpts = agent?.options ?? {};
+  const tlsOpts: Record<string, unknown> = { rejectUnauthorized: false };
+  if (agentOpts.cert) tlsOpts.cert = agentOpts.cert;
+  if (agentOpts.key) tlsOpts.key = agentOpts.key;
+
+  const wrapped = await fetch(url, {
+    method,
+    headers: reqHeaders,
+    body: body as BodyInit | undefined,
+    signal: request.getSignal(),
+    // Bun-specific TLS shape; under Node this option is ignored.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    tls: tlsOpts,
+  } as RequestInit);
+
+  // Before unwrapping, capture the inspector tag the daemon's fetch
+  // wrapper installed on the Response. We feed the seq back through
+  // AsyncLocalStorage so the API method's eventual return value can
+  // re-acquire it — otherwise the chain `core.listNode() →
+  // expect(result)` would record assertions with no back-reference.
+  const tag = readTag(wrapped);
+  const slot = callContext.getStore();
+  if (slot && tag) slot.seq = tag.sourceSeq;
+
+  // Reclassify the `http` event the fetch wrapper just recorded into a
+  // Kubernetes-specific `kube` event (verb/resource/namespace/name), so
+  // the timeline reads `list pods · default` rather than the raw API URL.
+  // A `tag` is only present when recording was active for this call, so
+  // this is a no-op outside instrumented test runs.
+  if (tag && tag.sourceSeq !== undefined) {
+    const meta = describeKubeRequest(method, url);
+    if (meta) {
+      recorderAnnotate(tag.sourceSeq, { kind: "kube", ...meta });
+    } else if (isKubeDiscoveryPath(url)) {
+      // The dynamic client (`objects`, KubernetesObjectApi) can't know a
+      // kind's resource path ahead of time, so before the real request it
+      // GETs the group's resource list (`/apis/<group>/<version>` →
+      // APIResourceList) to map e.g. Ingress → `ingresses`/namespaced, then
+      // caches it (apiVersionResourceCache). That discovery GET is library
+      // plumbing the test author never wrote, and because the cache is
+      // per-daemon-process it surfaces non-deterministically across forks
+      // (first access pays it; `dependsOn` children inheriting the warm
+      // cache don't). Retract it rather than leave a bare `http` row — the
+      // real list/read that follows is recorded and reclassified as usual.
+      recorderRemove(tag.sourceSeq);
+    }
+    // Other non-resource paths (/healthz, /version, /openapi, …) fall
+    // through and stay rendered as plain `http`.
+  }
+
+  // The daemon's fetch wrapper proxies `Response.status` and similar
+  // primitives as carrier objects (so test assertions can fold under
+  // the originating HTTP event). The kubernetes/client-node lib calls
+  // `httpStatusCode.toString()` which would then return
+  // "[object Object]" and the status-code dispatch falls through to
+  // "Unknown API Status Code!". Pull out the raw Response.
+  const response =
+    (wrapped as { unwrap?: () => Response }).unwrap?.() ?? wrapped;
+
+  const resHeaders: Record<string, string> = {};
+  response.headers.forEach((v, k) => {
+    resHeaders[k] = v;
+  });
+  const buf = Buffer.from(await response.arrayBuffer());
+  return new ResponseContext(response.status, resHeaders, {
+    text: async () => buf.toString("utf8"),
+    binary: async () => buf,
+  });
+}
+
+/**
+ * Wrap a `@kubernetes/client-node` Api instance so each method call
+ * runs in its own AsyncLocalStorage slot — `doFetch` writes the HTTP
+ * event's `sourceSeq` into the slot, and after the lib parses the
+ * response we re-attach the seq to the returned object. Downstream
+ * `expect(result.items[0].status…)` assertions then fold under that
+ * HTTP event in the test event log, the same way `expect(res.status)`
+ * does for plain `fetch` calls.
+ *
+ * Method arguments are deep-unwrapped on the way in so values pulled
+ * from a previous API response (still carrying the inspector wrappers)
+ * can be passed straight back into another call.
+ *
+ * Non-function properties pass through untagged.
+ */
+function withTagging<T extends object>(api: T): Tagged<T> {
+  return new Proxy(api, {
+    get(target, prop, receiver) {
+      const value = Reflect.get(target, prop, receiver);
+      if (typeof value !== "function") return value;
+      // Bind the original method to `target` so the lib's internal
+      // `this.configuration` accesses keep working.
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const method = (value as any).bind(target);
+      return (...args: unknown[]): unknown => {
+        const unwrappedArgs = args.map(deepUnwrap);
+        const slot: CallSlot = {};
+        const result = callContext.run(slot, () =>
+          method(...unwrappedArgs),
+        );
+        // Wrap unconditionally — `slot.seq` is undefined when no event was
+        // recorded (setup/eval, no active recorder), but the result's type is
+        // wrapped, so the value must be wrapped at runtime too (just without a
+        // provenance link). Keeps `.unwrap()` available in every context.
+        if (result && typeof (result as Promise<unknown>).then === "function") {
+          return (result as Promise<unknown>).then((v) => wrap(v, slot.seq));
+        }
+        return wrap(result, slot.seq);
+      };
+    },
+  }) as unknown as Tagged<T>;
+}
+
+/**
+ * Image tag for the Traefik we install. Pulled on first cluster boot
+ * through the host `zot` mirror (`docker.io` → cache) configured in
+ * `registries.yaml`, then captured into the warm-template snapshot so
+ * warm starts never pull.
+ */
+const TRAEFIK_IMAGE = "rancher/mirrored-library-traefik:3.3.2";
+
+/**
+ * Names of the in-cluster resources that carry the CA-signed default
+ * ingress certificate (created in `setupK3sCluster` when TLS is enabled).
+ * The Secret holds the leaf cert+key; the ConfigMap holds the Traefik
+ * file-provider snippet that points the `default` TLS store at it.
+ */
+const TRAEFIK_TLS_SECRET = "traefik-default-tls";
+const TRAEFIK_DYNAMIC_CONFIGMAP = "traefik-dynamic";
+
+/**
+ * Traefik file-provider dynamic config: make the in-VM-CA leaf the
+ * `default` store certificate, so every router on the `websecure`
+ * entrypoint (which we force to TLS) serves it with no per-Ingress
+ * `spec.tls` needed.
+ */
+const TRAEFIK_DYNAMIC_TLS = `tls:
+  stores:
+    default:
+      defaultCertificate:
+        certFile: /certs/tls.crt
+        keyFile: /certs/tls.key
+`;
+
+/**
+ * Traefik manifest applied during setup(). `hostNetwork: true` puts
+ * Traefik in the k3s container's netns, so it binds the container's :80
+ * (and, with `tls`, :443) directly — no CNI portmap involved (that path
+ * still trips on the kernel's missing xt_comment match).
+ *
+ * When `tls` is set we add a `websecure` :443 entrypoint with TLS forced
+ * on (served from the `default` store, i.e. the in-VM-CA leaf mounted
+ * from the `traefik-default-tls` Secret via the file provider). HTTPS
+ * then works for any routed host under the cluster's `ingressDomains`
+ * with zero per-Ingress config; the :80 `web` entrypoint is unchanged.
+ */
+function buildTraefikManifest(tls: boolean): string {
+  const args = [
+    "        - --entrypoints.web.address=:80",
+    ...(tls
+      ? [
+          "        - --entrypoints.websecure.address=:443",
+          "        - --entrypoints.websecure.http.tls=true",
+        ]
+      : []),
+    "        - --providers.kubernetesingress=true",
+    "        - --providers.kubernetesingress.ingressclass=traefik",
+    ...(tls
+      ? [
+          "        - --providers.file.directory=/dynamic",
+          "        - --providers.file.watch=true",
+        ]
+      : []),
+    "        - --log.level=INFO",
+  ].join("\n");
+  const ports = [
+    "        - name: web",
+    "          containerPort: 80",
+    ...(tls
+      ? ["        - name: websecure", "          containerPort: 443"]
+      : []),
+  ].join("\n");
+  const volumeMounts = tls
+    ? `
+        volumeMounts:
+        - name: default-cert
+          mountPath: /certs
+          readOnly: true
+        - name: dynamic
+          mountPath: /dynamic
+          readOnly: true`
+    : "";
+  const volumes = tls
+    ? `
+      volumes:
+      - name: default-cert
+        secret:
+          secretName: ${TRAEFIK_TLS_SECRET}
+      - name: dynamic
+        configMap:
+          name: ${TRAEFIK_DYNAMIC_CONFIGMAP}`
+    : "";
+  return `apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: traefik
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: traefik
+rules:
+- apiGroups: [""]
+  resources: ["services", "endpoints", "secrets", "nodes"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["discovery.k8s.io"]
+  resources: ["endpointslices"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["networking.k8s.io"]
+  resources: ["ingresses", "ingressclasses"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["networking.k8s.io"]
+  resources: ["ingresses/status"]
+  verbs: ["update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: traefik
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: traefik
+subjects:
+- kind: ServiceAccount
+  name: traefik
+  namespace: kube-system
+---
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  name: traefik
+  annotations:
+    ingressclass.kubernetes.io/is-default-class: "true"
+spec:
+  controller: traefik.io/ingress-controller
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: traefik
+  namespace: kube-system
+  labels:
+    app: traefik
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: traefik
+  template:
+    metadata:
+      labels:
+        app: traefik
+    spec:
+      serviceAccountName: traefik
+      hostNetwork: true
+      dnsPolicy: Default
+      tolerations:
+      - operator: Exists
+      containers:
+      - name: traefik
+        image: ${TRAEFIK_IMAGE}
+        imagePullPolicy: IfNotPresent
+        args:
+${args}
+        ports:
+${ports}${volumeMounts}${volumes}
+`;
+}
+
+/**
+ * Host-side `zot` pull-through cache layout (local Firecracker provider
+ * only). One zot instance per upstream registry, all bound to the
+ * `spectest-br0` gateway `10.42.0.1` on the ports below — **kept in sync
+ * with `scripts/install-zot.sh`**. We mirror the cluster's containerd
+ * through these so every image pull reuses the shared host cache instead
+ * of hitting the public registry, and we list the canonical upstream as
+ * a fallback endpoint so a missing/cold mirror only ever slows a pull,
+ * never breaks it.
+ */
+const ZOT_MIRRORS: Array<{ registry: string; port: number; upstream: string }> = [
+  { registry: "docker.io", port: 5000, upstream: "https://registry-1.docker.io" },
+  { registry: "ghcr.io", port: 5001, upstream: "https://ghcr.io" },
+  { registry: "quay.io", port: 5002, upstream: "https://quay.io" },
+  { registry: "registry.k8s.io", port: 5003, upstream: "https://registry.k8s.io" },
+  { registry: "public.ecr.aws", port: 5004, upstream: "https://public.ecr.aws" },
+  { registry: "gcr.io", port: 5005, upstream: "https://gcr.io" },
+  { registry: "mcr.microsoft.com", port: 5006, upstream: "https://mcr.microsoft.com" },
+];
+
+/**
+ * Discover the host-side image cache gateway by reading the same
+ * `registry-mirrors` entry the in-VM dockerd already uses (baked into
+ * the golden rootfs's `/etc/docker/daemon.json`). Returns the
+ * gateway host (`"10.42.0.1"`) when present, or `null` when there's no
+ * host cache, in which case the cluster pulls every image direct. Runs inside the daemon (VM) at `index.ts` load time, so
+ * the result is stable per host and never poisons the warm-template
+ * cache.
+ */
+function detectHostMirrorGateway(): string | null {
+  try {
+    const cfg = JSON.parse(
+      readFileSync("/etc/docker/daemon.json", "utf8"),
+    ) as { "registry-mirrors"?: string[] };
+    const first = cfg["registry-mirrors"]?.[0];
+    return first ? new URL(first).hostname || null : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Build `/etc/rancher/k3s/registries.yaml`. k3s reads this **once, at
+ * startup**, to configure its embedded containerd — which is why it has
+ * to be seeded via `files` (a pre-start bind mount) rather than a
+ * `setup` hook. Two jobs:
+ *   1. Mirror the cluster's image pulls through the host `zot` cache
+ *      (omitted when there's no host cache).
+ *   2. Trust the in-cluster registry, addressed as `<key>.internal:5000`
+ *      (the `{{SPECTEST_SERVICE}}` token is expanded to the cluster's
+ *      service key when the file is written). Image *references* use
+ *      that peer-reachable name, but containerd pulls via the loopback
+ *      endpoint `http://127.0.0.1:5000` — the hostNetwork registry pod
+ *      shares the node's netns, so this needs no in-container DNS and
+ *      can't be broken by a clobbered `hostnames`.
+ * Returns `null` when there's nothing to configure (no host cache and
+ * `registry` disabled), in which case no file is injected.
+ */
+function buildRegistriesYaml(registryEnabled: boolean): string | null {
+  const gateway = detectHostMirrorGateway();
+  if (!gateway && !registryEnabled) return null;
+
+  const lines: string[] = ["mirrors:"];
+  if (gateway) {
+    for (const { registry, port, upstream } of ZOT_MIRRORS) {
+      lines.push(
+        `  "${registry}":`,
+        `    endpoint:`,
+        `      - "http://${gateway}:${port}"`,
+        `      - "${upstream}"`,
+      );
+    }
+  }
+  if (registryEnabled) {
+    const host = `{{SPECTEST_SERVICE}}.internal:${K3S_REGISTRY_PORT}`;
+    lines.push(
+      `  "${host}":`,
+      `    endpoint:`,
+      `      - "http://127.0.0.1:${K3S_REGISTRY_PORT}"`,
+      "configs:",
+      // The endpoint is plain HTTP; the config (keyed by endpoint host)
+      // makes that explicit and disables any TLS attempt against it.
+      `  "127.0.0.1:${K3S_REGISTRY_PORT}":`,
+      `    tls:`,
+      `      insecure_skip_verify: true`,
+    );
+  }
+  return lines.join("\n") + "\n";
+}
+
+/**
+ * In-cluster OCI registry (CNCF `distribution`). `hostNetwork: true`
+ * binds the cluster container's `:5000` directly — the same trick
+ * Traefik uses — so peer services reach it at `<cluster-key>.internal:5000`
+ * (the cluster service's own alias) and the node's own containerd reaches
+ * it at `127.0.0.1:5000`. Storage is an `emptyDir`, so pushed images live
+ * in the cluster and are captured by snapshot / isolated per test fork
+ * like all other in-VM state.
+ */
+const REGISTRY_MANIFEST = `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: spectest-registry
+  namespace: kube-system
+  labels:
+    app: spectest-registry
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: spectest-registry
+  template:
+    metadata:
+      labels:
+        app: spectest-registry
+    spec:
+      hostNetwork: true
+      dnsPolicy: Default
+      tolerations:
+      - operator: Exists
+      containers:
+      - name: registry
+        image: registry:2
+        imagePullPolicy: IfNotPresent
+        env:
+        - name: REGISTRY_HTTP_ADDR
+          value: ":${K3S_REGISTRY_PORT}"
+        - name: REGISTRY_STORAGE_DELETE_ENABLED
+          value: "true"
+        ports:
+        - name: registry
+          containerPort: ${K3S_REGISTRY_PORT}
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/registry
+      volumes:
+      - name: data
+        emptyDir: {}
+`;
+
+// ────────────────────────────────────────────────────────────────────────
+// Apply — server-side, converging
+//
+// `kubectl apply --server-side --force-conflicts`, over the API. Two
+// properties make a real manifest set work where plain `create` didn't:
+//
+//   * create-or-update. SSA PATCHes with `application/apply-patch+yaml`,
+//     so re-applying an edited manifest converges instead of failing
+//     `AlreadyExists` — which is what a `dependsOn` child re-applying over
+//     its parent's state, or a second apply of a shared base, needs.
+//   * ordering by retry, not by sorting. A CR whose CRD is in the same
+//     manifest, or a resource an admission webhook (cert-manager, CNPG)
+//     rejects while its own pod is still starting, fails the first round
+//     and lands on a later one. Only errors that read as "not yet" are
+//     retried; a genuinely malformed object throws immediately.
+// ────────────────────────────────────────────────────────────────────────
+
+/** Default SSA field manager. Stable across runs on purpose: server-side
+ *  apply keys ownership on it, so a changing value would make every run
+ *  conflict with the previous one's fields. */
+const APPLY_FIELD_MANAGER = "spectest";
+const APPLY_TIMEOUT_MS = 120_000;
+const WAIT_TIMEOUT_MS = 120_000;
+const WAIT_INTERVAL_MS = 500;
+
+/**
+ * API-server responses that mean "this document can't land *yet*" — the
+ * CRD isn't established, an admission webhook's own pod isn't serving,
+ * the namespace is still being created, or the write raced another
+ * writer. Everything else (a schema violation, a bad field, RBAC) is a
+ * real failure and is thrown on the first round.
+ */
+const TRANSIENT_APPLY_PATTERNS: readonly RegExp[] = [
+  /no matches for kind/i,
+  // The client's own message when it can't resolve a kind to a resource
+  // path — i.e. the CRD that defines it isn't established yet. This is
+  // the "CR next to its CRD in one manifest" case, so it MUST be retried.
+  /Failed to fetch resource metadata/i,
+  /could not find the requested resource/i,
+  /the server (?:could not find|does not recognize)/i,
+  /failed calling webhook/i,
+  /webhook .* denied the request: .*(?:not ready|unavailable|connection refused)/i,
+  /connection refused/i,
+  /no endpoints available/i,
+  /context deadline exceeded/i,
+  /etcdserver: (?:leader changed|request timed out)/i,
+  /the object has been modified/i,
+  /namespaces? "[^"]+" not found/i,
+  /Internal error occurred/i,
+  /(?:^|\n)HTTP-Code: (?:429|500|503|504)/,
+];
+
+function errorText(err: unknown): string {
+  return (err as Error)?.message ?? String(err);
+}
+
+function isTransientApplyError(err: unknown): boolean {
+  const msg = errorText(err);
+  return TRANSIENT_APPLY_PATTERNS.some((re) => re.test(msg));
+}
+
+function describeDoc(doc: KubernetesObject): string {
+  const ns = doc.metadata?.namespace;
+  const name = doc.metadata?.name ?? doc.metadata?.generateName ?? "?";
+  return `${doc.kind ?? "?"}/${name}${ns ? ` (ns ${ns})` : ""}`;
+}
+
+/** CRDs and Namespaces go first so the common "CR next to its CRD in one
+ *  file" case lands in round one rather than costing a retry. Everything
+ *  else keeps input order (results are always returned in input order). */
+function applyRank(doc: KubernetesObject): number {
+  return doc.kind === "CustomResourceDefinition" || doc.kind === "Namespace"
+    ? 0
+    : 1;
+}
+
+/**
+ * `KubernetesObjectApi` with its resource-discovery lookup exposed, so
+ * `apply` can tell a namespaced kind from a cluster-scoped one before
+ * defaulting `metadata.namespace` (setting it on a cluster-scoped object
+ * puts a namespace in the body the API server then rejects).
+ */
+class ResourceAwareObjectApi extends KubernetesObjectApi {
+  async resourceInfo(
+    apiVersion: string,
+    kind: string,
+  ): Promise<V1APIResource | undefined> {
+    try {
+      return await this.resource(apiVersion, kind);
+    } catch {
+      // Discovery failure (the CRD isn't established yet) — the caller
+      // treats "unknown" as "leave the document alone"; the apply that
+      // follows fails transiently and is retried once discovery works.
+      return undefined;
+    }
+  }
+}
+
+/** What the apply/wait implementations need from the enclosing cluster. */
+interface ClusterDeps {
+  clusterName: string;
+  client: K3sClient;
+  objects: TaggedObjectApi;
+  meta: ResourceAwareObjectApi;
+  poll: SpectestContext["poll"];
+}
+
+const sleep = (ms: number): Promise<void> =>
+  new Promise((r) => setTimeout(r, ms));
+
+/** Default a document's namespace, but only for kinds that have one. */
+async function withDefaultNamespace(
+  deps: ClusterDeps,
+  doc: KubernetesObject,
+  namespace: string | undefined,
+): Promise<KubernetesObject> {
+  if (!namespace || doc.metadata?.namespace) return doc;
+  const info = await deps.meta.resourceInfo(doc.apiVersion ?? "v1", doc.kind!);
+  // Unknown kind (CRD not established yet) or cluster-scoped: don't touch it.
+  if (!info?.namespaced) return doc;
+  return { ...doc, metadata: { ...doc.metadata, namespace } };
+}
+
+async function applyOne(
+  deps: ClusterDeps,
+  doc: KubernetesObject,
+  opts: { namespace?: string; fieldManager: string; force: boolean },
+): Promise<Wrapped<KubernetesObject>> {
+  const spec = await withDefaultNamespace(deps, doc, opts.namespace);
+  // Server-side apply addresses the object by name; a `generateName`-only
+  // document has none, so it can only ever be created.
+  if (!spec.metadata?.name) {
+    return deps.objects.create(spec, undefined, undefined, opts.fieldManager);
+  }
+  return deps.objects.patch(
+    spec,
+    undefined,
+    undefined,
+    opts.fieldManager,
+    opts.force,
+    PatchStrategy.ServerSideApply,
+  );
+}
+
+async function applyManifest(
+  deps: ClusterDeps,
+  manifest: string,
+  opts?: ApplyOptions,
+): Promise<Wrapped<KubernetesObject>[]> {
+  const docs = (loadAllYaml(manifest) as KubernetesObject[]).filter(
+    (d): d is KubernetesObject =>
+      !!d && typeof d === "object" && typeof d.kind === "string",
+  );
+  const fieldManager = opts?.fieldManager ?? APPLY_FIELD_MANAGER;
+  const force = opts?.force ?? true;
+  const timeoutMs = opts?.timeoutMs ?? APPLY_TIMEOUT_MS;
+  const deadline = Date.now() + timeoutMs;
+  const out: Wrapped<KubernetesObject>[] = new Array(docs.length);
+
+  let pending = docs
+    .map((doc, index) => ({ doc, index }))
+    .sort((a, b) => applyRank(a.doc) - applyRank(b.doc) || a.index - b.index);
+  let backoff = 0;
+  let stalled: string[] = [];
+
+  while (pending.length > 0) {
+    const failed: typeof pending = [];
+    stalled = [];
+    for (const item of pending) {
+      try {
+        out[item.index] = await applyOne(deps, item.doc, {
+          namespace: opts?.namespace,
+          fieldManager,
+          force,
+        });
+      } catch (err) {
+        if (!isTransientApplyError(err)) {
+          throw new Error(
+            `k3s(${deps.clusterName}): applying ${describeDoc(item.doc)} failed: ${errorText(err)}`,
+          );
+        }
+        failed.push(item);
+        stalled.push(`${describeDoc(item.doc)}: ${errorText(err).replace(/\s+/g, " ").slice(0, 300)}`);
+      }
+    }
+    if (failed.length === 0) break;
+    if (Date.now() >= deadline) {
+      throw new Error(
+        `k3s(${deps.clusterName}): apply did not converge within ${timeoutMs / 1000}s — ` +
+          `${failed.length} of ${docs.length} document(s) still failing:\n  ${stalled.join("\n  ")}`,
+      );
+    }
+    // Nothing landed this round: whatever is missing (a webhook pod, a
+    // CRD's establishment) needs wall-clock, so back off before retrying.
+    // Progress resets the backoff — a long CRD chain shouldn't decay.
+    if (failed.length === pending.length) {
+      await sleep(Math.min(2_000, 250 * 2 ** backoff));
+      backoff += 1;
+    } else {
+      backoff = 0;
+    }
+    pending = failed;
+  }
+  return out;
+}
+
+// ────────────────────────────────────────────────────────────────────────
+// Readiness — "is it actually serving?"
+// ────────────────────────────────────────────────────────────────────────
+
+/** A workload reference resolved from a {@link RolloutTarget}. */
+interface WorkloadRef {
+  kind: "Deployment" | "StatefulSet" | "DaemonSet";
+  name: string;
+  namespace: string;
+}
+
+const ROLLOUT_KINDS: Record<string, WorkloadRef["kind"]> = {
+  deployment: "Deployment",
+  deployments: "Deployment",
+  deploy: "Deployment",
+  statefulset: "StatefulSet",
+  statefulsets: "StatefulSet",
+  sts: "StatefulSet",
+  daemonset: "DaemonSet",
+  daemonsets: "DaemonSet",
+  ds: "DaemonSet",
+};
+
+function parseRolloutTarget(
+  target: RolloutTarget,
+  namespace: string | undefined,
+): WorkloadRef {
+  if (typeof target === "string") {
+    const slash = target.indexOf("/");
+    const kindPart = slash === -1 ? "deployment" : target.slice(0, slash);
+    const name = slash === -1 ? target : target.slice(slash + 1);
+    const kind = ROLLOUT_KINDS[kindPart.toLowerCase()];
+    if (!kind) {
+      throw new Error(
+        `waitForRollout("${target}"): unknown workload kind "${kindPart}" — ` +
+          `use deployment, statefulset or daemonset (a bare name is a Deployment).`,
+      );
+    }
+    if (!name) throw new Error(`waitForRollout("${target}"): missing name.`);
+    return { kind, name, namespace: namespace ?? "default" };
+  }
+  const kind = ROLLOUT_KINDS[String(target.kind ?? "").toLowerCase()];
+  const name = target.metadata?.name;
+  if (!kind || !name) {
+    throw new Error(
+      `waitForRollout(): expected a Deployment/StatefulSet/DaemonSet object with metadata.name, got ` +
+        `${JSON.stringify({ kind: target.kind, name })}.`,
+    );
+  }
+  return {
+    kind,
+    name,
+    namespace: namespace ?? target.metadata?.namespace ?? "default",
+  };
+}
+
+/** Rollout state of one workload: settled, or why not. */
+async function rolloutStatus(
+  deps: ClusterDeps,
+  ref: WorkloadRef,
+): Promise<{ done: boolean; reason: string }> {
+  const { name, namespace } = ref;
+  try {
+    if (ref.kind === "Deployment") {
+      const d = (
+        await deps.client.apps.readNamespacedDeployment({ name, namespace })
+      ).unwrap();
+      const want = d.spec?.replicas ?? 1;
+      const st: Partial<V1DeploymentStatus> = d.status ?? {};
+      if ((st.observedGeneration ?? 0) < (d.metadata?.generation ?? 0)) {
+        return { done: false, reason: "controller has not observed the latest update yet" };
+      }
+      if (want === 0) {
+        return {
+          done: (st.replicas ?? 0) === 0,
+          reason: `scaling down: ${st.replicas ?? 0} replica(s) remaining`,
+        };
+      }
+      const updated = st.updatedReplicas ?? 0;
+      const available = st.availableReplicas ?? 0;
+      const total = st.replicas ?? 0;
+      if (updated < want) {
+        return { done: false, reason: `${updated}/${want} replica(s) updated` };
+      }
+      if (total > updated) {
+        return { done: false, reason: `${total - updated} old replica(s) still terminating` };
+      }
+      return {
+        done: available >= want,
+        reason: `${available}/${want} replica(s) available`,
+      };
+    }
+    if (ref.kind === "StatefulSet") {
+      const s = (
+        await deps.client.apps.readNamespacedStatefulSet({ name, namespace })
+      ).unwrap();
+      const want = s.spec?.replicas ?? 1;
+      const st: Partial<V1StatefulSetStatus> = s.status ?? {};
+      if ((st.observedGeneration ?? 0) < (s.metadata?.generation ?? 0)) {
+        return { done: false, reason: "controller has not observed the latest update yet" };
+      }
+      const ready = st.readyReplicas ?? 0;
+      const updated = st.updatedReplicas ?? 0;
+      if (st.updateRevision && st.currentRevision !== st.updateRevision) {
+        return {
+          done: false,
+          reason: `rolling update in progress (${updated}/${want} pod(s) updated)`,
+        };
+      }
+      return { done: ready >= want, reason: `${ready}/${want} pod(s) ready` };
+    }
+    const ds = (
+      await deps.client.apps.readNamespacedDaemonSet({ name, namespace })
+    ).unwrap();
+    const st: Partial<V1DaemonSetStatus> = ds.status ?? {};
+    if ((st.observedGeneration ?? 0) < (ds.metadata?.generation ?? 0)) {
+      return { done: false, reason: "controller has not observed the latest update yet" };
+    }
+    const want = st.desiredNumberScheduled ?? 0;
+    const ready = st.numberReady ?? 0;
+    const updated = st.updatedNumberScheduled ?? 0;
+    if (updated < want) {
+      return { done: false, reason: `${updated}/${want} node(s) updated` };
+    }
+    return { done: want > 0 && ready >= want, reason: `${ready}/${want} node(s) ready` };
+  } catch (err) {
+    const msg = errorText(err);
+    return {
+      done: false,
+      reason: /not found|HTTP-Code: 404/i.test(msg)
+        ? `${ref.kind} ${name} does not exist (yet)`
+        : msg.replace(/\s+/g, " ").slice(0, 300),
+    };
+  }
+}
+
+async function waitForRollout(
+  deps: ClusterDeps,
+  target: RolloutTarget,
+  opts?: WaitOptions,
+): Promise<void> {
+  const ref = parseRolloutTarget(target, opts?.namespace);
+  const timeoutMs = opts?.timeoutMs ?? WAIT_TIMEOUT_MS;
+  let last = "";
+  try {
+    await deps.poll(
+      `${ref.kind.toLowerCase()}/${ref.name} rolled out`,
+      async () => {
+        const st = await rolloutStatus(deps, ref);
+        last = st.reason;
+        return st.done || null;
+      },
+      { timeoutMs, intervalMs: opts?.intervalMs ?? WAIT_INTERVAL_MS },
+    );
+  } catch (err) {
+    const diag = await collectDiagnostics(deps.client, ref.namespace);
+    throw new Error(
+      `k3s(${deps.clusterName}): ${ref.kind}/${ref.name} did not roll out within ` +
+        `${timeoutMs / 1000}s${last ? ` — ${last}` : ""}.\n${diag}\n(${errorText(err)})`,
+    );
+  }
+}
+
+/** Pod names in a namespace, optionally filtered by label selector. */
+async function podNames(
+  client: K3sClient,
+  namespace: string,
+  labelSelector?: string,
+): Promise<string[]> {
+  const pods = (
+    await client.core.listNamespacedPod({
+      namespace,
+      ...(labelSelector ? { labelSelector } : {}),
+    })
+  ).unwrap();
+  return (pods.items ?? [])
+    .map((p) => p.metadata?.name)
+    .filter((n): n is string => !!n);
+}
+
+async function readLogs(
+  client: K3sClient,
+  opts: LogsOptions,
+): Promise<string> {
+  const namespace = opts.namespace ?? "default";
+  const tailLines = opts.tailLines ?? 200;
+  if (!opts.pod && !opts.selector) {
+    throw new Error("ctx.svc.<k3s>.logs(): pass either `pod` or `selector`.");
+  }
+  const names = opts.pod
+    ? [opts.pod]
+    : await podNames(client, namespace, opts.selector);
+  const chunks: string[] = [];
+  for (const name of names) {
+    try {
+      const text = (
+        await client.core.readNamespacedPodLog({
+          name,
+          namespace,
+          tailLines,
+          ...(opts.container ? { container: opts.container } : {}),
+        })
+      ).unwrap();
+      chunks.push(names.length > 1 ? `==> ${name} <==\n${text}` : String(text));
+    } catch (err) {
+      chunks.push(`==> ${name} <==\n(reading logs failed: ${errorText(err)})`);
+    }
+  }
+  return chunks.join("\n");
+}
+
+async function waitForJob(
+  deps: ClusterDeps,
+  name: string,
+  opts?: WaitOptions,
+): Promise<Wrapped<V1Job>> {
+  const namespace = opts?.namespace ?? "default";
+  const timeoutMs = opts?.timeoutMs ?? WAIT_TIMEOUT_MS;
+  let last = "";
+  try {
+    // The predicate returns the RAW job; `poll` wraps the winning value, so
+    // the returned object carries provenance to the wait step (double-
+    // wrapping a proxy would not).
+    return (await deps.poll(
+      `job/${name} completed`,
+      async () => {
+        let job: V1Job;
+        try {
+          job = (
+            await deps.client.batch.readNamespacedJob({ name, namespace })
+          ).unwrap();
+        } catch (err) {
+          const msg = errorText(err);
+          if (!/not found|HTTP-Code: 404/i.test(msg)) throw err;
+          last = `Job ${name} does not exist (yet)`;
+          return null;
+        }
+        const conditions = job.status?.conditions ?? [];
+        const failure = conditions.find(
+          (c) => c.type === "Failed" && c.status === "True",
+        );
+        if (failure) {
+          const logs = await readLogs(deps.client, {
+            namespace,
+            selector: `job-name=${name}`,
+          }).catch((e) => `(reading pod logs failed: ${errorText(e)})`);
+          throw new Error(
+            `k3s(${deps.clusterName}): Job ${name} failed` +
+              `${failure.reason ? ` (${failure.reason})` : ""}` +
+              `${failure.message ? `: ${failure.message}` : ""}\n${logs}`,
+          );
+        }
+        const want = job.spec?.completions ?? 1;
+        const succeeded = job.status?.succeeded ?? 0;
+        const complete = conditions.some(
+          (c) => c.type === "Complete" && c.status === "True",
+        );
+        if (complete || succeeded >= want) return job;
+        last = `${succeeded}/${want} completion(s), ${job.status?.active ?? 0} pod(s) active`;
+        return null;
+      },
+      { timeoutMs, intervalMs: opts?.intervalMs ?? WAIT_INTERVAL_MS },
+    )) as Wrapped<V1Job>;
+  } catch (err) {
+    // A Job that failed already carries its pods' logs — don't bury it.
+    if (/Job .* failed/.test(errorText(err))) throw err;
+    const logs = await readLogs(deps.client, {
+      namespace,
+      selector: `job-name=${name}`,
+    }).catch(() => "");
+    throw new Error(
+      `k3s(${deps.clusterName}): Job ${name} did not complete within ${timeoutMs / 1000}s` +
+        `${last ? ` — ${last}` : ""}.\n${logs}\n(${errorText(err)})`,
+    );
+  }
+}
+
+/**
+ * Post-Ready setup. Apply the Traefik manifest (hostNetwork) and, when
+ * enabled, the in-cluster registry; wait for each Deployment to come
+ * Ready. Captured by the warm-template snapshot, so warm starts pay none
+ * of this cost.
+ *
+ * When the cluster declares `ingressDomains` (and the in-VM CA is
+ * present), TLS is enabled: we mint a CA-signed leaf covering `*.<domain>`
+ * for each domain, stash it in the `traefik-default-tls` Secret + a
+ * file-provider ConfigMap, and bring Traefik up with a `websecure` :443
+ * entrypoint serving it as the default cert. Those domains are then
+ * reachable over HTTPS with a cert the test framework already trusts.
+ */
+async function setupK3sCluster(
+  name: string,
+  helpers: K3sHelpers,
+  opts: { registry: boolean; ingressDomains: string[]; traefik: boolean },
+): Promise<void> {
+  const tlsEnabled = opts.traefik && opts.ingressDomains.length > 0 && caPresent();
+  if (tlsEnabled) {
+    const { cert, key } = await issueIngressCert(
+      opts.ingressDomains.map((d) => `*.${d}`),
+    );
+    // Apply the cert Secret + dynamic-config ConfigMap before the
+    // Deployment that mounts them. `stringData` lets us hand over plain
+    // PEM; the API server base64-encodes it.
+    await helpers.client.core.createNamespacedSecret({
+      namespace: "kube-system",
+      body: {
+        metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
+        type: "kubernetes.io/tls",
+        stringData: { "tls.crt": cert, "tls.key": key },
+      },
+    });
+    await helpers.client.core.createNamespacedConfigMap({
+      namespace: "kube-system",
+      body: {
+        metadata: {
+          name: TRAEFIK_DYNAMIC_CONFIGMAP,
+          namespace: "kube-system",
+        },
+        data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
+      },
+    });
+  }
+  if (opts.traefik) await helpers.apply(buildTraefikManifest(tlsEnabled));
+  if (opts.registry) await helpers.apply(REGISTRY_MANIFEST);
+  // Both rollouts proceed independently inside the cluster — wait on them
+  // concurrently (they used to serialize, wasting up to a rollout's tail).
+  // 250ms polling: these two sit on the cold start's critical path, where
+  // the default 500ms interval wastes real time.
+  const wait = (deployment: string): Promise<void> =>
+    helpers.waitForRollout(deployment, {
+      namespace: "kube-system",
+      timeoutMs: 120_000,
+      intervalMs: 250,
+    });
+  const waits = opts.traefik ? [wait("traefik")] : [];
+  if (opts.registry) waits.push(wait("spectest-registry"));
+  await Promise.all(waits);
+}
+
+/**
+ * Snapshot of a namespace's state, dumped whenever a wait times out: pod
+ * phases, the container reasons/messages behind them, and recent Warning
+ * events. That set answers the question a rollout timeout actually raises —
+ * did the pods schedule, did the image pull, did the container crash — so
+ * the failure is diagnosable without a second round-trip to the cluster.
+ */
+async function collectDiagnostics(
+  client: K3sClient,
+  namespace: string,
+): Promise<string> {
+  const lines: string[] = [];
+  try {
+    const pods = await client.core.listNamespacedPod({ namespace });
+    lines.push(`${namespace} pods (${pods.items.length}):`);
+    for (const p of pods.items) {
+      const phase = p.status?.phase ?? "?";
+      const cs = p.status?.containerStatuses ?? [];
+      const reasons = cs
+        .map((c) => c.state?.waiting?.reason ?? c.state?.terminated?.reason ?? "")
+        .filter((s) => s)
+        .join(",");
+      lines.push(
+        `  ${p.metadata?.name ?? "?"}: phase=${phase}${reasons ? ` reasons=${reasons}` : ""}`,
+      );
+      // The waiting `message` carries containerd's actual error — e.g. the
+      // failing endpoint, an upstream `429 Too Many Requests`, or a
+      // `connection refused`. The `reason` alone (`ErrImagePull`) hides all
+      // of that, which is exactly what we need when a pull won't settle.
+      for (const c of cs) {
+        const msg =
+          c.state?.waiting?.message ?? c.state?.terminated?.message ?? "";
+        if (msg) lines.push(`      ${c.name}: ${msg.replace(/\s+/g, " ").trim()}`);
+      }
+    }
+  } catch (err) {
+    lines.push(`(listing pods failed: ${(err as Error)?.message ?? String(err)})`);
+  }
+  // Recent Warning events surface pull failures the kubelet emits before a
+  // container status even settles (FailedPull / Failed / BackOff), with the
+  // raw containerd message attached. Best-effort: never let diagnostics throw.
+  try {
+    const events = await client.core.listNamespacedEvent({ namespace });
+    const warnings = (events.items ?? [])
+      .filter((e) => e.type === "Warning")
+      .map((e) => ({
+        obj: e.involvedObject?.name ?? "?",
+        reason: e.reason ?? "?",
+        message: (e.message ?? "").replace(/\s+/g, " ").trim(),
+      }))
+      .filter((e) => e.message);
+    if (warnings.length) {
+      lines.push(`${namespace} Warning events (${warnings.length}):`);
+      // Keep the tail — newest events are appended last by the API.
+      for (const w of warnings.slice(-12)) {
+        lines.push(`  ${w.obj} [${w.reason}] ${w.message}`);
+      }
+    }
+  } catch (err) {
+    lines.push(`(listing events failed: ${(err as Error)?.message ?? String(err)})`);
+  }
+  return lines.join("\n");
+}
+
+/**
+ * A ready-to-use single-node Kubernetes cluster (k3s). Drop into
+ * `environment.services`:
+ *
+ * ```ts
+ * services: { k8s: k3s() }
+ * ```
+ *
+ * Tests get `@kubernetes/client-node` API objects pre-wired to this
+ * cluster at `ctx.svc.<key>.client` — `core`, `apps`, and a generic
+ * `objects` (`KubernetesObjectApi`). Every API call is recorded on the
+ * test event log alongside `fetch` calls. There's also `apply(yaml)`
+ * sugar for piping a multi-document manifest in.
+ *
+ * **Ingress.** We deploy Traefik ourselves in `hostNetwork` mode
+ * during `setup()`. Traefik binds the cluster container's :80 directly
+ * (no ServiceLB / klipper-lb needed), watches Ingress objects via the
+ * API, and routes incoming requests to pod Endpoints. Any `hostnames`
+ * declared on this service in env.ts therefore route through Traefik:
+ * a peer doing `fetch("http://app.example.com")` resolves the host to
+ * the k3s container's IP (via systest-resolver), lands on Traefik,
+ * and gets dispatched to the matching Ingress rule's backend pods.
+ *
+ * **kube-proxy runs in `nftables` mode** (`--kube-proxy-arg=proxy-mode
+ * =nftables`), not the iptables default. This started as a workaround
+ * for a stock kernel with no `xt_comment` match — kube-proxy's iptables
+ * rules carry `-m comment`, and the kernel rejected every one of them,
+ * breaking pod→ClusterIP routing and any pod talking to the in-cluster
+ * API (helm-install Jobs, CoreDNS, …). We build the guest kernel
+ * ourselves now and it has the match, but nftables mode is the better
+ * path regardless, so it stays. It is GA in k8s 1.32, which is what
+ * pins `DEFAULT_K3S_VERSION`.
+ *
+ * Flannel uses the `host-gw` backend: single-node clusters never route
+ * pod traffic off the node, so VXLAN encapsulation is pure overhead.
+ */
+/**
+ * Default k3s docker image tag.
+ *
+ * The cluster's system images (the `rancher/k3s` image itself, plus
+ * coredns / local-path-provisioner / pause and the Traefik we deploy)
+ * are pulled on first boot through the host `zot` pull-through cache —
+ * `registries.yaml` (seeded via `files`) mirrors `docker.io` and
+ * `registry.k8s.io` at it. The first-ever cluster boot on a cold-cache
+ * host pays the upstream pull once; thereafter zot serves the blobs
+ * host-wide and the warm-template snapshot captures the booted cluster,
+ * so neither cold-cache nor warm starts re-pull. Any `opts.version`
+ * works — there's no base-snapshot release to keep in sync with.
+ *
+ * **Why v1.32.x:** kube-proxy's `nftables` proxy mode is GA in k8s 1.32
+ * (beta in 1.31, alpha-gated in 1.30). The component runs kube-proxy in
+ * that mode; dropping below 1.31 falls back to the iptables path.
+ */
+const DEFAULT_K3S_VERSION = "v1.32.1-k3s1";
+
+export function k3s(opts: K3sOptions = {}) {
+  const version = opts.version ?? DEFAULT_K3S_VERSION;
+  const extra = opts.extraArgs ?? [];
+  assertUsableExtraArgs(extra);
+  const registryEnabled = opts.registry !== false;
+  const traefikEnabled = opts.traefik !== false;
+  const loadBalancerEnabled = opts.loadBalancer !== false;
+  // Wildcard ingress domains. Drives both the `provides(... dnsName)`
+  // wiring below and (when non-empty) the CA-signed TLS default cert that
+  // setupK3sCluster mints so these domains are reachable over HTTPS.
+  const ingressDomains = opts.ingressDomains ?? [];
+  // `/etc/rancher/k3s/registries.yaml` (host-cache mirrors + trust for
+  // the in-cluster registry). Seeded via `files` because k3s reads it
+  // only at startup, before any setup hook could run.
+  const registriesYaml = buildRegistriesYaml(registryEnabled);
+  const serverArgs = [
+    "k3s",
+    "server",
+    // CoreDNS / pod DNS upstream. Without this, k3s sees only the
+    // loopback 127.0.0.11 (Docker's embedded DNS) in the container's
+    // /etc/resolv.conf, decides no usable nameserver exists, and writes a
+    // fallback `nameserver 8.8.8.8` that CoreDNS then forwards to — so
+    // pods reach the public internet but NOT peer services on
+    // spectest-net (`<svc>.internal`, fakes, service-TLS hosts all
+    // NXDOMAIN). We instead point k3s at the container's default gateway
+    // — the spectest-net bridge gateway, where spectest-resolver binds a
+    // second listener for exactly this. The file is written by the
+    // command wrapper below because the gateway IP is only known at
+    // container start.
+    "--resolv-conf=/run/spectest-resolv.conf",
+    // metrics-server isn't useful in a test cluster.
+    "--disable=metrics-server",
+    // The bundled traefik is disabled because we install our own with
+    // hostNetwork (see setupK3sCluster) — ours is wired to the in-VM CA
+    // for `ingressDomains` TLS and to a fixed IngressClass. Opt out of
+    // ours entirely with `k3s({ traefik: false })`; passing
+    // `--disable=traefik` via `extraArgs` does NOT do that (it only
+    // re-disables the bundled one, which is already off) and is rejected
+    // below rather than silently ignored.
+    "--disable=traefik",
+    // ServiceLB (klipper-lb) stays ENABLED, so `type: LoadBalancer`
+    // Services get an address and work. It was disabled for years
+    // because klipper-lb binds its ports with a CNI portmap hostPort,
+    // and portmap's iptables-nft rules need the `xt_comment` netfilter
+    // match — absent from the stock kernel we used to run on, which made
+    // every LoadBalancer hang at <pending> with no svclb DaemonSet. We
+    // build the guest kernel ourselves now and it carries
+    // CONFIG_NETFILTER_XT_MATCH_COMMENT (see
+    // scripts/local-vms-kernel-additions.config), so the constraint is
+    // gone. local-storage stays enabled too: a controller pod that binds
+    // no host ports.
+    ...(loadBalancerEnabled ? [] : ["--disable=servicelb"]),
+    // Pod CIDR MUST avoid 10.42.0.0/16: that's the spectest-br0 host
+    // bridge subnet, whose gateway 10.42.0.1 fronts the host image caches
+    // (zot :5000-5007, buildkitd :1234). k3s's *default* pod CIDR is also
+    // 10.42.0.0/16 — with --flannel-backend=host-gw, flannel programs that
+    // route into the node's own routing table and gives cni0 the subnet's
+    // .1 (10.42.0.1). That shadows the route to the host gateway, so once
+    // CNI comes up the node can no longer reach 10.42.0.1 and every
+    // subsequent registry pull dies with "connect: connection refused"
+    // (e.g. the in-cluster registry's `registry:2`, applied after the
+    // cluster is up — the airgap-bundled system images pull *before* CNI
+    // and so sneak through). Move pods to 10.44/service to 10.45.
+    "--cluster-cidr=10.44.0.0/16",
+    "--service-cidr=10.45.0.0/16",
+    "--cluster-dns=10.45.0.10",
+    "--flannel-backend=host-gw",
+    "--write-kubeconfig-mode=644",
+    // kube-proxy in nftables mode: native nftables rules, no
+    // xt_comment dependency. Pod→ClusterIP routing works, so
+    // CoreDNS / helm-install / anything-talking-to-the-API works.
+    // GA in k8s 1.32.
+    "--kube-proxy-arg=proxy-mode=nftables",
+    ...extra,
+  ].join(" ");
+  // The service `command` runs under `/bin/sh -c` (see runContainer in
+  // daemon.ts), so derive the bridge gateway from the container's default
+  // route at start time, write it as the k3s resolv-conf, then exec k3s
+  // (exec so it stays the container's main process and signals / the
+  // readyCheck behave exactly as before). `/run` is a tmpfs on this
+  // service, so the file is writable and never persisted into a snapshot.
+  const cmd =
+    "GW=\"$(ip route 2>/dev/null | awk '/^default/{print $3; exit}')\"; " +
+    'if [ -n "$GW" ]; then ' +
+    "printf 'nameserver %s\\noptions ndots:0\\n' \"$GW\" > /run/spectest-resolv.conf; " +
+    "else echo 'spectest: no default gateway found; k3s pod DNS for peer services will not resolve' >&2; " +
+    ": > /run/spectest-resolv.conf; fi; " +
+    // Make the root mount shared. A CSI node plugin bind-mounts volumes
+    // under /var/lib/kubelet and needs that propagation to reach the
+    // kubelet and the workload pod; docker gives a container a private
+    // root, so without this every CSI node plugin fails to publish and
+    // any snapshot/PVC-backed storage test is impossible. `kind` does the
+    // same in its entrypoint for the same reason. Best-effort: on a
+    // kernel/runtime that refuses it, the cluster still boots — only CSI
+    // is affected.
+    "mount --make-rshared / 2>/dev/null || " +
+    "echo 'spectest: could not make / rshared; CSI node plugins may fail to publish volumes' >&2; " +
+    `exec ${serverArgs}`;
+  // Plain /readyz probe. On a warm zot cache the cluster's images are
+  // already local, so the first boot completes in seconds; the
+  // first-ever boot on a cold-cache host pulls through the mirror and
+  // can take a couple of minutes (covered by readyTimeoutSecs).
+  const readyCmd = "kubectl get --raw=/readyz >/dev/null 2>&1";
+  const def = {
+    image: { type: "registry", reference: `rancher/k3s:${version}` },
+    command: cmd,
+    privileged: true,
+    tmpfs: ["/run", "/var/run"],
+    cgroupns: "host",
+    // 80/443 are advisory — peer services and host code reach them via
+    // the k3s container's IP. The component's Traefik binds them directly
+    // (hostNetwork, so it shares the container's netns). 5000 is the
+    // in-cluster registry (hostNetwork pod bound to the container netns),
+    // reached by peers at the cluster's own `<key>.internal:5000` alias.
+    // A `type: LoadBalancer` Service's ports are bound by klipper-lb in
+    // the same netns and are reachable the same way, without appearing
+    // in this list (it's documentation, not a firewall).
+    ports: registryEnabled ? [80, 443, 6443, K3S_REGISTRY_PORT] : [80, 443, 6443],
+    // NOTE: do NOT mount /var/lib/rancher/k3s/agent/containerd as a cache
+    // volume. It was tried (to spare a recreated cluster re-pulling its
+    // system images on delta restores) and a fresh k3s server against the
+    // previous container's containerd store — killed un-cleanly by the
+    // teardown's `docker rm -f` — wedged the apiserver minutes in
+    // (rollouts never settled, pod listing started failing). The zot
+    // mirror already makes those re-pulls cheap; the residual win wasn't
+    // worth the recovery semantics of a crash-state store under a fresh
+    // cluster db.
+    ...(registriesYaml
+      ? {
+          files: [
+            { path: "/etc/rancher/k3s/registries.yaml", content: registriesYaml },
+          ],
+        }
+      : {}),
+    readyCheck: {
+      type: "exec" as const,
+      command: readyCmd,
+      timeoutSecs: opts.readyTimeoutSecs ?? 120,
+    },
+    setup: async ({ name, helpers }: { name: string; helpers: K3sHelpers }) => {
+      await setupK3sCluster(name, helpers, {
+        registry: registryEnabled,
+        ingressDomains,
+        traefik: traefikEnabled,
+      });
+    },
+    helpers: async ({ name, exec, poll }: ServiceHelpersContext): Promise<K3sHelpers> => {
+      // Read the cluster's kubeconfig and address the API server by its
+      // auto-assigned `<name>.internal` hostname on spectest-net. TLS
+      // verification is off (see the K3sHelpers docstring), so the
+      // server's cert SAN list doesn't need to include the .internal
+      // name.
+      const kcRead = await exec(name, ["cat", "/etc/rancher/k3s/k3s.yaml"]);
+      if (kcRead.exitCode !== 0) {
+        throw new Error(
+          `k3s(${name}): failed to read kubeconfig from container: ${kcRead.stderr.trim()}`,
+        );
+      }
+
+      const kubeconfig = new KubeConfig();
+      kubeconfig.loadFromString(kcRead.stdout);
+
+      const server = `https://${name}.internal:6443`;
+      // Update kc.clusters so any code that reads kubeconfig sees the
+      // right server URL, but the actual request server comes from the
+      // Configuration we build below. `Cluster.server` is typed `readonly`
+      // by @kubernetes/client-node, but the loaded object is a plain mutable
+      // record — write through a mutable view rather than rebuild the config.
+      for (const cluster of kubeconfig.clusters) {
+        (cluster as { -readonly [K in keyof typeof cluster]: typeof cluster[K] }).server =
+          server;
+      }
+
+      const httpApi = new FetchHttpLibrary();
+      const baseServer = new ServerConfiguration(server, {});
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const config = createConfiguration({
+        baseServer,
+        authMethods: { default: kubeconfig as any },
+        httpApi: httpApi as any,
+      });
+
+      const core = withTagging(new CoreV1Api(config));
+      const apps = withTagging(new AppsV1Api(config));
+      const batch = withTagging(new BatchV1Api(config));
+      // The generic dynamic API keeps its type parameters (see
+      // {@link TaggedObjectApi}); `withTagging` only rewrites the runtime
+      // values, so the cast re-states what the proxy actually returns.
+      const objectApi = new ResourceAwareObjectApi(config);
+      const objects = withTagging(objectApi) as unknown as TaggedObjectApi;
+
+      const client: K3sClient = { core, apps, batch, objects };
+      const deps: ClusterDeps = {
+        clusterName: name,
+        client,
+        objects,
+        meta: objectApi,
+        poll,
+      };
+
+      return {
+        kubeconfig,
+        client,
+        apply: (manifest, applyOpts) => applyManifest(deps, manifest, applyOpts),
+        waitForRollout: (target, waitOpts) => waitForRollout(deps, target, waitOpts),
+        waitForJob: (jobName, waitOpts) => waitForJob(deps, jobName, waitOpts),
+        logs: async (logOpts) =>
+          wrap(await readLogs(client, logOpts), undefined) as unknown as Wrapped<string>,
+        list: <T extends KubernetesObject = KubernetesObject>(listOpts: {
+          apiVersion: string;
+          kind: string;
+          namespace?: string;
+          fieldSelector?: string;
+          labelSelector?: string;
+          limit?: number;
+        }): Promise<Wrapped<KubernetesListObject<T>>> =>
+          objects.list<T>(
+            listOpts.apiVersion,
+            listOpts.kind,
+            listOpts.namespace,
+            undefined,
+            undefined,
+            undefined,
+            listOpts.fieldSelector,
+            listOpts.labelSelector,
+            listOpts.limit,
+          ),
+      };
+    },
+  } satisfies ServiceDefinition<K3sHelpers>;
+
+  // Wildcard ingress domains → a dnsName(`*.<domain>`, { service: self })
+  // each, attached via provides(). SELF_SERVICE_TOKEN resolves to this
+  // service's key at load time (the component can't know it here). The
+  // resolver then points every host under the domain at the cluster.
+  if (ingressDomains.length === 0) return def;
+  return provides(
+    def,
+    ingressDomains.map((domain) =>
+      dnsName(`*.${domain}`, { service: SELF_SERVICE_TOKEN }),
+    ),
+  );
+}