@specific.dev/spectest 0.26.0 → 0.27.0

package/dist/components/k3s.js

@@ -0,0 +1,1067 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { spawn as nodeSpawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { existsSync, readFileSync } from "node:fs";
+import { readFile, unlink } from "node:fs/promises";
+import { AppsV1Api, CoreV1Api, KubeConfig, KubernetesObjectApi, ResponseContext, ServerConfiguration, createConfiguration, loadAllYaml, } from "@kubernetes/client-node";
+import { Observable } from "@kubernetes/client-node/dist/gen/rxjsStub.js";
+import { dnsName, provides, SELF_SERVICE_TOKEN } from "../index.js";
+import { readRaw, readTag, wrap } from "../inspect.js";
+import { recorderAnnotate, recorderRemove } from "../recorder.js";
+/** Port the in-cluster registry listens on (plain HTTP). */
+const K3S_REGISTRY_PORT = 5000;
+function runProcess(cmd, args, timeoutMs = 30_000) {
+    return new Promise((resolve, reject) => {
+        const cp = nodeSpawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"] });
+        const out = [];
+        const err = [];
+        cp.stdout.on("data", (c) => out.push(c));
+        cp.stderr.on("data", (c) => err.push(c));
+        const t = setTimeout(() => cp.kill("SIGKILL"), timeoutMs);
+        cp.on("error", (e) => {
+            clearTimeout(t);
+            reject(e);
+        });
+        cp.on("close", (code) => {
+            clearTimeout(t);
+            resolve({
+                stdout: Buffer.concat(out).toString("utf8"),
+                stderr: Buffer.concat(err).toString("utf8"),
+                code: code ?? -1,
+            });
+        });
+    });
+}
+// In-VM root CA, generated once into the base snapshot (see
+// control-plane `base.rs`). Trusted everywhere the test framework runs —
+// Node (`NODE_EXTRA_CA_CERTS`), Chromium (NSS DB), Python, the system
+// store — so a leaf signed by it gives `ctx.fetch`/`ctx.browser()` a
+// clean HTTPS handshake. The k3s `setup` hook runs inside the daemon's
+// Bun process (root in the VM), so it can read the CA key and mint
+// directly. These constants are intentionally redeclared here rather than
+// imported from the daemon: the SDK ships to end users and must not
+// depend on daemon internals.
+const CA_PATH = process.env.SPECTEST_CA_PATH ?? "/etc/spectest/ca.crt";
+const CA_KEY_PATH = process.env.SPECTEST_CA_KEY_PATH ?? "/etc/spectest/ca.key";
+function caPresent() {
+    return existsSync(CA_PATH) && existsSync(CA_KEY_PATH);
+}
+/**
+ * Mint a leaf certificate from the in-VM root CA covering `hostnames`
+ * (used here as the SANs of the cluster's wildcard ingress domains).
+ * Returns the cert + key as PEM strings. Self-contained openssl shell-out
+ * — deliberately not shared with the daemon's own cert minting to keep
+ * the distributed SDK decoupled from daemon code.
+ */
+async function issueIngressCert(hostnames) {
+    const id = `spectest-k3s-ingress-${randomUUID().slice(0, 8)}`;
+    const keyPath = `/tmp/${id}.key`;
+    const crtPath = `/tmp/${id}.crt`;
+    const sans = hostnames.map((h) => `DNS:${h}`).join(",");
+    const r = await runProcess("openssl", [
+        "req",
+        "-newkey",
+        "rsa:2048",
+        "-nodes",
+        "-keyout",
+        keyPath,
+        "-out",
+        crtPath,
+        "-x509",
+        "-CA",
+        CA_PATH,
+        "-CAkey",
+        CA_KEY_PATH,
+        "-days",
+        "3650",
+        "-subj",
+        "/CN=spectest-k3s-ingress",
+        "-addext",
+        `subjectAltName=${sans}`,
+        "-addext",
+        "basicConstraints=CA:FALSE",
+        "-addext",
+        "extendedKeyUsage=serverAuth",
+        "-addext",
+        "keyUsage=digitalSignature,keyEncipherment",
+    ], 30_000);
+    if (r.code !== 0) {
+        throw new Error(`k3s ingress cert minting failed (openssl rc=${r.code}): ${r.stderr.trim() || r.stdout.trim()}`);
+    }
+    try {
+        const [cert, key] = await Promise.all([
+            readFile(crtPath, "utf8"),
+            readFile(keyPath, "utf8"),
+        ]);
+        return { cert, key };
+    }
+    finally {
+        await Promise.all([
+            unlink(keyPath).catch(() => { }),
+            unlink(crtPath).catch(() => { }),
+        ]);
+    }
+}
+const callContext = new AsyncLocalStorage();
+// HTTP transport for `@kubernetes/client-node` that routes through
+// `globalThis.fetch`. Two reasons:
+//   1. The daemon's per-test `installFetchWrapper` already records every
+//      `globalThis.fetch` call as an HTTP event — using fetch here gets
+//      k8s API calls recorded for free, no library-specific wiring.
+//   2. Bun's fetch needs Bun-shaped TLS options (`tls: { ... }`) for
+//      mTLS; node-fetch's `agent` parameter — which the library's
+//      default transport relies on — is silently ignored under Bun.
+//      We honor the lib's auth flow (`KubeConfig.applySecurityAuthentication`
+//      sets an Agent on the request) by extracting cert/key off that
+//      agent and passing them via the Bun-shaped option.
+class FetchHttpLibrary {
+    send(request) {
+        const promise = doFetch(request);
+        return new Observable(promise);
+    }
+}
+/**
+ * Map a Kubernetes API request to its `(verb, group/version, resource,
+ * namespace, name, subresource)` from the URL + HTTP method alone.
+ *
+ * Path grammar (the two API roots):
+ *   - core group: `/api/<version>/...`
+ *   - named group: `/apis/<group>/<version>/...`
+ * after which the remainder is either a cluster-scoped resource
+ * (`nodes`, `namespaces`, …) or `namespaces/<ns>/<resource>...`. The
+ * trailing `<resource>[/<name>[/<subresource>]]` shape plus the method
+ * (and `?watch=`) yields the verb.
+ *
+ * Returns `null` for non-resource paths — discovery (`/api`, `/apis`,
+ * `/apis/<group>/<version>`), `/version`, `/healthz`, `/openapi/...` —
+ * so those stay rendered as plain `http`.
+ */
+function describeKubeRequest(method, rawUrl) {
+    let path;
+    let query;
+    try {
+        const u = new URL(rawUrl);
+        path = u.pathname;
+        query = u.searchParams;
+    }
+    catch {
+        const q = rawUrl.indexOf("?");
+        path = q === -1 ? rawUrl : rawUrl.slice(0, q);
+        query = new URLSearchParams(q === -1 ? "" : rawUrl.slice(q + 1));
+    }
+    const segs = path.split("/").filter((s) => s.length > 0);
+    if (segs.length === 0)
+        return null;
+    let group;
+    let apiVersion;
+    let rest;
+    if (segs[0] === "api") {
+        group = "";
+        apiVersion = segs[1];
+        rest = segs.slice(2);
+    }
+    else if (segs[0] === "apis") {
+        group = segs[1];
+        apiVersion = segs[2];
+        rest = segs.slice(3);
+    }
+    else {
+        return null; // /version, /healthz, /openapi, …
+    }
+    if (!apiVersion)
+        return null; // discovery root (/api, /apis/<group>)
+    // `namespaces/<ns>/<resource>...` is namespaced; everything else
+    // (including `namespaces` and `namespaces/<name>` themselves, and
+    // cluster-scoped resources like `nodes`) is taken as-is.
+    let namespace;
+    let resourcePath = rest;
+    if (rest[0] === "namespaces" && rest.length >= 3) {
+        namespace = rest[1];
+        resourcePath = rest.slice(2);
+    }
+    if (resourcePath.length === 0)
+        return null; // APIResourceList discovery
+    const resource = resourcePath[0];
+    const name = resourcePath.length >= 2 ? resourcePath[1] : undefined;
+    const subresource = resourcePath.length >= 3 ? resourcePath[2] : undefined;
+    const watchParam = query.get("watch");
+    const watch = watchParam === "true" || watchParam === "1";
+    const hasName = name !== undefined;
+    let verb;
+    switch (method.toUpperCase()) {
+        case "GET":
+        case "HEAD":
+            verb = hasName ? "get" : watch ? "watch" : "list";
+            break;
+        case "POST":
+            verb = "create";
+            break;
+        case "PUT":
+            verb = "update";
+            break;
+        case "PATCH":
+            verb = "patch";
+            break;
+        case "DELETE":
+            verb = hasName ? "delete" : "deletecollection";
+            break;
+        default:
+            verb = method.toLowerCase();
+    }
+    return { verb, group, apiVersion, resource, subresource, name, namespace };
+}
+/**
+ * True for Kubernetes API *discovery* paths — the version/group/resource
+ * enumeration endpoints (`/api`, `/api/<version>`, `/apis`, `/apis/<group>`,
+ * `/apis/<group>/<version>`) the dynamic client hits to resolve a kind to
+ * its resource path. They carry no resource operation (so
+ * `describeKubeRequest` returns null), and `doFetch` retracts their events
+ * from the timeline. Non-resource paths that are NOT discovery (`/healthz`,
+ * `/version`, `/openapi`, …) are deliberately not matched — they stay as
+ * `http`.
+ */
+function isKubeDiscoveryPath(rawUrl) {
+    let path;
+    try {
+        path = new URL(rawUrl).pathname;
+    }
+    catch {
+        const q = rawUrl.indexOf("?");
+        path = q === -1 ? rawUrl : rawUrl.slice(0, q);
+    }
+    const segs = path.split("/").filter((s) => s.length > 0);
+    if (segs.length === 0)
+        return false;
+    // `/api` + `/api/<version>`; `/apis` + `/apis/<group>` + `/apis/<group>/<version>`.
+    // Anything longer carries a resource segment and is handled as `kube`.
+    if (segs[0] === "api")
+        return segs.length <= 2;
+    if (segs[0] === "apis")
+        return segs.length <= 3;
+    return false;
+}
+async function doFetch(request) {
+    const url = request.getUrl();
+    const method = String(request.getHttpMethod());
+    const body = request.getBody();
+    const reqHeaders = {};
+    for (const [k, v] of Object.entries(request.getHeaders())) {
+        reqHeaders[k] = String(v);
+    }
+    // The library's auth flow puts client cert/key on an https.Agent
+    // attached to the request. Pull them out so we can hand them to Bun's
+    // fetch via its `tls` option.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const agent = request.getAgent();
+    const agentOpts = agent?.options ?? {};
+    const tlsOpts = { rejectUnauthorized: false };
+    if (agentOpts.cert)
+        tlsOpts.cert = agentOpts.cert;
+    if (agentOpts.key)
+        tlsOpts.key = agentOpts.key;
+    const wrapped = await fetch(url, {
+        method,
+        headers: reqHeaders,
+        body: body,
+        signal: request.getSignal(),
+        // Bun-specific TLS shape; under Node this option is ignored.
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        tls: tlsOpts,
+    });
+    // Before unwrapping, capture the inspector tag the daemon's fetch
+    // wrapper installed on the Response. We feed the seq back through
+    // AsyncLocalStorage so the API method's eventual return value can
+    // re-acquire it — otherwise the chain `core.listNode() →
+    // expect(result)` would record assertions with no back-reference.
+    const tag = readTag(wrapped);
+    const slot = callContext.getStore();
+    if (slot && tag)
+        slot.seq = tag.sourceSeq;
+    // Reclassify the `http` event the fetch wrapper just recorded into a
+    // Kubernetes-specific `kube` event (verb/resource/namespace/name), so
+    // the timeline reads `list pods · default` rather than the raw API URL.
+    // A `tag` is only present when recording was active for this call, so
+    // this is a no-op outside instrumented test runs.
+    if (tag && tag.sourceSeq !== undefined) {
+        const meta = describeKubeRequest(method, url);
+        if (meta) {
+            recorderAnnotate(tag.sourceSeq, { kind: "kube", ...meta });
+        }
+        else if (isKubeDiscoveryPath(url)) {
+            // The dynamic client (`objects`, KubernetesObjectApi) can't know a
+            // kind's resource path ahead of time, so before the real request it
+            // GETs the group's resource list (`/apis/<group>/<version>` →
+            // APIResourceList) to map e.g. Ingress → `ingresses`/namespaced, then
+            // caches it (apiVersionResourceCache). That discovery GET is library
+            // plumbing the test author never wrote, and because the cache is
+            // per-daemon-process it surfaces non-deterministically across forks
+            // (first access pays it; `dependsOn` children inheriting the warm
+            // cache don't). Retract it rather than leave a bare `http` row — the
+            // real list/read that follows is recorded and reclassified as usual.
+            recorderRemove(tag.sourceSeq);
+        }
+        // Other non-resource paths (/healthz, /version, /openapi, …) fall
+        // through and stay rendered as plain `http`.
+    }
+    // The daemon's fetch wrapper proxies `Response.status` and similar
+    // primitives as carrier objects (so test assertions can fold under
+    // the originating HTTP event). The kubernetes/client-node lib calls
+    // `httpStatusCode.toString()` which would then return
+    // "[object Object]" and the status-code dispatch falls through to
+    // "Unknown API Status Code!". Pull out the raw Response.
+    const response = wrapped.unwrap?.() ?? wrapped;
+    const resHeaders = {};
+    response.headers.forEach((v, k) => {
+        resHeaders[k] = v;
+    });
+    const buf = Buffer.from(await response.arrayBuffer());
+    return new ResponseContext(response.status, resHeaders, {
+        text: async () => buf.toString("utf8"),
+        binary: async () => buf,
+    });
+}
+/**
+ * Recursively strip the inspector's carrier/proxy wrappers from a
+ * value. Needed for arguments flowing into the kubernetes/client-node
+ * API methods — if a wrapped pod's `metadata.name` (a primitive-carrier
+ * object) reaches a URL template, the lib stringifies it to
+ * `"[object Object]"` and the request 404s.
+ */
+function deepUnwrap(value) {
+    if (value === null || value === undefined)
+        return value;
+    const raw = readRaw(value);
+    if (raw !== value)
+        return deepUnwrap(raw);
+    if (typeof value !== "object")
+        return value;
+    if (Array.isArray(value))
+        return value.map(deepUnwrap);
+    const out = {};
+    for (const [k, v] of Object.entries(value)) {
+        out[k] = deepUnwrap(v);
+    }
+    return out;
+}
+/**
+ * Wrap a `@kubernetes/client-node` Api instance so each method call
+ * runs in its own AsyncLocalStorage slot — `doFetch` writes the HTTP
+ * event's `sourceSeq` into the slot, and after the lib parses the
+ * response we re-attach the seq to the returned object. Downstream
+ * `expect(result.items[0].status…)` assertions then fold under that
+ * HTTP event in the test event log, the same way `expect(res.status)`
+ * does for plain `fetch` calls.
+ *
+ * Method arguments are deep-unwrapped on the way in so values pulled
+ * from a previous API response (still carrying the inspector wrappers)
+ * can be passed straight back into another call.
+ *
+ * Non-function properties pass through untagged.
+ */
+function withTagging(api) {
+    return new Proxy(api, {
+        get(target, prop, receiver) {
+            const value = Reflect.get(target, prop, receiver);
+            if (typeof value !== "function")
+                return value;
+            // Bind the original method to `target` so the lib's internal
+            // `this.configuration` accesses keep working.
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const method = value.bind(target);
+            return (...args) => {
+                const unwrappedArgs = args.map(deepUnwrap);
+                const slot = {};
+                const result = callContext.run(slot, () => method(...unwrappedArgs));
+                // Wrap unconditionally — `slot.seq` is undefined when no event was
+                // recorded (setup/eval, no active recorder), but the result's type is
+                // wrapped, so the value must be wrapped at runtime too (just without a
+                // provenance link). Keeps `.unwrap()` available in every context.
+                if (result && typeof result.then === "function") {
+                    return result.then((v) => wrap(v, slot.seq));
+                }
+                return wrap(result, slot.seq);
+            };
+        },
+    });
+}
+/**
+ * Image tag for the Traefik we install. Pulled on first cluster boot
+ * through the host `zot` mirror (`docker.io` → cache) configured in
+ * `registries.yaml`, then captured into the warm-template snapshot so
+ * warm starts never pull.
+ */
+const TRAEFIK_IMAGE = "rancher/mirrored-library-traefik:3.3.2";
+/**
+ * Names of the in-cluster resources that carry the CA-signed default
+ * ingress certificate (created in `setupK3sCluster` when TLS is enabled).
+ * The Secret holds the leaf cert+key; the ConfigMap holds the Traefik
+ * file-provider snippet that points the `default` TLS store at it.
+ */
+const TRAEFIK_TLS_SECRET = "traefik-default-tls";
+const TRAEFIK_DYNAMIC_CONFIGMAP = "traefik-dynamic";
+/**
+ * Traefik file-provider dynamic config: make the in-VM-CA leaf the
+ * `default` store certificate, so every router on the `websecure`
+ * entrypoint (which we force to TLS) serves it with no per-Ingress
+ * `spec.tls` needed.
+ */
+const TRAEFIK_DYNAMIC_TLS = `tls:
+  stores:
+    default:
+      defaultCertificate:
+        certFile: /certs/tls.crt
+        keyFile: /certs/tls.key
+`;
+/**
+ * Traefik manifest applied during setup(). `hostNetwork: true` puts
+ * Traefik in the k3s container's netns, so it binds the container's :80
+ * (and, with `tls`, :443) directly — no CNI portmap involved (that path
+ * still trips on the kernel's missing xt_comment match).
+ *
+ * When `tls` is set we add a `websecure` :443 entrypoint with TLS forced
+ * on (served from the `default` store, i.e. the in-VM-CA leaf mounted
+ * from the `traefik-default-tls` Secret via the file provider). HTTPS
+ * then works for any routed host under the cluster's `ingressDomains`
+ * with zero per-Ingress config; the :80 `web` entrypoint is unchanged.
+ */
+function buildTraefikManifest(tls) {
+    const args = [
+        "        - --entrypoints.web.address=:80",
+        ...(tls
+            ? [
+                "        - --entrypoints.websecure.address=:443",
+                "        - --entrypoints.websecure.http.tls=true",
+            ]
+            : []),
+        "        - --providers.kubernetesingress=true",
+        "        - --providers.kubernetesingress.ingressclass=traefik",
+        ...(tls
+            ? [
+                "        - --providers.file.directory=/dynamic",
+                "        - --providers.file.watch=true",
+            ]
+            : []),
+        "        - --log.level=INFO",
+    ].join("\n");
+    const ports = [
+        "        - name: web",
+        "          containerPort: 80",
+        ...(tls
+            ? ["        - name: websecure", "          containerPort: 443"]
+            : []),
+    ].join("\n");
+    const volumeMounts = tls
+        ? `
+        volumeMounts:
+        - name: default-cert
+          mountPath: /certs
+          readOnly: true
+        - name: dynamic
+          mountPath: /dynamic
+          readOnly: true`
+        : "";
+    const volumes = tls
+        ? `
+      volumes:
+      - name: default-cert
+        secret:
+          secretName: ${TRAEFIK_TLS_SECRET}
+      - name: dynamic
+        configMap:
+          name: ${TRAEFIK_DYNAMIC_CONFIGMAP}`
+        : "";
+    return `apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: traefik
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: traefik
+rules:
+- apiGroups: [""]
+  resources: ["services", "endpoints", "secrets", "nodes"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["discovery.k8s.io"]
+  resources: ["endpointslices"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["networking.k8s.io"]
+  resources: ["ingresses", "ingressclasses"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["networking.k8s.io"]
+  resources: ["ingresses/status"]
+  verbs: ["update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: traefik
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: traefik
+subjects:
+- kind: ServiceAccount
+  name: traefik
+  namespace: kube-system
+---
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  name: traefik
+  annotations:
+    ingressclass.kubernetes.io/is-default-class: "true"
+spec:
+  controller: traefik.io/ingress-controller
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: traefik
+  namespace: kube-system
+  labels:
+    app: traefik
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: traefik
+  template:
+    metadata:
+      labels:
+        app: traefik
+    spec:
+      serviceAccountName: traefik
+      hostNetwork: true
+      dnsPolicy: Default
+      tolerations:
+      - operator: Exists
+      containers:
+      - name: traefik
+        image: ${TRAEFIK_IMAGE}
+        imagePullPolicy: IfNotPresent
+        args:
+${args}
+        ports:
+${ports}${volumeMounts}${volumes}
+`;
+}
+/**
+ * Host-side `zot` pull-through cache layout (local Firecracker provider
+ * only). One zot instance per upstream registry, all bound to the
+ * `spectest-br0` gateway `10.42.0.1` on the ports below — **kept in sync
+ * with `scripts/install-zot.sh`**. We mirror the cluster's containerd
+ * through these so every image pull reuses the shared host cache instead
+ * of hitting the public registry, and we list the canonical upstream as
+ * a fallback endpoint so a missing/cold mirror only ever slows a pull,
+ * never breaks it.
+ */
+const ZOT_MIRRORS = [
+    { registry: "docker.io", port: 5000, upstream: "https://registry-1.docker.io" },
+    { registry: "ghcr.io", port: 5001, upstream: "https://ghcr.io" },
+    { registry: "quay.io", port: 5002, upstream: "https://quay.io" },
+    { registry: "registry.k8s.io", port: 5003, upstream: "https://registry.k8s.io" },
+    { registry: "public.ecr.aws", port: 5004, upstream: "https://public.ecr.aws" },
+    { registry: "gcr.io", port: 5005, upstream: "https://gcr.io" },
+    { registry: "mcr.microsoft.com", port: 5006, upstream: "https://mcr.microsoft.com" },
+];
+/**
+ * Discover the host-side image cache gateway by reading the same
+ * `registry-mirrors` entry the in-VM dockerd already uses (baked into
+ * the local provider's golden `/etc/docker/daemon.json`). Returns the
+ * gateway host (`"10.42.0.1"`) when present, or `null` when there's no
+ * host cache — e.g. on Freestyle, where the cluster then pulls every
+ * image direct. Runs inside the daemon (VM) at `index.ts` load time, so
+ * the result is stable per host and never poisons the warm-template
+ * cache.
+ */
+function detectHostMirrorGateway() {
+    try {
+        const cfg = JSON.parse(readFileSync("/etc/docker/daemon.json", "utf8"));
+        const first = cfg["registry-mirrors"]?.[0];
+        return first ? new URL(first).hostname || null : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Build `/etc/rancher/k3s/registries.yaml`. k3s reads this **once, at
+ * startup**, to configure its embedded containerd — which is why it has
+ * to be seeded via `files` (a pre-start bind mount) rather than a
+ * `setup` hook. Two jobs:
+ *   1. Mirror the cluster's image pulls through the host `zot` cache
+ *      (local provider only; omitted when there's no host cache).
+ *   2. Trust the in-cluster registry, addressed as `<key>.internal:5000`
+ *      (the `{{SPECTEST_SERVICE}}` token is expanded to the cluster's
+ *      service key when the file is written). Image *references* use
+ *      that peer-reachable name, but containerd pulls via the loopback
+ *      endpoint `http://127.0.0.1:5000` — the hostNetwork registry pod
+ *      shares the node's netns, so this needs no in-container DNS and
+ *      can't be broken by a clobbered `hostnames`.
+ * Returns `null` when there's nothing to configure (no host cache and
+ * `registry` disabled), in which case no file is injected.
+ */
+function buildRegistriesYaml(registryEnabled) {
+    const gateway = detectHostMirrorGateway();
+    if (!gateway && !registryEnabled)
+        return null;
+    const lines = ["mirrors:"];
+    if (gateway) {
+        for (const { registry, port, upstream } of ZOT_MIRRORS) {
+            lines.push(`  "${registry}":`, `    endpoint:`, `      - "http://${gateway}:${port}"`, `      - "${upstream}"`);
+        }
+    }
+    if (registryEnabled) {
+        const host = `{{SPECTEST_SERVICE}}.internal:${K3S_REGISTRY_PORT}`;
+        lines.push(`  "${host}":`, `    endpoint:`, `      - "http://127.0.0.1:${K3S_REGISTRY_PORT}"`, "configs:", 
+        // The endpoint is plain HTTP; the config (keyed by endpoint host)
+        // makes that explicit and disables any TLS attempt against it.
+        `  "127.0.0.1:${K3S_REGISTRY_PORT}":`, `    tls:`, `      insecure_skip_verify: true`);
+    }
+    return lines.join("\n") + "\n";
+}
+/**
+ * In-cluster OCI registry (CNCF `distribution`). `hostNetwork: true`
+ * binds the cluster container's `:5000` directly — the same trick
+ * Traefik uses — so peer services reach it at `<cluster-key>.internal:5000`
+ * (the cluster service's own alias) and the node's own containerd reaches
+ * it at `127.0.0.1:5000`. Storage is an `emptyDir`, so pushed images live
+ * in the cluster and are captured by snapshot / isolated per test fork
+ * like all other in-VM state.
+ */
+const REGISTRY_MANIFEST = `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: spectest-registry
+  namespace: kube-system
+  labels:
+    app: spectest-registry
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: spectest-registry
+  template:
+    metadata:
+      labels:
+        app: spectest-registry
+    spec:
+      hostNetwork: true
+      dnsPolicy: Default
+      tolerations:
+      - operator: Exists
+      containers:
+      - name: registry
+        image: registry:2
+        imagePullPolicy: IfNotPresent
+        env:
+        - name: REGISTRY_HTTP_ADDR
+          value: ":${K3S_REGISTRY_PORT}"
+        - name: REGISTRY_STORAGE_DELETE_ENABLED
+          value: "true"
+        ports:
+        - name: registry
+          containerPort: ${K3S_REGISTRY_PORT}
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/registry
+      volumes:
+      - name: data
+        emptyDir: {}
+`;
+/**
+ * Wait for a Deployment to reach its desired ready-replica count,
+ * polling once a second up to `timeoutMs`. Throws with the last-seen
+ * status (plus kube-system pod diagnostics) on timeout.
+ */
+async function waitForDeployment(clusterName, helpers, deployment, timeoutMs) {
+    const deadline = Date.now() + timeoutMs;
+    let lastErr;
+    while (Date.now() < deadline) {
+        try {
+            // `.unwrap()` recovers the plain object — the client wraps its result in
+            // every context now (provenance-free here, since this internal poll runs
+            // during setup with no active recorder). We're reading for control flow,
+            // not asserting, so go straight to raw.
+            const dep = (await helpers.client.apps.readNamespacedDeployment({
+                name: deployment,
+                namespace: "kube-system",
+            })).unwrap();
+            const ready = dep.status?.readyReplicas ?? 0;
+            const want = dep.spec?.replicas ?? 1;
+            if (ready >= want && want > 0)
+                return;
+            lastErr = `${deployment} Deployment exists but only ${ready}/${want} replicas Ready`;
+        }
+        catch (err) {
+            const msg = err?.message ?? String(err);
+            lastErr = /not found|404/i.test(msg)
+                ? `${deployment} Deployment does not exist yet`
+                : msg;
+        }
+        // 250ms: the two sequential rollout waits in setup sit on the cold
+        // start's critical path, and a 1s poll wasted up to ~2s of it.
+        await new Promise((r) => setTimeout(r, 250));
+    }
+    const diag = await collectTraefikDiagnostics(helpers);
+    throw new Error(`k3s(${clusterName}): ${deployment} did not reach Ready within ${timeoutMs / 1000}s. ${lastErr ?? ""}\n${diag}`);
+}
+/**
+ * Post-Ready setup. Apply the Traefik manifest (hostNetwork) and, when
+ * enabled, the in-cluster registry; wait for each Deployment to come
+ * Ready. Captured by the warm-template snapshot, so warm starts pay none
+ * of this cost.
+ *
+ * When the cluster declares `ingressDomains` (and the in-VM CA is
+ * present), TLS is enabled: we mint a CA-signed leaf covering `*.<domain>`
+ * for each domain, stash it in the `traefik-default-tls` Secret + a
+ * file-provider ConfigMap, and bring Traefik up with a `websecure` :443
+ * entrypoint serving it as the default cert. Those domains are then
+ * reachable over HTTPS with a cert the test framework already trusts.
+ */
+async function setupK3sCluster(name, helpers, opts) {
+    const tlsEnabled = opts.ingressDomains.length > 0 && caPresent();
+    if (tlsEnabled) {
+        const { cert, key } = await issueIngressCert(opts.ingressDomains.map((d) => `*.${d}`));
+        // Apply the cert Secret + dynamic-config ConfigMap before the
+        // Deployment that mounts them. `stringData` lets us hand over plain
+        // PEM; the API server base64-encodes it.
+        await helpers.client.core.createNamespacedSecret({
+            namespace: "kube-system",
+            body: {
+                metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
+                type: "kubernetes.io/tls",
+                stringData: { "tls.crt": cert, "tls.key": key },
+            },
+        });
+        await helpers.client.core.createNamespacedConfigMap({
+            namespace: "kube-system",
+            body: {
+                metadata: {
+                    name: TRAEFIK_DYNAMIC_CONFIGMAP,
+                    namespace: "kube-system",
+                },
+                data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
+            },
+        });
+    }
+    await helpers.apply(buildTraefikManifest(tlsEnabled));
+    if (opts.registry)
+        await helpers.apply(REGISTRY_MANIFEST);
+    // Both rollouts proceed independently inside the cluster — wait on them
+    // concurrently (they used to serialize, wasting up to a rollout's tail).
+    const waits = [waitForDeployment(name, helpers, "traefik", 120_000)];
+    if (opts.registry) {
+        waits.push(waitForDeployment(name, helpers, "spectest-registry", 120_000));
+    }
+    await Promise.all(waits);
+}
+/**
+ * Snapshot of kube-system state, dumped on traefik-wait timeout. With
+ * the static install, the failure surface is just "did our Deployment
+ * schedule and become Ready?" — pod listing covers that.
+ */
+async function collectTraefikDiagnostics(helpers) {
+    const lines = [];
+    try {
+        const pods = await helpers.client.core.listNamespacedPod({
+            namespace: "kube-system",
+        });
+        lines.push(`kube-system pods (${pods.items.length}):`);
+        for (const p of pods.items) {
+            const phase = p.status?.phase ?? "?";
+            const cs = p.status?.containerStatuses ?? [];
+            const reasons = cs
+                .map((c) => c.state?.waiting?.reason ?? c.state?.terminated?.reason ?? "")
+                .filter((s) => s)
+                .join(",");
+            lines.push(`  ${p.metadata?.name ?? "?"}: phase=${phase}${reasons ? ` reasons=${reasons}` : ""}`);
+            // The waiting `message` carries containerd's actual error — e.g. the
+            // failing endpoint, an upstream `429 Too Many Requests`, or a
+            // `connection refused`. The `reason` alone (`ErrImagePull`) hides all
+            // of that, which is exactly what we need when a pull won't settle.
+            for (const c of cs) {
+                const msg = c.state?.waiting?.message ?? c.state?.terminated?.message ?? "";
+                if (msg)
+                    lines.push(`      ${c.name}: ${msg.replace(/\s+/g, " ").trim()}`);
+            }
+        }
+    }
+    catch (err) {
+        lines.push(`(listing pods failed: ${err?.message ?? String(err)})`);
+    }
+    // Recent Warning events surface pull failures the kubelet emits before a
+    // container status even settles (FailedPull / Failed / BackOff), with the
+    // raw containerd message attached. Best-effort: never let diagnostics throw.
+    try {
+        const events = await helpers.client.core.listNamespacedEvent({
+            namespace: "kube-system",
+        });
+        const warnings = (events.items ?? [])
+            .filter((e) => e.type === "Warning")
+            .map((e) => ({
+            obj: e.involvedObject?.name ?? "?",
+            reason: e.reason ?? "?",
+            message: (e.message ?? "").replace(/\s+/g, " ").trim(),
+        }))
+            .filter((e) => e.message);
+        if (warnings.length) {
+            lines.push(`kube-system Warning events (${warnings.length}):`);
+            // Keep the tail — newest events are appended last by the API.
+            for (const w of warnings.slice(-12)) {
+                lines.push(`  ${w.obj} [${w.reason}] ${w.message}`);
+            }
+        }
+    }
+    catch (err) {
+        lines.push(`(listing events failed: ${err?.message ?? String(err)})`);
+    }
+    return lines.join("\n");
+}
+/**
+ * A ready-to-use single-node Kubernetes cluster (k3s). Drop into
+ * `environment.services`:
+ *
+ * ```ts
+ * services: { k8s: k3s() }
+ * ```
+ *
+ * Tests get `@kubernetes/client-node` API objects pre-wired to this
+ * cluster at `ctx.svc.<key>.client` — `core`, `apps`, and a generic
+ * `objects` (`KubernetesObjectApi`). Every API call is recorded on the
+ * test event log alongside `fetch` calls. There's also `apply(yaml)`
+ * sugar for piping a multi-document manifest in.
+ *
+ * **Ingress.** We deploy Traefik ourselves in `hostNetwork` mode
+ * during `setup()`. Traefik binds the cluster container's :80 directly
+ * (no ServiceLB / klipper-lb needed), watches Ingress objects via the
+ * API, and routes incoming requests to pod Endpoints. Any `hostnames`
+ * declared on this service in env.ts therefore route through Traefik:
+ * a peer doing `fetch("http://app.example.com")` resolves the host to
+ * the k3s container's IP (via systest-resolver), lands on Traefik,
+ * and gets dispatched to the matching Ingress rule's backend pods.
+ *
+ * **Workarounds for Freestyle's kernel** (Linux 6.1.0-x-freestyle).
+ * The stock kernel is missing the `xt_comment` netfilter match
+ * extension. Two consequences, each handled below:
+ *
+ *   1. *kube-proxy* in default iptables mode generates rules with
+ *      `-m comment --comment "..."`, which the kernel rejects —
+ *      breaking pod→ClusterIP routing and every pod that talks to the
+ *      in-cluster API (helm-install Jobs, CoreDNS, …). Fixed by
+ *      `--kube-proxy-arg=proxy-mode=nftables`: kube-proxy emits
+ *      native nftables rules where comments are a first-class
+ *      construct, no xt_comment dependency. nftables proxy mode is
+ *      GA in k8s 1.32, which is why we pin that.
+ *
+ *   2. *CNI portmap plugin* (used by klipper-lb's hostPort to expose
+ *      LoadBalancer ports on the host) still uses iptables-nft and
+ *      hits the same xt_comment failure — there's no equivalent
+ *      flag to switch it to native nftables. Workaround: disable the
+ *      bundled traefik + ServiceLB and run Traefik with
+ *      `hostNetwork: true` ourselves. hostNetwork pods don't go
+ *      through portmap at all (they share the node's netns directly),
+ *      so the broken plugin is never invoked.
+ *
+ * Flannel uses the `host-gw` backend because Freestyle's stock kernel
+ * lacks the VXLAN module — fine for single-node clusters.
+ */
+/**
+ * Default k3s docker image tag.
+ *
+ * The cluster's system images (the `rancher/k3s` image itself, plus
+ * coredns / local-path-provisioner / pause and the Traefik we deploy)
+ * are pulled on first boot through the host `zot` pull-through cache —
+ * `registries.yaml` (seeded via `files`) mirrors `docker.io` and
+ * `registry.k8s.io` at it. The first-ever cluster boot on a cold-cache
+ * host pays the upstream pull once; thereafter zot serves the blobs
+ * host-wide and the warm-template snapshot captures the booted cluster,
+ * so neither cold-cache nor warm starts re-pull. Any `opts.version`
+ * works — there's no base-snapshot release to keep in sync with.
+ *
+ * **Why v1.32.x:** kube-proxy's `nftables` proxy mode is GA in k8s 1.32
+ * (beta in 1.31, alpha-gated in 1.30). The component runs kube-proxy in
+ * this mode to sidestep Freestyle's missing `xt_comment` netfilter
+ * extension; dropping below 1.31 reintroduces the broken iptables path.
+ */
+const DEFAULT_K3S_VERSION = "v1.32.1-k3s1";
+export function k3s(opts = {}) {
+    const version = opts.version ?? DEFAULT_K3S_VERSION;
+    const extra = opts.extraArgs ?? [];
+    const registryEnabled = opts.registry !== false;
+    // Wildcard ingress domains. Drives both the `provides(... dnsName)`
+    // wiring below and (when non-empty) the CA-signed TLS default cert that
+    // setupK3sCluster mints so these domains are reachable over HTTPS.
+    const ingressDomains = opts.ingressDomains ?? [];
+    // `/etc/rancher/k3s/registries.yaml` (host-cache mirrors + trust for
+    // the in-cluster registry). Seeded via `files` because k3s reads it
+    // only at startup, before any setup hook could run.
+    const registriesYaml = buildRegistriesYaml(registryEnabled);
+    const serverArgs = [
+        "k3s",
+        "server",
+        // CoreDNS / pod DNS upstream. Without this, k3s sees only the
+        // loopback 127.0.0.11 (Docker's embedded DNS) in the container's
+        // /etc/resolv.conf, decides no usable nameserver exists, and writes a
+        // fallback `nameserver 8.8.8.8` that CoreDNS then forwards to — so
+        // pods reach the public internet but NOT peer services on
+        // spectest-net (`<svc>.internal`, fakes, service-TLS hosts all
+        // NXDOMAIN). We instead point k3s at the container's default gateway
+        // — the spectest-net bridge gateway, where spectest-resolver binds a
+        // second listener for exactly this. The file is written by the
+        // command wrapper below because the gateway IP is only known at
+        // container start.
+        "--resolv-conf=/run/spectest-resolv.conf",
+        // metrics-server isn't useful in a test cluster.
+        "--disable=metrics-server",
+        // traefik + servicelb disabled: their klipper-lb DaemonSet uses
+        // CNI portmap to bind host port 80, which still needs xt_comment
+        // (the iptables compat path that the kernel can't satisfy).
+        // We install Traefik with hostNetwork in setup() — same effect,
+        // no portmap involved. local-storage stays enabled: it's a
+        // controller pod that doesn't bind host ports.
+        "--disable=traefik",
+        "--disable=servicelb",
+        // Pod CIDR MUST avoid 10.42.0.0/16: that's the spectest-br0 host
+        // bridge subnet, whose gateway 10.42.0.1 fronts the host image caches
+        // (zot :5000-5007, buildkitd :1234). k3s's *default* pod CIDR is also
+        // 10.42.0.0/16 — with --flannel-backend=host-gw, flannel programs that
+        // route into the node's own routing table and gives cni0 the subnet's
+        // .1 (10.42.0.1). That shadows the route to the host gateway, so once
+        // CNI comes up the node can no longer reach 10.42.0.1 and every
+        // subsequent registry pull dies with "connect: connection refused"
+        // (e.g. the in-cluster registry's `registry:2`, applied after the
+        // cluster is up — the airgap-bundled system images pull *before* CNI
+        // and so sneak through). Move pods to 10.44/service to 10.45.
+        "--cluster-cidr=10.44.0.0/16",
+        "--service-cidr=10.45.0.0/16",
+        "--cluster-dns=10.45.0.10",
+        "--flannel-backend=host-gw",
+        "--write-kubeconfig-mode=644",
+        // kube-proxy in nftables mode: native nftables rules, no
+        // xt_comment dependency. Pod→ClusterIP routing works, so
+        // CoreDNS / helm-install / anything-talking-to-the-API works.
+        // GA in k8s 1.32.
+        "--kube-proxy-arg=proxy-mode=nftables",
+        ...extra,
+    ].join(" ");
+    // The service `command` runs under `/bin/sh -c` (see runContainer in
+    // daemon.ts), so derive the bridge gateway from the container's default
+    // route at start time, write it as the k3s resolv-conf, then exec k3s
+    // (exec so it stays the container's main process and signals / the
+    // readyCheck behave exactly as before). `/run` is a tmpfs on this
+    // service, so the file is writable and never persisted into a snapshot.
+    const cmd = "GW=\"$(ip route 2>/dev/null | awk '/^default/{print $3; exit}')\"; " +
+        'if [ -n "$GW" ]; then ' +
+        "printf 'nameserver %s\\noptions ndots:0\\n' \"$GW\" > /run/spectest-resolv.conf; " +
+        "else echo 'spectest: no default gateway found; k3s pod DNS for peer services will not resolve' >&2; " +
+        ": > /run/spectest-resolv.conf; fi; " +
+        `exec ${serverArgs}`;
+    // Plain /readyz probe. On a warm zot cache the cluster's images are
+    // already local, so the first boot completes in seconds; the
+    // first-ever boot on a cold-cache host pulls through the mirror and
+    // can take a couple of minutes (covered by readyTimeoutSecs).
+    const readyCmd = "kubectl get --raw=/readyz >/dev/null 2>&1";
+    const def = {
+        image: { type: "registry", reference: `rancher/k3s:${version}` },
+        command: cmd,
+        privileged: true,
+        tmpfs: ["/run", "/var/run"],
+        cgroupns: "host",
+        // 80/443 are advisory — peer services and host code reach them via
+        // the k3s container's IP. ServiceLB (klipper-lb) binds them inside
+        // the container's netns and forwards to the traefik pod. 5000 is the
+        // in-cluster registry (hostNetwork pod bound to the container netns),
+        // reached by peers at the cluster's own `<key>.internal:5000` alias.
+        ports: registryEnabled ? [80, 443, 6443, K3S_REGISTRY_PORT] : [80, 443, 6443],
+        // NOTE: do NOT mount /var/lib/rancher/k3s/agent/containerd as a cache
+        // volume. It was tried (to spare a recreated cluster re-pulling its
+        // system images on delta restores) and a fresh k3s server against the
+        // previous container's containerd store — killed un-cleanly by the
+        // teardown's `docker rm -f` — wedged the apiserver minutes in
+        // (rollouts never settled, pod listing started failing). The zot
+        // mirror already makes those re-pulls cheap; the residual win wasn't
+        // worth the recovery semantics of a crash-state store under a fresh
+        // cluster db.
+        ...(registriesYaml
+            ? {
+                files: [
+                    { path: "/etc/rancher/k3s/registries.yaml", content: registriesYaml },
+                ],
+            }
+            : {}),
+        readyCheck: {
+            type: "exec",
+            command: readyCmd,
+            timeoutSecs: opts.readyTimeoutSecs ?? 120,
+        },
+        setup: async ({ name, helpers }) => {
+            await setupK3sCluster(name, helpers, {
+                registry: registryEnabled,
+                ingressDomains,
+            });
+        },
+        helpers: async ({ name, exec }) => {
+            // Read the cluster's kubeconfig and address the API server by its
+            // auto-assigned `<name>.internal` hostname on spectest-net. TLS
+            // verification is off (see the K3sHelpers docstring), so the
+            // server's cert SAN list doesn't need to include the .internal
+            // name.
+            const kcRead = await exec(name, ["cat", "/etc/rancher/k3s/k3s.yaml"]);
+            if (kcRead.exitCode !== 0) {
+                throw new Error(`k3s(${name}): failed to read kubeconfig from container: ${kcRead.stderr.trim()}`);
+            }
+            const kubeconfig = new KubeConfig();
+            kubeconfig.loadFromString(kcRead.stdout);
+            const server = `https://${name}.internal:6443`;
+            // Update kc.clusters so any code that reads kubeconfig sees the
+            // right server URL, but the actual request server comes from the
+            // Configuration we build below. `Cluster.server` is typed `readonly`
+            // by @kubernetes/client-node, but the loaded object is a plain mutable
+            // record — write through a mutable view rather than rebuild the config.
+            for (const cluster of kubeconfig.clusters) {
+                cluster.server =
+                    server;
+            }
+            const httpApi = new FetchHttpLibrary();
+            const baseServer = new ServerConfiguration(server, {});
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const config = createConfiguration({
+                baseServer,
+                authMethods: { default: kubeconfig },
+                httpApi: httpApi,
+            });
+            const core = withTagging(new CoreV1Api(config));
+            const apps = withTagging(new AppsV1Api(config));
+            const objects = withTagging(new KubernetesObjectApi(config));
+            const apply = async (manifest) => {
+                const docs = loadAllYaml(manifest);
+                const out = [];
+                for (const doc of docs) {
+                    if (!doc || typeof doc !== "object" || !("kind" in doc))
+                        continue;
+                    // `objects.create` is wrapped by `withTagging`, so each returned
+                    // object already carries the back-reference to its create call.
+                    const created = await objects.create(doc);
+                    out.push(created);
+                }
+                return out;
+            };
+            return {
+                kubeconfig,
+                client: { core, apps, objects },
+                apply,
+            };
+        },
+    };
+    // Wildcard ingress domains → a dnsName(`*.<domain>`, { service: self })
+    // each, attached via provides(). SELF_SERVICE_TOKEN resolves to this
+    // service's key at load time (the component can't know it here). The
+    // resolver then points every host under the domain at the cluster.
+    if (ingressDomains.length === 0)
+        return def;
+    return provides(def, ingressDomains.map((domain) => dnsName(`*.${domain}`, { service: SELF_SERVICE_TOKEN })));
+}