@specific.dev/spectest 0.24.0 → 0.27.0

package/src/components/k3s.ts

@@ -1,1312 +0,0 @@
-import { AsyncLocalStorage } from "node:async_hooks";
-import { spawn as nodeSpawn } from "node:child_process";
-import { randomUUID } from "node:crypto";
-import { existsSync, readFileSync } from "node:fs";
-import { readFile, unlink } from "node:fs/promises";
-
-import {
-  AppsV1Api,
-  CoreV1Api,
-  KubeConfig,
-  KubernetesObjectApi,
-  ResponseContext,
-  ServerConfiguration,
-  createConfiguration,
-  loadAllYaml,
-  type KubernetesObject,
-  type RequestContext,
-} from "@kubernetes/client-node";
-import { Observable } from "@kubernetes/client-node/dist/gen/rxjsStub.js";
-
-import type { ServiceDefinition, ServiceHelpersContext } from "../index.js";
-import { dnsName, provides, SELF_SERVICE_TOKEN } from "../index.js";
-import { readRaw, readTag, wrap } from "../inspect.js";
-import type { Wrapped } from "../inspect.js";
-import { recorderAnnotate, recorderRemove } from "../recorder.js";
-
-export interface K3sOptions {
-  /** Image tag for the official `rancher/k3s` image. Default `"v1.30.6-k3s1"`. */
-  version?: string;
-  /**
-   * Extra arguments appended to `k3s server`. Useful for `--tls-san=...`,
-   * additional `--disable=<addon>`, custom CIDRs, etc.
-   */
-  extraArgs?: string[];
-  /**
-   * Readiness probe timeout in seconds. k3s on a warm image is ready in
-   * a few seconds; the first cold start of an env (image pull + cluster
-   * bootstrap) can take 30–60s. Default `120`.
-   */
-  readyTimeoutSecs?: number;
-  /**
-   * Run an in-cluster OCI registry (CNCF `distribution` / `registry:2`)
-   * that the cluster's own containerd trusts. This is the hermetic
-   * stand-in for a cloud registry (ECR/GCR/GHCR): a peer service builds
-   * an image, pushes it here over plain HTTP, references
-   * `<cluster-key>.internal:5000/...` from a Deployment, and the kubelet
-   * pulls it straight back. Lets you test a real
-   * `build → push → deploy → pull` pipeline with no external registry
-   * and no image pre-baking.
-   *
-   * **The registry's push/pull address is `<cluster-key>.internal:5000`**,
-   * where `<cluster-key>` is the key you give this service in the
-   * `services` map (e.g. a cluster at `services.cluster` is reachable at
-   * `cluster.internal:5000`). That's the cluster service's own
-   * unconditional `.internal` alias — peer-reachable and never clobbered
-   * by any `hostnames` you set — so it resolves identically from peer
-   * containers (push) and the cluster's own containerd (pull). Wire it
-   * into your platform, e.g. `env: { REGISTRY_URL: "cluster.internal:5000" }`.
-   * Plain HTTP, so configure your push client for an insecure registry.
-   *
-   * On by default. Set `false` for clusters that only ever run public
-   * images — that skips the extra pod.
-   */
-  registry?: boolean;
-  /**
-   * Domains to route into this cluster's ingress via **wildcard DNS**. For
-   * each `"example.com"`, spectest-resolver answers any `*.example.com`
-   * query with the cluster container's IP, where Traefik dispatches by Host
-   * to the matching Ingress. This lets a test `kubectl apply` an Ingress for
-   * any host under the domain and reach it immediately — no need to
-   * pre-declare each hostname in `hostnames`.
-   *
-   * ```ts
-   * services: { k8s: k3s({ ingressDomains: ["example.com"] }) }
-   * // a test then applies an Ingress for foo.example.com and fetches it.
-   * ```
-   *
-   * For one-off hosts not under a declared domain, a test can also register
-   * dynamically with `ctx.dnsName(host, { service: "k8s" })`.
-   *
-   * **TLS.** Setting `ingressDomains` also makes those domains reachable
-   * over **HTTPS**: Traefik gains a `:443` entrypoint and serves a default
-   * certificate minted from the in-VM root CA with SANs `*.<domain>` for
-   * each declared domain. The CA is already trusted by the test framework
-   * (Node `fetch`, `ctx.browser()`, Python, the system store), so
-   * `ctx.fetch("https://foo.example.com")` gets a clean handshake — no
-   * per-Ingress `spec.tls` and no `--insecure` needed. Only hosts **under**
-   * a declared domain are covered by the cert; static `hostnames` not under
-   * one (and one-off `ctx.dnsName` hosts) remain HTTP-only.
-   */
-  ingressDomains?: string[];
-}
-
-/** Port the in-cluster registry listens on (plain HTTP). */
-const K3S_REGISTRY_PORT = 5000;
-
-/**
- * Rewrites a `@kubernetes/client-node` API class so each method's resolved
- * value comes back **inspect-wrapped** ({@link Wrapped}) — exactly what
- * `withTagging` does at runtime. The method *signatures* (argument types) are
- * untouched; only the `Promise<R>` result becomes `Promise<Wrapped<R>>`, so
- * `expect(pod.status.phase)` links to the API call with no cast and you
- * `.unwrap()` before using a value as raw data. Non-method
- * members pass through unchanged. Mirrors {@link RecordingSqlClient} on the
- * postgres side.
- */
-type Tagged<T> = {
-  [K in keyof T]: T[K] extends (...args: infer A) => Promise<infer R>
-    ? (...args: A) => Promise<Wrapped<R>>
-    : T[K];
-};
-
-/**
- * Pre-instantiated `@kubernetes/client-node` API clients sharing the
- * same recording HTTP transport. Every method call lands on the test
- * event log as an HTTP event alongside `fetch` calls, and its result is
- * inspect-wrapped (see {@link Tagged}) so assertions on it stay linked.
- */
-export interface K3sClient {
-  core: Tagged<CoreV1Api>;
-  apps: Tagged<AppsV1Api>;
-  /** Generic object API — `create()`, `read()`, `patch()`, `delete()`
-   *  against any Kubernetes resource (custom resources included). */
-  objects: Tagged<KubernetesObjectApi>;
-}
-
-/** Helpers a `k3s(...)` service exposes on `ctx.svc.<name>`. */
-export interface K3sHelpers {
-  /**
-   * Fully-loaded `KubeConfig`. The cluster server URL is rewritten to
-   * `https://<service-name>.internal:6443`, the auto-assigned DNS name
-   * for this service on `spectest-net`. TLS verification is off because
-   * Bun's fetch doesn't honor an https.Agent's CA option (the client
-   * cert from the kubeconfig still flows through for auth), so the
-   * server's cert SAN list doesn't need to include the .internal name.
-   */
-  kubeconfig: KubeConfig;
-  /** Pre-built API clients. */
-  client: K3sClient;
-  /**
-   * Apply a (multi-document) YAML manifest. Each parsed document is
-   * created via `KubernetesObjectApi.create`. Returns the API server's
-   * response objects in input order — each element inspect-wrapped (the
-   * array container itself is plain), so `expect(created[0]!.metadata.uid)`
-   * links to its create call.
-   */
-  apply: (manifest: string) => Promise<Wrapped<KubernetesObject>[]>;
-}
-
-interface DockerExecResult {
-  stdout: string;
-  stderr: string;
-  code: number;
-}
-
-function runProcess(
-  cmd: string,
-  args: string[],
-  timeoutMs = 30_000,
-): Promise<DockerExecResult> {
-  return new Promise((resolve, reject) => {
-    const cp = nodeSpawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"] });
-    const out: Buffer[] = [];
-    const err: Buffer[] = [];
-    cp.stdout!.on("data", (c) => out.push(c));
-    cp.stderr!.on("data", (c) => err.push(c));
-    const t = setTimeout(() => cp.kill("SIGKILL"), timeoutMs);
-    cp.on("error", (e) => {
-      clearTimeout(t);
-      reject(e);
-    });
-    cp.on("close", (code) => {
-      clearTimeout(t);
-      resolve({
-        stdout: Buffer.concat(out).toString("utf8"),
-        stderr: Buffer.concat(err).toString("utf8"),
-        code: code ?? -1,
-      });
-    });
-  });
-}
-
-// In-VM root CA, generated once into the base snapshot (see
-// control-plane `base.rs`). Trusted everywhere the test framework runs —
-// Node (`NODE_EXTRA_CA_CERTS`), Chromium (NSS DB), Python, the system
-// store — so a leaf signed by it gives `ctx.fetch`/`ctx.browser()` a
-// clean HTTPS handshake. The k3s `setup` hook runs inside the daemon's
-// Bun process (root in the VM), so it can read the CA key and mint
-// directly. These constants are intentionally redeclared here rather than
-// imported from the daemon: the SDK ships to end users and must not
-// depend on daemon internals.
-const CA_PATH = process.env.SPECTEST_CA_PATH ?? "/etc/spectest/ca.crt";
-const CA_KEY_PATH = process.env.SPECTEST_CA_KEY_PATH ?? "/etc/spectest/ca.key";
-
-function caPresent(): boolean {
-  return existsSync(CA_PATH) && existsSync(CA_KEY_PATH);
-}
-
-/**
- * Mint a leaf certificate from the in-VM root CA covering `hostnames`
- * (used here as the SANs of the cluster's wildcard ingress domains).
- * Returns the cert + key as PEM strings. Self-contained openssl shell-out
- * — deliberately not shared with the daemon's own cert minting to keep
- * the distributed SDK decoupled from daemon code.
- */
-async function issueIngressCert(
-  hostnames: string[],
-): Promise<{ cert: string; key: string }> {
-  const id = `spectest-k3s-ingress-${randomUUID().slice(0, 8)}`;
-  const keyPath = `/tmp/${id}.key`;
-  const crtPath = `/tmp/${id}.crt`;
-  const sans = hostnames.map((h) => `DNS:${h}`).join(",");
-  const r = await runProcess(
-    "openssl",
-    [
-      "req",
-      "-newkey",
-      "rsa:2048",
-      "-nodes",
-      "-keyout",
-      keyPath,
-      "-out",
-      crtPath,
-      "-x509",
-      "-CA",
-      CA_PATH,
-      "-CAkey",
-      CA_KEY_PATH,
-      "-days",
-      "3650",
-      "-subj",
-      "/CN=spectest-k3s-ingress",
-      "-addext",
-      `subjectAltName=${sans}`,
-      "-addext",
-      "basicConstraints=CA:FALSE",
-      "-addext",
-      "extendedKeyUsage=serverAuth",
-      "-addext",
-      "keyUsage=digitalSignature,keyEncipherment",
-    ],
-    30_000,
-  );
-  if (r.code !== 0) {
-    throw new Error(
-      `k3s ingress cert minting failed (openssl rc=${r.code}): ${
-        r.stderr.trim() || r.stdout.trim()
-      }`,
-    );
-  }
-  try {
-    const [cert, key] = await Promise.all([
-      readFile(crtPath, "utf8"),
-      readFile(keyPath, "utf8"),
-    ]);
-    return { cert, key };
-  } finally {
-    await Promise.all([
-      unlink(keyPath).catch(() => {}),
-      unlink(crtPath).catch(() => {}),
-    ]);
-  }
-}
-
-// Holds the inspector `sourceSeq` for the most recent HTTP call inside
-// a single API-method invocation. Filled in by `doFetch` after each
-// request, read by the `withTagging` proxy when the method's promise
-// resolves so the returned parsed object carries the back-reference.
-// AsyncLocalStorage is the right scope here — every call to an Api
-// method runs in its own holder and concurrent calls don't race.
-interface CallSlot {
-  seq?: number;
-}
-const callContext = new AsyncLocalStorage<CallSlot>();
-
-// HTTP transport for `@kubernetes/client-node` that routes through
-// `globalThis.fetch`. Two reasons:
-//   1. The daemon's per-test `installFetchWrapper` already records every
-//      `globalThis.fetch` call as an HTTP event — using fetch here gets
-//      k8s API calls recorded for free, no library-specific wiring.
-//   2. Bun's fetch needs Bun-shaped TLS options (`tls: { ... }`) for
-//      mTLS; node-fetch's `agent` parameter — which the library's
-//      default transport relies on — is silently ignored under Bun.
-//      We honor the lib's auth flow (`KubeConfig.applySecurityAuthentication`
-//      sets an Agent on the request) by extracting cert/key off that
-//      agent and passing them via the Bun-shaped option.
-class FetchHttpLibrary {
-  send(request: RequestContext): Observable<ResponseContext> {
-    const promise = doFetch(request);
-    return new Observable(promise);
-  }
-}
-
-/**
- * Parsed Kubernetes semantics of a single API request, derived purely
- * from the HTTP method + request path. Fed to `recorderAnnotate` to
- * reclassify the generic `http` event the fetch wrapper recorded into a
- * Kubernetes-specific `kube` event.
- */
-interface KubeRequestMeta {
-  verb: string;
-  group?: string;
-  apiVersion?: string;
-  resource?: string;
-  subresource?: string;
-  name?: string;
-  namespace?: string;
-}
-
-/**
- * Map a Kubernetes API request to its `(verb, group/version, resource,
- * namespace, name, subresource)` from the URL + HTTP method alone.
- *
- * Path grammar (the two API roots):
- *   - core group: `/api/<version>/...`
- *   - named group: `/apis/<group>/<version>/...`
- * after which the remainder is either a cluster-scoped resource
- * (`nodes`, `namespaces`, …) or `namespaces/<ns>/<resource>...`. The
- * trailing `<resource>[/<name>[/<subresource>]]` shape plus the method
- * (and `?watch=`) yields the verb.
- *
- * Returns `null` for non-resource paths — discovery (`/api`, `/apis`,
- * `/apis/<group>/<version>`), `/version`, `/healthz`, `/openapi/...` —
- * so those stay rendered as plain `http`.
- */
-function describeKubeRequest(
-  method: string,
-  rawUrl: string,
-): KubeRequestMeta | null {
-  let path: string;
-  let query: URLSearchParams;
-  try {
-    const u = new URL(rawUrl);
-    path = u.pathname;
-    query = u.searchParams;
-  } catch {
-    const q = rawUrl.indexOf("?");
-    path = q === -1 ? rawUrl : rawUrl.slice(0, q);
-    query = new URLSearchParams(q === -1 ? "" : rawUrl.slice(q + 1));
-  }
-
-  const segs = path.split("/").filter((s) => s.length > 0);
-  if (segs.length === 0) return null;
-
-  let group: string | undefined;
-  let apiVersion: string | undefined;
-  let rest: string[];
-  if (segs[0] === "api") {
-    group = "";
-    apiVersion = segs[1];
-    rest = segs.slice(2);
-  } else if (segs[0] === "apis") {
-    group = segs[1];
-    apiVersion = segs[2];
-    rest = segs.slice(3);
-  } else {
-    return null; // /version, /healthz, /openapi, …
-  }
-  if (!apiVersion) return null; // discovery root (/api, /apis/<group>)
-
-  // `namespaces/<ns>/<resource>...` is namespaced; everything else
-  // (including `namespaces` and `namespaces/<name>` themselves, and
-  // cluster-scoped resources like `nodes`) is taken as-is.
-  let namespace: string | undefined;
-  let resourcePath = rest;
-  if (rest[0] === "namespaces" && rest.length >= 3) {
-    namespace = rest[1];
-    resourcePath = rest.slice(2);
-  }
-  if (resourcePath.length === 0) return null; // APIResourceList discovery
-
-  const resource = resourcePath[0];
-  const name = resourcePath.length >= 2 ? resourcePath[1] : undefined;
-  const subresource = resourcePath.length >= 3 ? resourcePath[2] : undefined;
-
-  const watchParam = query.get("watch");
-  const watch = watchParam === "true" || watchParam === "1";
-  const hasName = name !== undefined;
-  let verb: string;
-  switch (method.toUpperCase()) {
-    case "GET":
-    case "HEAD":
-      verb = hasName ? "get" : watch ? "watch" : "list";
-      break;
-    case "POST":
-      verb = "create";
-      break;
-    case "PUT":
-      verb = "update";
-      break;
-    case "PATCH":
-      verb = "patch";
-      break;
-    case "DELETE":
-      verb = hasName ? "delete" : "deletecollection";
-      break;
-    default:
-      verb = method.toLowerCase();
-  }
-
-  return { verb, group, apiVersion, resource, subresource, name, namespace };
-}
-
-/**
- * True for Kubernetes API *discovery* paths — the version/group/resource
- * enumeration endpoints (`/api`, `/api/<version>`, `/apis`, `/apis/<group>`,
- * `/apis/<group>/<version>`) the dynamic client hits to resolve a kind to
- * its resource path. They carry no resource operation (so
- * `describeKubeRequest` returns null), and `doFetch` retracts their events
- * from the timeline. Non-resource paths that are NOT discovery (`/healthz`,
- * `/version`, `/openapi`, …) are deliberately not matched — they stay as
- * `http`.
- */
-function isKubeDiscoveryPath(rawUrl: string): boolean {
-  let path: string;
-  try {
-    path = new URL(rawUrl).pathname;
-  } catch {
-    const q = rawUrl.indexOf("?");
-    path = q === -1 ? rawUrl : rawUrl.slice(0, q);
-  }
-  const segs = path.split("/").filter((s) => s.length > 0);
-  if (segs.length === 0) return false;
-  // `/api` + `/api/<version>`; `/apis` + `/apis/<group>` + `/apis/<group>/<version>`.
-  // Anything longer carries a resource segment and is handled as `kube`.
-  if (segs[0] === "api") return segs.length <= 2;
-  if (segs[0] === "apis") return segs.length <= 3;
-  return false;
-}
-
-async function doFetch(request: RequestContext): Promise<ResponseContext> {
-  const url = request.getUrl();
-  const method = String(request.getHttpMethod());
-  const body = request.getBody();
-  const reqHeaders: Record<string, string> = {};
-  for (const [k, v] of Object.entries(request.getHeaders())) {
-    reqHeaders[k] = String(v);
-  }
-
-  // The library's auth flow puts client cert/key on an https.Agent
-  // attached to the request. Pull them out so we can hand them to Bun's
-  // fetch via its `tls` option.
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  const agent = request.getAgent() as any;
-  const agentOpts = agent?.options ?? {};
-  const tlsOpts: Record<string, unknown> = { rejectUnauthorized: false };
-  if (agentOpts.cert) tlsOpts.cert = agentOpts.cert;
-  if (agentOpts.key) tlsOpts.key = agentOpts.key;
-
-  const wrapped = await fetch(url, {
-    method,
-    headers: reqHeaders,
-    body: body as BodyInit | undefined,
-    signal: request.getSignal(),
-    // Bun-specific TLS shape; under Node this option is ignored.
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    tls: tlsOpts,
-  } as RequestInit);
-
-  // Before unwrapping, capture the inspector tag the daemon's fetch
-  // wrapper installed on the Response. We feed the seq back through
-  // AsyncLocalStorage so the API method's eventual return value can
-  // re-acquire it — otherwise the chain `core.listNode() →
-  // expect(result)` would record assertions with no back-reference.
-  const tag = readTag(wrapped);
-  const slot = callContext.getStore();
-  if (slot && tag) slot.seq = tag.sourceSeq;
-
-  // Reclassify the `http` event the fetch wrapper just recorded into a
-  // Kubernetes-specific `kube` event (verb/resource/namespace/name), so
-  // the timeline reads `list pods · default` rather than the raw API URL.
-  // A `tag` is only present when recording was active for this call, so
-  // this is a no-op outside instrumented test runs.
-  if (tag && tag.sourceSeq !== undefined) {
-    const meta = describeKubeRequest(method, url);
-    if (meta) {
-      recorderAnnotate(tag.sourceSeq, { kind: "kube", ...meta });
-    } else if (isKubeDiscoveryPath(url)) {
-      // The dynamic client (`objects`, KubernetesObjectApi) can't know a
-      // kind's resource path ahead of time, so before the real request it
-      // GETs the group's resource list (`/apis/<group>/<version>` →
-      // APIResourceList) to map e.g. Ingress → `ingresses`/namespaced, then
-      // caches it (apiVersionResourceCache). That discovery GET is library
-      // plumbing the test author never wrote, and because the cache is
-      // per-daemon-process it surfaces non-deterministically across forks
-      // (first access pays it; `dependsOn` children inheriting the warm
-      // cache don't). Retract it rather than leave a bare `http` row — the
-      // real list/read that follows is recorded and reclassified as usual.
-      recorderRemove(tag.sourceSeq);
-    }
-    // Other non-resource paths (/healthz, /version, /openapi, …) fall
-    // through and stay rendered as plain `http`.
-  }
-
-  // The daemon's fetch wrapper proxies `Response.status` and similar
-  // primitives as carrier objects (so test assertions can fold under
-  // the originating HTTP event). The kubernetes/client-node lib calls
-  // `httpStatusCode.toString()` which would then return
-  // "[object Object]" and the status-code dispatch falls through to
-  // "Unknown API Status Code!". Pull out the raw Response.
-  const response =
-    (wrapped as { unwrap?: () => Response }).unwrap?.() ?? wrapped;
-
-  const resHeaders: Record<string, string> = {};
-  response.headers.forEach((v, k) => {
-    resHeaders[k] = v;
-  });
-  const buf = Buffer.from(await response.arrayBuffer());
-  return new ResponseContext(response.status, resHeaders, {
-    text: async () => buf.toString("utf8"),
-    binary: async () => buf,
-  });
-}
-
-/**
- * Recursively strip the inspector's carrier/proxy wrappers from a
- * value. Needed for arguments flowing into the kubernetes/client-node
- * API methods — if a wrapped pod's `metadata.name` (a primitive-carrier
- * object) reaches a URL template, the lib stringifies it to
- * `"[object Object]"` and the request 404s.
- */
-function deepUnwrap(value: unknown): unknown {
-  if (value === null || value === undefined) return value;
-  const raw = readRaw(value);
-  if (raw !== value) return deepUnwrap(raw);
-  if (typeof value !== "object") return value;
-  if (Array.isArray(value)) return value.map(deepUnwrap);
-  const out: Record<string, unknown> = {};
-  for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
-    out[k] = deepUnwrap(v);
-  }
-  return out;
-}
-
-/**
- * Wrap a `@kubernetes/client-node` Api instance so each method call
- * runs in its own AsyncLocalStorage slot — `doFetch` writes the HTTP
- * event's `sourceSeq` into the slot, and after the lib parses the
- * response we re-attach the seq to the returned object. Downstream
- * `expect(result.items[0].status…)` assertions then fold under that
- * HTTP event in the test event log, the same way `expect(res.status)`
- * does for plain `fetch` calls.
- *
- * Method arguments are deep-unwrapped on the way in so values pulled
- * from a previous API response (still carrying the inspector wrappers)
- * can be passed straight back into another call.
- *
- * Non-function properties pass through untagged.
- */
-function withTagging<T extends object>(api: T): Tagged<T> {
-  return new Proxy(api, {
-    get(target, prop, receiver) {
-      const value = Reflect.get(target, prop, receiver);
-      if (typeof value !== "function") return value;
-      // Bind the original method to `target` so the lib's internal
-      // `this.configuration` accesses keep working.
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      const method = (value as any).bind(target);
-      return (...args: unknown[]): unknown => {
-        const unwrappedArgs = args.map(deepUnwrap);
-        const slot: CallSlot = {};
-        const result = callContext.run(slot, () =>
-          method(...unwrappedArgs),
-        );
-        // Wrap unconditionally — `slot.seq` is undefined when no event was
-        // recorded (setup/eval, no active recorder), but the result's type is
-        // wrapped, so the value must be wrapped at runtime too (just without a
-        // provenance link). Keeps `.unwrap()` available in every context.
-        if (result && typeof (result as Promise<unknown>).then === "function") {
-          return (result as Promise<unknown>).then((v) => wrap(v, slot.seq));
-        }
-        return wrap(result, slot.seq);
-      };
-    },
-  }) as unknown as Tagged<T>;
-}
-
-/**
- * Image tag for the Traefik we install. Pulled on first cluster boot
- * through the host `zot` mirror (`docker.io` → cache) configured in
- * `registries.yaml`, then captured into the warm-template snapshot so
- * warm starts never pull.
- */
-const TRAEFIK_IMAGE = "rancher/mirrored-library-traefik:3.3.2";
-
-/**
- * Names of the in-cluster resources that carry the CA-signed default
- * ingress certificate (created in `setupK3sCluster` when TLS is enabled).
- * The Secret holds the leaf cert+key; the ConfigMap holds the Traefik
- * file-provider snippet that points the `default` TLS store at it.
- */
-const TRAEFIK_TLS_SECRET = "traefik-default-tls";
-const TRAEFIK_DYNAMIC_CONFIGMAP = "traefik-dynamic";
-
-/**
- * Traefik file-provider dynamic config: make the in-VM-CA leaf the
- * `default` store certificate, so every router on the `websecure`
- * entrypoint (which we force to TLS) serves it with no per-Ingress
- * `spec.tls` needed.
- */
-const TRAEFIK_DYNAMIC_TLS = `tls:
-  stores:
-    default:
-      defaultCertificate:
-        certFile: /certs/tls.crt
-        keyFile: /certs/tls.key
-`;
-
-/**
- * Traefik manifest applied during setup(). `hostNetwork: true` puts
- * Traefik in the k3s container's netns, so it binds the container's :80
- * (and, with `tls`, :443) directly — no CNI portmap involved (that path
- * still trips on the kernel's missing xt_comment match).
- *
- * When `tls` is set we add a `websecure` :443 entrypoint with TLS forced
- * on (served from the `default` store, i.e. the in-VM-CA leaf mounted
- * from the `traefik-default-tls` Secret via the file provider). HTTPS
- * then works for any routed host under the cluster's `ingressDomains`
- * with zero per-Ingress config; the :80 `web` entrypoint is unchanged.
- */
-function buildTraefikManifest(tls: boolean): string {
-  const args = [
-    "        - --entrypoints.web.address=:80",
-    ...(tls
-      ? [
-          "        - --entrypoints.websecure.address=:443",
-          "        - --entrypoints.websecure.http.tls=true",
-        ]
-      : []),
-    "        - --providers.kubernetesingress=true",
-    "        - --providers.kubernetesingress.ingressclass=traefik",
-    ...(tls
-      ? [
-          "        - --providers.file.directory=/dynamic",
-          "        - --providers.file.watch=true",
-        ]
-      : []),
-    "        - --log.level=INFO",
-  ].join("\n");
-  const ports = [
-    "        - name: web",
-    "          containerPort: 80",
-    ...(tls
-      ? ["        - name: websecure", "          containerPort: 443"]
-      : []),
-  ].join("\n");
-  const volumeMounts = tls
-    ? `
-        volumeMounts:
-        - name: default-cert
-          mountPath: /certs
-          readOnly: true
-        - name: dynamic
-          mountPath: /dynamic
-          readOnly: true`
-    : "";
-  const volumes = tls
-    ? `
-      volumes:
-      - name: default-cert
-        secret:
-          secretName: ${TRAEFIK_TLS_SECRET}
-      - name: dynamic
-        configMap:
-          name: ${TRAEFIK_DYNAMIC_CONFIGMAP}`
-    : "";
-  return `apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: traefik
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: traefik
-rules:
-- apiGroups: [""]
-  resources: ["services", "endpoints", "secrets", "nodes"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["discovery.k8s.io"]
-  resources: ["endpointslices"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["networking.k8s.io"]
-  resources: ["ingresses", "ingressclasses"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["networking.k8s.io"]
-  resources: ["ingresses/status"]
-  verbs: ["update"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: traefik
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: traefik
-subjects:
-- kind: ServiceAccount
-  name: traefik
-  namespace: kube-system
----
-apiVersion: networking.k8s.io/v1
-kind: IngressClass
-metadata:
-  name: traefik
-  annotations:
-    ingressclass.kubernetes.io/is-default-class: "true"
-spec:
-  controller: traefik.io/ingress-controller
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: traefik
-  namespace: kube-system
-  labels:
-    app: traefik
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: traefik
-  template:
-    metadata:
-      labels:
-        app: traefik
-    spec:
-      serviceAccountName: traefik
-      hostNetwork: true
-      dnsPolicy: Default
-      tolerations:
-      - operator: Exists
-      containers:
-      - name: traefik
-        image: ${TRAEFIK_IMAGE}
-        imagePullPolicy: IfNotPresent
-        args:
-${args}
-        ports:
-${ports}${volumeMounts}${volumes}
-`;
-}
-
-/**
- * Host-side `zot` pull-through cache layout (local Firecracker provider
- * only). One zot instance per upstream registry, all bound to the
- * `spectest-br0` gateway `10.42.0.1` on the ports below — **kept in sync
- * with `scripts/install-zot.sh`**. We mirror the cluster's containerd
- * through these so every image pull reuses the shared host cache instead
- * of hitting the public registry, and we list the canonical upstream as
- * a fallback endpoint so a missing/cold mirror only ever slows a pull,
- * never breaks it.
- */
-const ZOT_MIRRORS: Array<{ registry: string; port: number; upstream: string }> = [
-  { registry: "docker.io", port: 5000, upstream: "https://registry-1.docker.io" },
-  { registry: "ghcr.io", port: 5001, upstream: "https://ghcr.io" },
-  { registry: "quay.io", port: 5002, upstream: "https://quay.io" },
-  { registry: "registry.k8s.io", port: 5003, upstream: "https://registry.k8s.io" },
-  { registry: "public.ecr.aws", port: 5004, upstream: "https://public.ecr.aws" },
-  { registry: "gcr.io", port: 5005, upstream: "https://gcr.io" },
-  { registry: "mcr.microsoft.com", port: 5006, upstream: "https://mcr.microsoft.com" },
-];
-
-/**
- * Discover the host-side image cache gateway by reading the same
- * `registry-mirrors` entry the in-VM dockerd already uses (baked into
- * the local provider's golden `/etc/docker/daemon.json`). Returns the
- * gateway host (`"10.42.0.1"`) when present, or `null` when there's no
- * host cache — e.g. on Freestyle, where the cluster then pulls every
- * image direct. Runs inside the daemon (VM) at `index.ts` load time, so
- * the result is stable per host and never poisons the warm-template
- * cache.
- */
-function detectHostMirrorGateway(): string | null {
-  try {
-    const cfg = JSON.parse(
-      readFileSync("/etc/docker/daemon.json", "utf8"),
-    ) as { "registry-mirrors"?: string[] };
-    const first = cfg["registry-mirrors"]?.[0];
-    return first ? new URL(first).hostname || null : null;
-  } catch {
-    return null;
-  }
-}
-
-/**
- * Build `/etc/rancher/k3s/registries.yaml`. k3s reads this **once, at
- * startup**, to configure its embedded containerd — which is why it has
- * to be seeded via `files` (a pre-start bind mount) rather than a
- * `setup` hook. Two jobs:
- *   1. Mirror the cluster's image pulls through the host `zot` cache
- *      (local provider only; omitted when there's no host cache).
- *   2. Trust the in-cluster registry, addressed as `<key>.internal:5000`
- *      (the `{{SPECTEST_SERVICE}}` token is expanded to the cluster's
- *      service key when the file is written). Image *references* use
- *      that peer-reachable name, but containerd pulls via the loopback
- *      endpoint `http://127.0.0.1:5000` — the hostNetwork registry pod
- *      shares the node's netns, so this needs no in-container DNS and
- *      can't be broken by a clobbered `hostnames`.
- * Returns `null` when there's nothing to configure (no host cache and
- * `registry` disabled), in which case no file is injected.
- */
-function buildRegistriesYaml(registryEnabled: boolean): string | null {
-  const gateway = detectHostMirrorGateway();
-  if (!gateway && !registryEnabled) return null;
-
-  const lines: string[] = ["mirrors:"];
-  if (gateway) {
-    for (const { registry, port, upstream } of ZOT_MIRRORS) {
-      lines.push(
-        `  "${registry}":`,
-        `    endpoint:`,
-        `      - "http://${gateway}:${port}"`,
-        `      - "${upstream}"`,
-      );
-    }
-  }
-  if (registryEnabled) {
-    const host = `{{SPECTEST_SERVICE}}.internal:${K3S_REGISTRY_PORT}`;
-    lines.push(
-      `  "${host}":`,
-      `    endpoint:`,
-      `      - "http://127.0.0.1:${K3S_REGISTRY_PORT}"`,
-      "configs:",
-      // The endpoint is plain HTTP; the config (keyed by endpoint host)
-      // makes that explicit and disables any TLS attempt against it.
-      `  "127.0.0.1:${K3S_REGISTRY_PORT}":`,
-      `    tls:`,
-      `      insecure_skip_verify: true`,
-    );
-  }
-  return lines.join("\n") + "\n";
-}
-
-/**
- * In-cluster OCI registry (CNCF `distribution`). `hostNetwork: true`
- * binds the cluster container's `:5000` directly — the same trick
- * Traefik uses — so peer services reach it at `<cluster-key>.internal:5000`
- * (the cluster service's own alias) and the node's own containerd reaches
- * it at `127.0.0.1:5000`. Storage is an `emptyDir`, so pushed images live
- * in the cluster and are captured by snapshot / isolated per test fork
- * like all other in-VM state.
- */
-const REGISTRY_MANIFEST = `apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: spectest-registry
-  namespace: kube-system
-  labels:
-    app: spectest-registry
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: spectest-registry
-  template:
-    metadata:
-      labels:
-        app: spectest-registry
-    spec:
-      hostNetwork: true
-      dnsPolicy: Default
-      tolerations:
-      - operator: Exists
-      containers:
-      - name: registry
-        image: registry:2
-        imagePullPolicy: IfNotPresent
-        env:
-        - name: REGISTRY_HTTP_ADDR
-          value: ":${K3S_REGISTRY_PORT}"
-        - name: REGISTRY_STORAGE_DELETE_ENABLED
-          value: "true"
-        ports:
-        - name: registry
-          containerPort: ${K3S_REGISTRY_PORT}
-        volumeMounts:
-        - name: data
-          mountPath: /var/lib/registry
-      volumes:
-      - name: data
-        emptyDir: {}
-`;
-
-/**
- * Wait for a Deployment to reach its desired ready-replica count,
- * polling once a second up to `timeoutMs`. Throws with the last-seen
- * status (plus kube-system pod diagnostics) on timeout.
- */
-async function waitForDeployment(
-  clusterName: string,
-  helpers: K3sHelpers,
-  deployment: string,
-  timeoutMs: number,
-): Promise<void> {
-  const deadline = Date.now() + timeoutMs;
-  let lastErr: string | undefined;
-  while (Date.now() < deadline) {
-    try {
-      // `.unwrap()` recovers the plain object — the client wraps its result in
-      // every context now (provenance-free here, since this internal poll runs
-      // during setup with no active recorder). We're reading for control flow,
-      // not asserting, so go straight to raw.
-      const dep = (
-        await helpers.client.apps.readNamespacedDeployment({
-          name: deployment,
-          namespace: "kube-system",
-        })
-      ).unwrap();
-      const ready = dep.status?.readyReplicas ?? 0;
-      const want = dep.spec?.replicas ?? 1;
-      if (ready >= want && want > 0) return;
-      lastErr = `${deployment} Deployment exists but only ${ready}/${want} replicas Ready`;
-    } catch (err) {
-      const msg = (err as Error)?.message ?? String(err);
-      lastErr = /not found|404/i.test(msg)
-        ? `${deployment} Deployment does not exist yet`
-        : msg;
-    }
-    // 250ms: the two sequential rollout waits in setup sit on the cold
-    // start's critical path, and a 1s poll wasted up to ~2s of it.
-    await new Promise((r) => setTimeout(r, 250));
-  }
-  const diag = await collectTraefikDiagnostics(helpers);
-  throw new Error(
-    `k3s(${clusterName}): ${deployment} did not reach Ready within ${
-      timeoutMs / 1000
-    }s. ${lastErr ?? ""}\n${diag}`,
-  );
-}
-
-/**
- * Post-Ready setup. Apply the Traefik manifest (hostNetwork) and, when
- * enabled, the in-cluster registry; wait for each Deployment to come
- * Ready. Captured by the warm-template snapshot, so warm starts pay none
- * of this cost.
- *
- * When the cluster declares `ingressDomains` (and the in-VM CA is
- * present), TLS is enabled: we mint a CA-signed leaf covering `*.<domain>`
- * for each domain, stash it in the `traefik-default-tls` Secret + a
- * file-provider ConfigMap, and bring Traefik up with a `websecure` :443
- * entrypoint serving it as the default cert. Those domains are then
- * reachable over HTTPS with a cert the test framework already trusts.
- */
-async function setupK3sCluster(
-  name: string,
-  helpers: K3sHelpers,
-  opts: { registry: boolean; ingressDomains: string[] },
-): Promise<void> {
-  const tlsEnabled = opts.ingressDomains.length > 0 && caPresent();
-  if (tlsEnabled) {
-    const { cert, key } = await issueIngressCert(
-      opts.ingressDomains.map((d) => `*.${d}`),
-    );
-    // Apply the cert Secret + dynamic-config ConfigMap before the
-    // Deployment that mounts them. `stringData` lets us hand over plain
-    // PEM; the API server base64-encodes it.
-    await helpers.client.core.createNamespacedSecret({
-      namespace: "kube-system",
-      body: {
-        metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
-        type: "kubernetes.io/tls",
-        stringData: { "tls.crt": cert, "tls.key": key },
-      },
-    });
-    await helpers.client.core.createNamespacedConfigMap({
-      namespace: "kube-system",
-      body: {
-        metadata: {
-          name: TRAEFIK_DYNAMIC_CONFIGMAP,
-          namespace: "kube-system",
-        },
-        data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
-      },
-    });
-  }
-  await helpers.apply(buildTraefikManifest(tlsEnabled));
-  if (opts.registry) await helpers.apply(REGISTRY_MANIFEST);
-  // Both rollouts proceed independently inside the cluster — wait on them
-  // concurrently (they used to serialize, wasting up to a rollout's tail).
-  const waits = [waitForDeployment(name, helpers, "traefik", 120_000)];
-  if (opts.registry) {
-    waits.push(waitForDeployment(name, helpers, "spectest-registry", 120_000));
-  }
-  await Promise.all(waits);
-}
-
-/**
- * Snapshot of kube-system state, dumped on traefik-wait timeout. With
- * the static install, the failure surface is just "did our Deployment
- * schedule and become Ready?" — pod listing covers that.
- */
-async function collectTraefikDiagnostics(helpers: K3sHelpers): Promise<string> {
-  const lines: string[] = [];
-  try {
-    const pods = await helpers.client.core.listNamespacedPod({
-      namespace: "kube-system",
-    });
-    lines.push(`kube-system pods (${pods.items.length}):`);
-    for (const p of pods.items) {
-      const phase = p.status?.phase ?? "?";
-      const cs = p.status?.containerStatuses ?? [];
-      const reasons = cs
-        .map((c) => c.state?.waiting?.reason ?? c.state?.terminated?.reason ?? "")
-        .filter((s) => s)
-        .join(",");
-      lines.push(
-        `  ${p.metadata?.name ?? "?"}: phase=${phase}${reasons ? ` reasons=${reasons}` : ""}`,
-      );
-      // The waiting `message` carries containerd's actual error — e.g. the
-      // failing endpoint, an upstream `429 Too Many Requests`, or a
-      // `connection refused`. The `reason` alone (`ErrImagePull`) hides all
-      // of that, which is exactly what we need when a pull won't settle.
-      for (const c of cs) {
-        const msg =
-          c.state?.waiting?.message ?? c.state?.terminated?.message ?? "";
-        if (msg) lines.push(`      ${c.name}: ${msg.replace(/\s+/g, " ").trim()}`);
-      }
-    }
-  } catch (err) {
-    lines.push(`(listing pods failed: ${(err as Error)?.message ?? String(err)})`);
-  }
-  // Recent Warning events surface pull failures the kubelet emits before a
-  // container status even settles (FailedPull / Failed / BackOff), with the
-  // raw containerd message attached. Best-effort: never let diagnostics throw.
-  try {
-    const events = await helpers.client.core.listNamespacedEvent({
-      namespace: "kube-system",
-    });
-    const warnings = (events.items ?? [])
-      .filter((e) => e.type === "Warning")
-      .map((e) => ({
-        obj: e.involvedObject?.name ?? "?",
-        reason: e.reason ?? "?",
-        message: (e.message ?? "").replace(/\s+/g, " ").trim(),
-      }))
-      .filter((e) => e.message);
-    if (warnings.length) {
-      lines.push(`kube-system Warning events (${warnings.length}):`);
-      // Keep the tail — newest events are appended last by the API.
-      for (const w of warnings.slice(-12)) {
-        lines.push(`  ${w.obj} [${w.reason}] ${w.message}`);
-      }
-    }
-  } catch (err) {
-    lines.push(`(listing events failed: ${(err as Error)?.message ?? String(err)})`);
-  }
-  return lines.join("\n");
-}
-
-/**
- * A ready-to-use single-node Kubernetes cluster (k3s). Drop into
- * `environment.services`:
- *
- * ```ts
- * services: { k8s: k3s() }
- * ```
- *
- * Tests get `@kubernetes/client-node` API objects pre-wired to this
- * cluster at `ctx.svc.<key>.client` — `core`, `apps`, and a generic
- * `objects` (`KubernetesObjectApi`). Every API call is recorded on the
- * test event log alongside `fetch` calls. There's also `apply(yaml)`
- * sugar for piping a multi-document manifest in.
- *
- * **Ingress.** We deploy Traefik ourselves in `hostNetwork` mode
- * during `setup()`. Traefik binds the cluster container's :80 directly
- * (no ServiceLB / klipper-lb needed), watches Ingress objects via the
- * API, and routes incoming requests to pod Endpoints. Any `hostnames`
- * declared on this service in env.ts therefore route through Traefik:
- * a peer doing `fetch("http://app.example.com")` resolves the host to
- * the k3s container's IP (via systest-resolver), lands on Traefik,
- * and gets dispatched to the matching Ingress rule's backend pods.
- *
- * **Workarounds for Freestyle's kernel** (Linux 6.1.0-x-freestyle).
- * The stock kernel is missing the `xt_comment` netfilter match
- * extension. Two consequences, each handled below:
- *
- *   1. *kube-proxy* in default iptables mode generates rules with
- *      `-m comment --comment "..."`, which the kernel rejects —
- *      breaking pod→ClusterIP routing and every pod that talks to the
- *      in-cluster API (helm-install Jobs, CoreDNS, …). Fixed by
- *      `--kube-proxy-arg=proxy-mode=nftables`: kube-proxy emits
- *      native nftables rules where comments are a first-class
- *      construct, no xt_comment dependency. nftables proxy mode is
- *      GA in k8s 1.32, which is why we pin that.
- *
- *   2. *CNI portmap plugin* (used by klipper-lb's hostPort to expose
- *      LoadBalancer ports on the host) still uses iptables-nft and
- *      hits the same xt_comment failure — there's no equivalent
- *      flag to switch it to native nftables. Workaround: disable the
- *      bundled traefik + ServiceLB and run Traefik with
- *      `hostNetwork: true` ourselves. hostNetwork pods don't go
- *      through portmap at all (they share the node's netns directly),
- *      so the broken plugin is never invoked.
- *
- * Flannel uses the `host-gw` backend because Freestyle's stock kernel
- * lacks the VXLAN module — fine for single-node clusters.
- */
-/**
- * Default k3s docker image tag.
- *
- * The cluster's system images (the `rancher/k3s` image itself, plus
- * coredns / local-path-provisioner / pause and the Traefik we deploy)
- * are pulled on first boot through the host `zot` pull-through cache —
- * `registries.yaml` (seeded via `files`) mirrors `docker.io` and
- * `registry.k8s.io` at it. The first-ever cluster boot on a cold-cache
- * host pays the upstream pull once; thereafter zot serves the blobs
- * host-wide and the warm-template snapshot captures the booted cluster,
- * so neither cold-cache nor warm starts re-pull. Any `opts.version`
- * works — there's no base-snapshot release to keep in sync with.
- *
- * **Why v1.32.x:** kube-proxy's `nftables` proxy mode is GA in k8s 1.32
- * (beta in 1.31, alpha-gated in 1.30). The component runs kube-proxy in
- * this mode to sidestep Freestyle's missing `xt_comment` netfilter
- * extension; dropping below 1.31 reintroduces the broken iptables path.
- */
-const DEFAULT_K3S_VERSION = "v1.32.1-k3s1";
-
-export function k3s(opts: K3sOptions = {}) {
-  const version = opts.version ?? DEFAULT_K3S_VERSION;
-  const extra = opts.extraArgs ?? [];
-  const registryEnabled = opts.registry !== false;
-  // Wildcard ingress domains. Drives both the `provides(... dnsName)`
-  // wiring below and (when non-empty) the CA-signed TLS default cert that
-  // setupK3sCluster mints so these domains are reachable over HTTPS.
-  const ingressDomains = opts.ingressDomains ?? [];
-  // `/etc/rancher/k3s/registries.yaml` (host-cache mirrors + trust for
-  // the in-cluster registry). Seeded via `files` because k3s reads it
-  // only at startup, before any setup hook could run.
-  const registriesYaml = buildRegistriesYaml(registryEnabled);
-  const serverArgs = [
-    "k3s",
-    "server",
-    // CoreDNS / pod DNS upstream. Without this, k3s sees only the
-    // loopback 127.0.0.11 (Docker's embedded DNS) in the container's
-    // /etc/resolv.conf, decides no usable nameserver exists, and writes a
-    // fallback `nameserver 8.8.8.8` that CoreDNS then forwards to — so
-    // pods reach the public internet but NOT peer services on
-    // spectest-net (`<svc>.internal`, fakes, service-TLS hosts all
-    // NXDOMAIN). We instead point k3s at the container's default gateway
-    // — the spectest-net bridge gateway, where spectest-resolver binds a
-    // second listener for exactly this. The file is written by the
-    // command wrapper below because the gateway IP is only known at
-    // container start.
-    "--resolv-conf=/run/spectest-resolv.conf",
-    // metrics-server isn't useful in a test cluster.
-    "--disable=metrics-server",
-    // traefik + servicelb disabled: their klipper-lb DaemonSet uses
-    // CNI portmap to bind host port 80, which still needs xt_comment
-    // (the iptables compat path that the kernel can't satisfy).
-    // We install Traefik with hostNetwork in setup() — same effect,
-    // no portmap involved. local-storage stays enabled: it's a
-    // controller pod that doesn't bind host ports.
-    "--disable=traefik",
-    "--disable=servicelb",
-    // Pod CIDR MUST avoid 10.42.0.0/16: that's the spectest-br0 host
-    // bridge subnet, whose gateway 10.42.0.1 fronts the host image caches
-    // (zot :5000-5007, buildkitd :1234). k3s's *default* pod CIDR is also
-    // 10.42.0.0/16 — with --flannel-backend=host-gw, flannel programs that
-    // route into the node's own routing table and gives cni0 the subnet's
-    // .1 (10.42.0.1). That shadows the route to the host gateway, so once
-    // CNI comes up the node can no longer reach 10.42.0.1 and every
-    // subsequent registry pull dies with "connect: connection refused"
-    // (e.g. the in-cluster registry's `registry:2`, applied after the
-    // cluster is up — the airgap-bundled system images pull *before* CNI
-    // and so sneak through). Move pods to 10.44/service to 10.45.
-    "--cluster-cidr=10.44.0.0/16",
-    "--service-cidr=10.45.0.0/16",
-    "--cluster-dns=10.45.0.10",
-    "--flannel-backend=host-gw",
-    "--write-kubeconfig-mode=644",
-    // kube-proxy in nftables mode: native nftables rules, no
-    // xt_comment dependency. Pod→ClusterIP routing works, so
-    // CoreDNS / helm-install / anything-talking-to-the-API works.
-    // GA in k8s 1.32.
-    "--kube-proxy-arg=proxy-mode=nftables",
-    ...extra,
-  ].join(" ");
-  // The service `command` runs under `/bin/sh -c` (see runContainer in
-  // daemon.ts), so derive the bridge gateway from the container's default
-  // route at start time, write it as the k3s resolv-conf, then exec k3s
-  // (exec so it stays the container's main process and signals / the
-  // readyCheck behave exactly as before). `/run` is a tmpfs on this
-  // service, so the file is writable and never persisted into a snapshot.
-  const cmd =
-    "GW=\"$(ip route 2>/dev/null | awk '/^default/{print $3; exit}')\"; " +
-    'if [ -n "$GW" ]; then ' +
-    "printf 'nameserver %s\\noptions ndots:0\\n' \"$GW\" > /run/spectest-resolv.conf; " +
-    "else echo 'spectest: no default gateway found; k3s pod DNS for peer services will not resolve' >&2; " +
-    ": > /run/spectest-resolv.conf; fi; " +
-    `exec ${serverArgs}`;
-  // Plain /readyz probe. On a warm zot cache the cluster's images are
-  // already local, so the first boot completes in seconds; the
-  // first-ever boot on a cold-cache host pulls through the mirror and
-  // can take a couple of minutes (covered by readyTimeoutSecs).
-  const readyCmd = "kubectl get --raw=/readyz >/dev/null 2>&1";
-  const def = {
-    image: { type: "registry", reference: `rancher/k3s:${version}` },
-    command: cmd,
-    privileged: true,
-    tmpfs: ["/run", "/var/run"],
-    cgroupns: "host",
-    // 80/443 are advisory — peer services and host code reach them via
-    // the k3s container's IP. ServiceLB (klipper-lb) binds them inside
-    // the container's netns and forwards to the traefik pod. 5000 is the
-    // in-cluster registry (hostNetwork pod bound to the container netns),
-    // reached by peers at the cluster's own `<key>.internal:5000` alias.
-    ports: registryEnabled ? [80, 443, 6443, K3S_REGISTRY_PORT] : [80, 443, 6443],
-    // NOTE: do NOT mount /var/lib/rancher/k3s/agent/containerd as a cache
-    // volume. It was tried (to spare a recreated cluster re-pulling its
-    // system images on delta restores) and a fresh k3s server against the
-    // previous container's containerd store — killed un-cleanly by the
-    // teardown's `docker rm -f` — wedged the apiserver minutes in
-    // (rollouts never settled, pod listing started failing). The zot
-    // mirror already makes those re-pulls cheap; the residual win wasn't
-    // worth the recovery semantics of a crash-state store under a fresh
-    // cluster db.
-    ...(registriesYaml
-      ? {
-          files: [
-            { path: "/etc/rancher/k3s/registries.yaml", content: registriesYaml },
-          ],
-        }
-      : {}),
-    readyCheck: {
-      type: "exec" as const,
-      command: readyCmd,
-      timeoutSecs: opts.readyTimeoutSecs ?? 120,
-    },
-    setup: async ({ name, helpers }: { name: string; helpers: K3sHelpers }) => {
-      await setupK3sCluster(name, helpers, {
-        registry: registryEnabled,
-        ingressDomains,
-      });
-    },
-    helpers: async ({ name, exec }: ServiceHelpersContext): Promise<K3sHelpers> => {
-      // Read the cluster's kubeconfig and address the API server by its
-      // auto-assigned `<name>.internal` hostname on spectest-net. TLS
-      // verification is off (see the K3sHelpers docstring), so the
-      // server's cert SAN list doesn't need to include the .internal
-      // name.
-      const kcRead = await exec(name, ["cat", "/etc/rancher/k3s/k3s.yaml"]);
-      if (kcRead.exitCode !== 0) {
-        throw new Error(
-          `k3s(${name}): failed to read kubeconfig from container: ${kcRead.stderr.trim()}`,
-        );
-      }
-
-      const kubeconfig = new KubeConfig();
-      kubeconfig.loadFromString(kcRead.stdout);
-
-      const server = `https://${name}.internal:6443`;
-      // Update kc.clusters so any code that reads kubeconfig sees the
-      // right server URL, but the actual request server comes from the
-      // Configuration we build below. `Cluster.server` is typed `readonly`
-      // by @kubernetes/client-node, but the loaded object is a plain mutable
-      // record — write through a mutable view rather than rebuild the config.
-      for (const cluster of kubeconfig.clusters) {
-        (cluster as { -readonly [K in keyof typeof cluster]: typeof cluster[K] }).server =
-          server;
-      }
-
-      const httpApi = new FetchHttpLibrary();
-      const baseServer = new ServerConfiguration(server, {});
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      const config = createConfiguration({
-        baseServer,
-        authMethods: { default: kubeconfig as any },
-        httpApi: httpApi as any,
-      });
-
-      const core = withTagging(new CoreV1Api(config));
-      const apps = withTagging(new AppsV1Api(config));
-      const objects = withTagging(new KubernetesObjectApi(config));
-
-      const apply = async (
-        manifest: string,
-      ): Promise<Wrapped<KubernetesObject>[]> => {
-        const docs = loadAllYaml(manifest) as KubernetesObject[];
-        const out: Wrapped<KubernetesObject>[] = [];
-        for (const doc of docs) {
-          if (!doc || typeof doc !== "object" || !("kind" in doc)) continue;
-          // `objects.create` is wrapped by `withTagging`, so each returned
-          // object already carries the back-reference to its create call.
-          const created = await objects.create(doc);
-          out.push(created as unknown as Wrapped<KubernetesObject>);
-        }
-        return out;
-      };
-
-      return {
-        kubeconfig,
-        client: { core, apps, objects },
-        apply,
-      };
-    },
-  } satisfies ServiceDefinition<K3sHelpers>;
-
-  // Wildcard ingress domains → a dnsName(`*.<domain>`, { service: self })
-  // each, attached via provides(). SELF_SERVICE_TOKEN resolves to this
-  // service's key at load time (the component can't know it here). The
-  // resolver then points every host under the domain at the cluster.
-  if (ingressDomains.length === 0) return def;
-  return provides(
-    def,
-    ingressDomains.map((domain) =>
-      dnsName(`*.${domain}`, { service: SELF_SERVICE_TOKEN }),
-    ),
-  );
-}