@specific.dev/spectest 0.69.0 → 0.71.0

package/dist/components/k3s.d.ts

@@ -332,4 +332,7 @@ export declare function k3s(opts?: K3sOptions): {
     tmpfs: string[];
     cgroupns: string;
     ports: number[];
+    volumes: {
+        target: string;
+    }[];
 };

package/dist/components/k3s.js

@@ -1,7 +1,7 @@
 import { AsyncLocalStorage } from "node:async_hooks";
 import { spawn as nodeSpawn } from "node:child_process";
 import { randomUUID } from "node:crypto";
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync } from "node:fs";
 import { readFile, unlink } from "node:fs/promises";
 import { AppsV1Api, BatchV1Api, CoreV1Api, KubeConfig, KubernetesObjectApi, PatchStrategy, ResponseContext, ServerConfiguration, createConfiguration, loadAllYaml, } from "@kubernetes/client-node";
 import { Observable } from "@kubernetes/client-node/dist/gen/rxjsStub.js";
@@ -10,6 +10,114 @@ import { deepUnwrap, readTag, wrap } from "../inspect.js";
 import { recorderAnnotate, recorderRemove } from "../recorder.js";
 /** Port the in-cluster registry listens on (plain HTTP). */
 const K3S_REGISTRY_PORT = 5000;
+/** The cluster's own containerd root, inside the k3s container. */
+const CLUSTER_STORE_TARGET = "/var/lib/rancher/k3s/agent/containerd";
+/**
+ * Written before the server starts and removed by the ready probe.
+ *
+ * It survives in the store only when a boot never reached a live API
+ * server, which is the one thing the sweep below cannot work out for
+ * itself. Blacksmith's rule, from the other end: they refuse to commit a
+ * cache when the integrity check *did not run*, not only when it failed.
+ */
+const BOOT_MARKER = ".spectest-boot-incomplete";
+/**
+ * Bring the cluster's image store back into service, or throw it away.
+ *
+ * The store is the cluster's containerd root on a directory that outlives
+ * this container (`NESTED_STORE_ROOT`), so a build finds coredns, traefik,
+ * the local-path provisioner and every image the project deployed already
+ * pulled **and already extracted**. Extraction is 48x the I/O of the
+ * download (`STICKY_DISKS.md` §7.2), which is why the zot mirror never
+ * helped here and why an extracted store is the whole prize.
+ *
+ * What it inherits with them is one build's crash state: the store was
+ * captured from a running guest, and the teardown's `docker rm -f` killed
+ * that cluster where it stood. **Deleting the dead files is not enough,
+ * and the failure is not subtle.** containerd's index still lists the
+ * previous build's containers, so the CRI plugin offers them to a kubelet
+ * whose API server has never heard of them; the kubelet spends its startup
+ * reconciling containers whose tasks are gone (`Failed to create existing
+ * container … task not found`) and does not register its own Node in time,
+ * and the CSI plugin — which waits on that Node with a fixed budget —
+ * calls `Fatalf`. The kubelet dies, k3s exits 2, and what the author sees
+ * is a rollout that timed out against a hostname that no longer resolves.
+ * Measured: 2 of 2 delta restores, against 2 of 2 clean without the store.
+ *
+ * So the sweep removes those containers through **containerd itself**,
+ * which is the only thing that can edit its index. The k3s image ships a
+ * standalone `containerd` and `ctr`, so a throwaway daemon runs against
+ * the store with the CRI plugin disabled, deletes the previous build's
+ * containers and their tasks, and stops. Images, content blobs and
+ * extracted snapshots — the whole cache — stay. It is the same discipline
+ * `spectest-image-cache-up` applies to the guest's own store one level up,
+ * where it sweeps the `moby` namespace before dockerd starts.
+ *
+ * Two failures cost a cache and never a build:
+ *
+ *  1. **A sweep that cannot run wipes the store.** A daemon that will not
+ *     start leaves behind exactly the containers this cluster fatals on,
+ *     so carrying on is the one option that is known bad.
+ *  2. **A store that did not work is never inherited twice.** The marker
+ *     is written before the server starts and removed by the ready probe,
+ *     so it survives only when a boot never reached a live API server.
+ *     That is the backstop for the sweep being incomplete in some way we
+ *     have not seen: the cost is one cold build's pulls, and it repairs
+ *     itself. Blacksmith's rule from the other end — they refuse to commit
+ *     a cache when the integrity check *did not run*, not only when it
+ *     failed.
+ */
+/** Plugins the sweep daemon does not need. Every plugin is startup time,
+ *  and the sweep sits on the critical path of the cluster's boot. CRI is
+ *  off for a second reason: this daemon exists to edit an index, and a CRI
+ *  plugin would set about being a container runtime on a store we are
+ *  seconds from handing to the real one. */
+const SWEEP_DISABLED_PLUGINS = [
+    "io.containerd.grpc.v1.cri",
+    "io.containerd.snapshotter.v1.btrfs",
+    "io.containerd.snapshotter.v1.native",
+    "io.containerd.snapshotter.v1.aufs",
+    "io.containerd.snapshotter.v1.zfs",
+    "io.containerd.snapshotter.v1.devmapper",
+    "io.containerd.snapshotter.v1.stargz",
+    "io.containerd.snapshotter.v1.fuse-overlayfs",
+];
+const STORE_SWEEP_SH = `S=${CLUSTER_STORE_TARGET}; M="$S/${BOOT_MARKER}"; A=/run/spectest-sweep.sock; ` +
+    `sweep_store() { ` +
+    `printf '%s\\n' 'version = 2' ` +
+    `'disabled_plugins = [${SWEEP_DISABLED_PLUGINS.map((p) => `"${p}"`).join(", ")}]' ` +
+    `> /run/spectest-sweep.toml || return 1; ` +
+    `containerd -c /run/spectest-sweep.toml --root "$S" --state /run/spectest-sweep ` +
+    `--address "$A" > /run/spectest-sweep.log 2>&1 & ` +
+    `cd_pid=$!; i=0; ` +
+    `while [ ! -S "$A" ] && [ $i -lt 600 ]; do sleep 0.1; i=$((i+1)); done; ` +
+    `if [ ! -S "$A" ]; then kill $cd_pid 2>/dev/null; return 1; fi; ` +
+    // One invocation for the whole set, not two per container. Each `ctr` is
+    // a Go binary start plus a gRPC round trip, and a build leaves ~20
+    // containers behind — the per-container form spent seconds of the
+    // cluster's boot on process starts alone (measured: ready 3.6s -> 7.0s).
+    `ids=$(timeout 30 ctr -a "$A" -n k8s.io containers ls -q 2>/dev/null); ` +
+    `if [ -n "$ids" ]; then ` +
+    `timeout 60 ctr -a "$A" -n k8s.io tasks rm -f $ids >/dev/null 2>&1 || true; ` +
+    `timeout 60 ctr -a "$A" -n k8s.io containers rm $ids >/dev/null 2>&1 || true; fi; ` +
+    // containerd 2.x keeps sandboxes in a store of their own; on the 1.7 k3s
+    // ships they are ordinary containers and this is a no-op.
+    `sb=$(timeout 30 ctr -a "$A" -n k8s.io sandboxes ls -q 2>/dev/null); ` +
+    `[ -n "$sb" ] && timeout 60 ctr -a "$A" -n k8s.io sandboxes rm $sb >/dev/null 2>&1; ` +
+    `kill $cd_pid 2>/dev/null; wait $cd_pid 2>/dev/null; ` +
+    `rm -rf "$S/io.containerd.runtime.v2.task" "$S/tmpmounts" /run/spectest-sweep "$A" 2>/dev/null || true; ` +
+    `return 0; }; ` +
+    `if [ -e "$M" ]; then ` +
+    `echo 'spectest: the previous cluster on this image store never became ready; starting from an empty one' >&2; ` +
+    `rm -rf "$S"/* "$S"/.[!.]* 2>/dev/null || true; ` +
+    `elif [ -d "$S/io.containerd.metadata.v1.bolt" ]; then ` +
+    // Timed and printed always: the sweep is inside the cluster's ready time,
+    // so it is the first thing to suspect if that time moves.
+    `T0=$(date +%s); ` +
+    `if sweep_store; then echo "spectest: reused the cluster image store at $S (swept in $(($(date +%s)-T0))s)"; ` +
+    `else echo 'spectest: could not sweep the inherited image store; starting from an empty one' >&2; ` +
+    `rm -rf "$S"/* "$S"/.[!.]* 2>/dev/null || true; fi; fi; ` +
+    `mkdir -p "$S" && : > "$M" 2>/dev/null || true; `;
 /**
  * `extraArgs` entries the component overrides anyway. Passing one of
  * these looks like it works — the flag really is appended to the k3s
@@ -551,50 +659,21 @@ ${ports}${volumeMounts}${volumes}
 `;
 }
 /**
- * Host-side `zot` pull-through cache layout (local Firecracker provider
- * only). One zot instance per upstream registry, all bound to the
- * `spectest-br0` gateway `10.42.0.1` on the ports below — **kept in sync
- * with `scripts/install-zot.sh`**. We mirror the cluster's containerd
- * through these so every image pull reuses the shared host cache instead
- * of hitting the public registry, and we list the canonical upstream as
- * a fallback endpoint so a missing/cold mirror only ever slows a pull,
- * never breaks it.
+ * Docker Hub through Google's public mirror, with the canonical upstream
+ * as the fallback endpoint so a cold or unavailable mirror only ever
+ * slows a pull, never breaks it. The same route the guest's own dockerd
+ * takes (golden's daemon.json) and its in-VM BuildKit. Every other
+ * registry is pulled direct. Nothing here points at a host-side service:
+ * the cluster's own containerd store lives in the environment and is
+ * captured with it (CONTAINER_STORE.md).
  */
-const ZOT_MIRRORS = [
-    { registry: "docker.io", port: 5000, upstream: "https://registry-1.docker.io" },
-    { registry: "ghcr.io", port: 5001, upstream: "https://ghcr.io" },
-    { registry: "quay.io", port: 5002, upstream: "https://quay.io" },
-    { registry: "registry.k8s.io", port: 5003, upstream: "https://registry.k8s.io" },
-    { registry: "public.ecr.aws", port: 5004, upstream: "https://public.ecr.aws" },
-    { registry: "gcr.io", port: 5005, upstream: "https://gcr.io" },
-    { registry: "mcr.microsoft.com", port: 5006, upstream: "https://mcr.microsoft.com" },
-];
-/**
- * Discover the host-side image cache gateway by reading the same
- * `registry-mirrors` entry the in-VM dockerd already uses (baked into
- * the golden rootfs's `/etc/docker/daemon.json`). Returns the
- * gateway host (`"10.42.0.1"`) when present, or `null` when there's no
- * host cache, in which case the cluster pulls every image direct. Runs inside the daemon (VM) at `index.ts` load time, so
- * the result is stable per host and never poisons the warm-template
- * cache.
- */
-function detectHostMirrorGateway() {
-    try {
-        const cfg = JSON.parse(readFileSync("/etc/docker/daemon.json", "utf8"));
-        const first = cfg["registry-mirrors"]?.[0];
-        return first ? new URL(first).hostname || null : null;
-    }
-    catch {
-        return null;
-    }
-}
+const HUB_MIRROR = { registry: "docker.io", mirror: "https://mirror.gcr.io", upstream: "https://registry-1.docker.io" };
 /**
  * Build `/etc/rancher/k3s/registries.yaml`. k3s reads this **once, at
  * startup**, to configure its embedded containerd — which is why it has
  * to be seeded via `files` (a pre-start bind mount) rather than a
  * `setup` hook. Two jobs:
- *   1. Mirror the cluster's image pulls through the host `zot` cache
- *      (omitted when there's no host cache).
+ *   1. Mirror the cluster's Docker Hub pulls through `mirror.gcr.io`.
  *   2. Trust the in-cluster registry, addressed as `<key>.internal:5000`
  *      (the `{{SPECTEST_SERVICE}}` token is expanded to the cluster's
  *      service key when the file is written). Image *references* use
@@ -602,19 +681,15 @@ function detectHostMirrorGateway() {
  *      endpoint `http://127.0.0.1:5000` — the hostNetwork registry pod
  *      shares the node's netns, so this needs no in-container DNS and
  *      can't be broken by a clobbered `hostnames`.
- * Returns `null` when there's nothing to configure (no host cache and
- * `registry` disabled), in which case no file is injected.
  */
 function buildRegistriesYaml(registryEnabled) {
-    const gateway = detectHostMirrorGateway();
-    if (!gateway && !registryEnabled)
-        return null;
-    const lines = ["mirrors:"];
-    if (gateway) {
-        for (const { registry, port, upstream } of ZOT_MIRRORS) {
-            lines.push(`  "${registry}":`, `    endpoint:`, `      - "http://${gateway}:${port}"`, `      - "${upstream}"`);
-        }
-    }
+    const lines = [
+        "mirrors:",
+        `  "${HUB_MIRROR.registry}":`,
+        `    endpoint:`,
+        `      - "${HUB_MIRROR.mirror}"`,
+        `      - "${HUB_MIRROR.upstream}"`,
+    ];
     if (registryEnabled) {
         const host = `{{SPECTEST_SERVICE}}.internal:${K3S_REGISTRY_PORT}`;
         lines.push(`  "${host}":`, `    endpoint:`, `      - "http://127.0.0.1:${K3S_REGISTRY_PORT}"`, "configs:", 
@@ -1312,12 +1387,18 @@ export function k3s(opts = {}) {
         // is affected.
         "mount --make-rshared / 2>/dev/null || " +
         "echo 'spectest: could not make / rshared; CSI node plugins may fail to publish volumes' >&2; " +
+        STORE_SWEEP_SH +
         `exec ${serverArgs}`;
     // Plain /readyz probe. On a warm zot cache the cluster's images are
     // already local, so the first boot completes in seconds; the
     // first-ever boot on a cold-cache host pulls through the mirror and
     // can take a couple of minutes (covered by readyTimeoutSecs).
-    const readyCmd = "kubectl get --raw=/readyz >/dev/null 2>&1";
+    //
+    // Clearing the boot marker is part of the probe on purpose: "the API
+    // server answers" is exactly the condition that proves this cluster came
+    // up on the store it was given, and it is the only signal the sweep can
+    // read on the next boot. See STORE_SWEEP_SH.
+    const readyCmd = `kubectl get --raw=/readyz >/dev/null 2>&1 && rm -f ${CLUSTER_STORE_TARGET}/${BOOT_MARKER}`;
     const def = {
         image: { type: "registry", reference: `rancher/k3s:${version}` },
         command: cmd,
@@ -1333,15 +1414,30 @@ export function k3s(opts = {}) {
         // the same netns and are reachable the same way, without appearing
         // in this list (it's documentation, not a firewall).
         ports: registryEnabled ? [80, 443, 6443, K3S_REGISTRY_PORT] : [80, 443, 6443],
-        // NOTE: do NOT mount /var/lib/rancher/k3s/agent/containerd as a cache
-        // volume. It was tried (to spare a recreated cluster re-pulling its
-        // system images on delta restores) and a fresh k3s server against the
-        // previous container's containerd store — killed un-cleanly by the
-        // teardown's `docker rm -f` — wedged the apiserver minutes in
-        // (rollouts never settled, pod listing started failing). The zot
-        // mirror already makes those re-pulls cheap; the residual win wasn't
-        // worth the recovery semantics of a crash-state store under a fresh
-        // cluster db.
+        // The cluster's image store, on a volume that outlives this
+        // container. Without it the store is the container's writable layer,
+        // which `docker rm -f` deletes at teardown — so every build re-pulls
+        // and, far more expensively, re-extracts coredns, traefik, the
+        // local-path provisioner and everything the project deploys.
+        // Measured on the one project on the image cache: 20.6 s per
+        // build, identical on the cold and delta tiers, i.e. the one term
+        // neither the delta restore nor the cache image store reached.
+        //
+        // `cache` names a cache disk, which is the ordinary way any project
+        // asks for one — there is nothing Kubernetes-shaped in the platform
+        // for this. Two clusters in one environment may name the same disk
+        // and still get separate containerd roots, because a volume is rooted
+        // per service on whatever disk it names; sharing a root would corrupt
+        // it, since two containerd daemons cannot share a store (bolt takes
+        // an exclusive lock).
+        //
+        // This reverses a NOTE that stood here for a year: a fresh k3s server
+        // over a store killed un-cleanly by `docker rm -f` wedged the
+        // apiserver minutes in, and nothing recovered. What was missing was
+        // the discipline `base.rs::IMAGE_CACHE_UP_SH` applies to the guest's own
+        // store — sweep the crash state, and never inherit a store that did
+        // not work. Both are in STORE_SWEEP_SH.
+        volumes: [{ target: CLUSTER_STORE_TARGET }],
         ...(registriesYaml
             ? {
                 files: [