@specific.dev/spectest 0.70.0 → 0.71.1

package/src/components/k3s.ts

@@ -142,6 +142,119 @@ export interface K3sOptions {
 /** Port the in-cluster registry listens on (plain HTTP). */
 const K3S_REGISTRY_PORT = 5000;
 
+/** The cluster's own containerd root, inside the k3s container. */
+const CLUSTER_STORE_TARGET = "/var/lib/rancher/k3s/agent/containerd";
+
+
+/**
+ * Written before the server starts and removed by the ready probe.
+ *
+ * It survives in the store only when a boot never reached a live API
+ * server, which is the one thing the sweep below cannot work out for
+ * itself. Blacksmith's rule, from the other end: they refuse to commit a
+ * cache when the integrity check *did not run*, not only when it failed.
+ */
+const BOOT_MARKER = ".spectest-boot-incomplete";
+/**
+ * Bring the cluster's image store back into service, or throw it away.
+ *
+ * The store is the cluster's containerd root on a directory that outlives
+ * this container (`NESTED_STORE_ROOT`), so a build finds coredns, traefik,
+ * the local-path provisioner and every image the project deployed already
+ * pulled **and already extracted**. Extraction is 48x the I/O of the
+ * download (`STICKY_DISKS.md` §7.2), which is why the zot mirror never
+ * helped here and why an extracted store is the whole prize.
+ *
+ * What it inherits with them is one build's crash state: the store was
+ * captured from a running guest, and the teardown's `docker rm -f` killed
+ * that cluster where it stood. **Deleting the dead files is not enough,
+ * and the failure is not subtle.** containerd's index still lists the
+ * previous build's containers, so the CRI plugin offers them to a kubelet
+ * whose API server has never heard of them; the kubelet spends its startup
+ * reconciling containers whose tasks are gone (`Failed to create existing
+ * container … task not found`) and does not register its own Node in time,
+ * and the CSI plugin — which waits on that Node with a fixed budget —
+ * calls `Fatalf`. The kubelet dies, k3s exits 2, and what the author sees
+ * is a rollout that timed out against a hostname that no longer resolves.
+ * Measured: 2 of 2 delta restores, against 2 of 2 clean without the store.
+ *
+ * So the sweep removes those containers through **containerd itself**,
+ * which is the only thing that can edit its index. The k3s image ships a
+ * standalone `containerd` and `ctr`, so a throwaway daemon runs against
+ * the store with the CRI plugin disabled, deletes the previous build's
+ * containers and their tasks, and stops. Images, content blobs and
+ * extracted snapshots — the whole cache — stay. It is the same discipline
+ * `spectest-image-cache-up` applies to the guest's own store one level up,
+ * where it sweeps the `moby` namespace before dockerd starts.
+ *
+ * Two failures cost a cache and never a build:
+ *
+ *  1. **A sweep that cannot run wipes the store.** A daemon that will not
+ *     start leaves behind exactly the containers this cluster fatals on,
+ *     so carrying on is the one option that is known bad.
+ *  2. **A store that did not work is never inherited twice.** The marker
+ *     is written before the server starts and removed by the ready probe,
+ *     so it survives only when a boot never reached a live API server.
+ *     That is the backstop for the sweep being incomplete in some way we
+ *     have not seen: the cost is one cold build's pulls, and it repairs
+ *     itself. Blacksmith's rule from the other end — they refuse to commit
+ *     a cache when the integrity check *did not run*, not only when it
+ *     failed.
+ */
+/** Plugins the sweep daemon does not need. Every plugin is startup time,
+ *  and the sweep sits on the critical path of the cluster's boot. CRI is
+ *  off for a second reason: this daemon exists to edit an index, and a CRI
+ *  plugin would set about being a container runtime on a store we are
+ *  seconds from handing to the real one. */
+const SWEEP_DISABLED_PLUGINS = [
+  "io.containerd.grpc.v1.cri",
+  "io.containerd.snapshotter.v1.btrfs",
+  "io.containerd.snapshotter.v1.native",
+  "io.containerd.snapshotter.v1.aufs",
+  "io.containerd.snapshotter.v1.zfs",
+  "io.containerd.snapshotter.v1.devmapper",
+  "io.containerd.snapshotter.v1.stargz",
+  "io.containerd.snapshotter.v1.fuse-overlayfs",
+];
+
+const STORE_SWEEP_SH =
+  `S=${CLUSTER_STORE_TARGET}; M="$S/${BOOT_MARKER}"; A=/run/spectest-sweep.sock; ` +
+  `sweep_store() { ` +
+  `printf '%s\\n' 'version = 2' ` +
+  `'disabled_plugins = [${SWEEP_DISABLED_PLUGINS.map((p) => `"${p}"`).join(", ")}]' ` +
+  `> /run/spectest-sweep.toml || return 1; ` +
+  `containerd -c /run/spectest-sweep.toml --root "$S" --state /run/spectest-sweep ` +
+  `--address "$A" > /run/spectest-sweep.log 2>&1 & ` +
+  `cd_pid=$!; i=0; ` +
+  `while [ ! -S "$A" ] && [ $i -lt 600 ]; do sleep 0.1; i=$((i+1)); done; ` +
+  `if [ ! -S "$A" ]; then kill $cd_pid 2>/dev/null; return 1; fi; ` +
+  // One invocation for the whole set, not two per container. Each `ctr` is
+  // a Go binary start plus a gRPC round trip, and a build leaves ~20
+  // containers behind — the per-container form spent seconds of the
+  // cluster's boot on process starts alone (measured: ready 3.6s -> 7.0s).
+  `ids=$(timeout 30 ctr -a "$A" -n k8s.io containers ls -q 2>/dev/null); ` +
+  `if [ -n "$ids" ]; then ` +
+  `timeout 60 ctr -a "$A" -n k8s.io tasks rm -f $ids >/dev/null 2>&1 || true; ` +
+  `timeout 60 ctr -a "$A" -n k8s.io containers rm $ids >/dev/null 2>&1 || true; fi; ` +
+  // containerd 2.x keeps sandboxes in a store of their own; on the 1.7 k3s
+  // ships they are ordinary containers and this is a no-op.
+  `sb=$(timeout 30 ctr -a "$A" -n k8s.io sandboxes ls -q 2>/dev/null); ` +
+  `[ -n "$sb" ] && timeout 60 ctr -a "$A" -n k8s.io sandboxes rm $sb >/dev/null 2>&1; ` +
+  `kill $cd_pid 2>/dev/null; wait $cd_pid 2>/dev/null; ` +
+  `rm -rf "$S/io.containerd.runtime.v2.task" "$S/tmpmounts" /run/spectest-sweep "$A" 2>/dev/null || true; ` +
+  `return 0; }; ` +
+  `if [ -e "$M" ]; then ` +
+  `echo 'spectest: the previous cluster on this image store never became ready; starting from an empty one' >&2; ` +
+  `rm -rf "$S"/* "$S"/.[!.]* 2>/dev/null || true; ` +
+  `elif [ -d "$S/io.containerd.metadata.v1.bolt" ]; then ` +
+  // Timed and printed always: the sweep is inside the cluster's ready time,
+  // so it is the first thing to suspect if that time moves.
+  `T0=$(date +%s); ` +
+  `if sweep_store; then echo "spectest: reused the cluster image store at $S (swept in $(($(date +%s)-T0))s)"; ` +
+  `else echo 'spectest: could not sweep the inherited image store; starting from an empty one' >&2; ` +
+  `rm -rf "$S"/* "$S"/.[!.]* 2>/dev/null || true; fi; fi; ` +
+  `mkdir -p "$S" && : > "$M" 2>/dev/null || true; `;
+
 /**
  * `extraArgs` entries the component overrides anyway. Passing one of
  * these looks like it works — the flag really is appended to the k3s
@@ -1013,53 +1126,22 @@ ${ports}${volumeMounts}${volumes}
 }
 
 /**
- * Host-side `zot` pull-through cache layout (local Firecracker provider
- * only). One zot instance per upstream registry, all bound to the
- * `spectest-br0` gateway `10.42.0.1` on the ports below — **kept in sync
- * with `scripts/install-zot.sh`**. We mirror the cluster's containerd
- * through these so every image pull reuses the shared host cache instead
- * of hitting the public registry, and we list the canonical upstream as
- * a fallback endpoint so a missing/cold mirror only ever slows a pull,
- * never breaks it.
+ * Docker Hub through Google's public mirror, with the canonical upstream
+ * as the fallback endpoint so a cold or unavailable mirror only ever
+ * slows a pull, never breaks it. The same route the guest's own dockerd
+ * takes (golden's daemon.json) and its in-VM BuildKit. Every other
+ * registry is pulled direct. Nothing here points at a host-side service:
+ * the cluster's own containerd store lives in the environment and is
+ * captured with it (CONTAINER_STORE.md).
  */
-const ZOT_MIRRORS: Array<{ registry: string; port: number; upstream: string }> = [
-  { registry: "docker.io", port: 5000, upstream: "https://registry-1.docker.io" },
-  { registry: "ghcr.io", port: 5001, upstream: "https://ghcr.io" },
-  { registry: "quay.io", port: 5002, upstream: "https://quay.io" },
-  { registry: "registry.k8s.io", port: 5003, upstream: "https://registry.k8s.io" },
-  { registry: "public.ecr.aws", port: 5004, upstream: "https://public.ecr.aws" },
-  { registry: "gcr.io", port: 5005, upstream: "https://gcr.io" },
-  { registry: "mcr.microsoft.com", port: 5006, upstream: "https://mcr.microsoft.com" },
-];
-
-/**
- * Discover the host-side image cache gateway by reading the same
- * `registry-mirrors` entry the in-VM dockerd already uses (baked into
- * the golden rootfs's `/etc/docker/daemon.json`). Returns the
- * gateway host (`"10.42.0.1"`) when present, or `null` when there's no
- * host cache, in which case the cluster pulls every image direct. Runs inside the daemon (VM) at `index.ts` load time, so
- * the result is stable per host and never poisons the warm-template
- * cache.
- */
-function detectHostMirrorGateway(): string | null {
-  try {
-    const cfg = JSON.parse(
-      readFileSync("/etc/docker/daemon.json", "utf8"),
-    ) as { "registry-mirrors"?: string[] };
-    const first = cfg["registry-mirrors"]?.[0];
-    return first ? new URL(first).hostname || null : null;
-  } catch {
-    return null;
-  }
-}
+const HUB_MIRROR = { registry: "docker.io", mirror: "https://mirror.gcr.io", upstream: "https://registry-1.docker.io" };
 
 /**
  * Build `/etc/rancher/k3s/registries.yaml`. k3s reads this **once, at
  * startup**, to configure its embedded containerd — which is why it has
  * to be seeded via `files` (a pre-start bind mount) rather than a
  * `setup` hook. Two jobs:
- *   1. Mirror the cluster's image pulls through the host `zot` cache
- *      (omitted when there's no host cache).
+ *   1. Mirror the cluster's Docker Hub pulls through `mirror.gcr.io`.
  *   2. Trust the in-cluster registry, addressed as `<key>.internal:5000`
  *      (the `{{SPECTEST_SERVICE}}` token is expanded to the cluster's
  *      service key when the file is written). Image *references* use
@@ -1067,24 +1149,15 @@ function detectHostMirrorGateway(): string | null {
  *      endpoint `http://127.0.0.1:5000` — the hostNetwork registry pod
  *      shares the node's netns, so this needs no in-container DNS and
  *      can't be broken by a clobbered `hostnames`.
- * Returns `null` when there's nothing to configure (no host cache and
- * `registry` disabled), in which case no file is injected.
  */
 function buildRegistriesYaml(registryEnabled: boolean): string | null {
-  const gateway = detectHostMirrorGateway();
-  if (!gateway && !registryEnabled) return null;
-
-  const lines: string[] = ["mirrors:"];
-  if (gateway) {
-    for (const { registry, port, upstream } of ZOT_MIRRORS) {
-      lines.push(
-        `  "${registry}":`,
-        `    endpoint:`,
-        `      - "http://${gateway}:${port}"`,
-        `      - "${upstream}"`,
-      );
-    }
-  }
+  const lines: string[] = [
+    "mirrors:",
+    `  "${HUB_MIRROR.registry}":`,
+    `    endpoint:`,
+    `      - "${HUB_MIRROR.mirror}"`,
+    `      - "${HUB_MIRROR.upstream}"`,
+  ];
   if (registryEnabled) {
     const host = `{{SPECTEST_SERVICE}}.internal:${K3S_REGISTRY_PORT}`;
     lines.push(
@@ -1913,12 +1986,18 @@ export function k3s(opts: K3sOptions = {}) {
     // is affected.
     "mount --make-rshared / 2>/dev/null || " +
     "echo 'spectest: could not make / rshared; CSI node plugins may fail to publish volumes' >&2; " +
+    STORE_SWEEP_SH +
     `exec ${serverArgs}`;
   // Plain /readyz probe. On a warm zot cache the cluster's images are
   // already local, so the first boot completes in seconds; the
   // first-ever boot on a cold-cache host pulls through the mirror and
   // can take a couple of minutes (covered by readyTimeoutSecs).
-  const readyCmd = "kubectl get --raw=/readyz >/dev/null 2>&1";
+  //
+  // Clearing the boot marker is part of the probe on purpose: "the API
+  // server answers" is exactly the condition that proves this cluster came
+  // up on the store it was given, and it is the only signal the sweep can
+  // read on the next boot. See STORE_SWEEP_SH.
+  const readyCmd = `kubectl get --raw=/readyz >/dev/null 2>&1 && rm -f ${CLUSTER_STORE_TARGET}/${BOOT_MARKER}`;
   const def = {
     image: { type: "registry", reference: `rancher/k3s:${version}` },
     command: cmd,
@@ -1934,15 +2013,30 @@ export function k3s(opts: K3sOptions = {}) {
     // the same netns and are reachable the same way, without appearing
     // in this list (it's documentation, not a firewall).
     ports: registryEnabled ? [80, 443, 6443, K3S_REGISTRY_PORT] : [80, 443, 6443],
-    // NOTE: do NOT mount /var/lib/rancher/k3s/agent/containerd as a cache
-    // volume. It was tried (to spare a recreated cluster re-pulling its
-    // system images on delta restores) and a fresh k3s server against the
-    // previous container's containerd store — killed un-cleanly by the
-    // teardown's `docker rm -f` — wedged the apiserver minutes in
-    // (rollouts never settled, pod listing started failing). The zot
-    // mirror already makes those re-pulls cheap; the residual win wasn't
-    // worth the recovery semantics of a crash-state store under a fresh
-    // cluster db.
+    // The cluster's image store, on a volume that outlives this
+    // container. Without it the store is the container's writable layer,
+    // which `docker rm -f` deletes at teardown — so every build re-pulls
+    // and, far more expensively, re-extracts coredns, traefik, the
+    // local-path provisioner and everything the project deploys.
+    // Measured on the one project on the image cache: 20.6 s per
+    // build, identical on the cold and delta tiers, i.e. the one term
+    // neither the delta restore nor the cache image store reached.
+    //
+    // `cache` names a cache disk, which is the ordinary way any project
+    // asks for one — there is nothing Kubernetes-shaped in the platform
+    // for this. Two clusters in one environment may name the same disk
+    // and still get separate containerd roots, because a volume is rooted
+    // per service on whatever disk it names; sharing a root would corrupt
+    // it, since two containerd daemons cannot share a store (bolt takes
+    // an exclusive lock).
+    //
+    // This reverses a NOTE that stood here for a year: a fresh k3s server
+    // over a store killed un-cleanly by `docker rm -f` wedged the
+    // apiserver minutes in, and nothing recovered. What was missing was
+    // the discipline `base.rs::IMAGE_CACHE_UP_SH` applies to the guest's own
+    // store — sweep the crash state, and never inherit a store that did
+    // not work. Both are in STORE_SWEEP_SH.
+    volumes: [{ target: CLUSTER_STORE_TARGET }],
     ...(registriesYaml
       ? {
           files: [