@specific.dev/spectest 0.71.0 → 0.72.0

package/dist/components/k3s.d.ts

@@ -3,7 +3,13 @@ import type { ServiceHelpersContext } from "../index.js";
 import type { Wrapped } from "../inspect.js";
 export type { KubernetesObject, KubernetesListObject, } from "@kubernetes/client-node";
 export interface K3sOptions {
-    /** Image tag for the official `rancher/k3s` image. Default `"v1.30.6-k3s1"`. */
+    /**
+     * Image tag for the official `rancher/k3s` image. Default
+     * `"v1.33.5-k3s1"`. Any tag works; a k3s older than 1.33 embeds a
+     * containerd (1.7) without the EROFS snapshotter, so its cluster keeps
+     * its images on overlayfs and shares the environment's image cache only
+     * as a directory that survives — see `k3s()`'s store section.
+     */
     version?: string;
     /**
      * Extra arguments appended to `k3s server`. Useful for `--tls-san=...`,
@@ -309,20 +315,6 @@ export interface K3sHelpers {
     }): Promise<Wrapped<KubernetesListObject<T>>>;
 }
 export declare function k3s(opts?: K3sOptions): {
-    readyCheck: {
-        type: "exec";
-        command: string;
-        timeoutSecs: number;
-    };
-    setup: ({ name, helpers }: {
-        name: string;
-        helpers: K3sHelpers;
-    }) => Promise<void>;
-    helpers: ({ name, exec, poll }: ServiceHelpersContext) => Promise<K3sHelpers>;
-    files?: {
-        path: string;
-        content: string;
-    }[] | undefined;
     image: {
         type: "registry";
         reference: string;
@@ -332,7 +324,29 @@ export declare function k3s(opts?: K3sOptions): {
     tmpfs: string[];
     cgroupns: string;
     ports: number[];
-    volumes: {
+    volumes: ({
+        source: string;
+        target: string;
+        readOnly?: undefined;
+    } | {
+        source: string;
         target: string;
+        readOnly: true;
+    })[] | {
+        target: string;
+    }[];
+    files: {
+        path: string;
+        content: string;
     }[];
+    readyCheck: {
+        type: "exec";
+        command: string;
+        timeoutSecs: number;
+    };
+    setup: ({ name, helpers }: {
+        name: string;
+        helpers: K3sHelpers;
+    }) => Promise<void>;
+    helpers: ({ name, exec, poll }: ServiceHelpersContext) => Promise<K3sHelpers>;
 };

package/dist/components/k3s.js

@@ -6,12 +6,22 @@ import { readFile, unlink } from "node:fs/promises";
 import { AppsV1Api, BatchV1Api, CoreV1Api, KubeConfig, KubernetesObjectApi, PatchStrategy, ResponseContext, ServerConfiguration, createConfiguration, loadAllYaml, } from "@kubernetes/client-node";
 import { Observable } from "@kubernetes/client-node/dist/gen/rxjsStub.js";
 import { dnsName, provides, SELF_SERVICE_TOKEN } from "../index.js";
+import { MKFS_EROFS_PATH, STORE_ADOPT_PATH, imageCachePathsSync, nestedStoreDir, } from "../harness/image-cache.js";
 import { deepUnwrap, readTag, wrap } from "../inspect.js";
 import { recorderAnnotate, recorderRemove } from "../recorder.js";
 /** Port the in-cluster registry listens on (plain HTTP). */
 const K3S_REGISTRY_PORT = 5000;
 /** The cluster's own containerd root, inside the k3s container. */
 const CLUSTER_STORE_TARGET = "/var/lib/rancher/k3s/agent/containerd";
+/** Where k3s (containerd 2.x layout) reads a containerd config template. */
+const CONTAINERD_TEMPLATE_PATH = "/var/lib/rancher/k3s/agent/etc/containerd/config-v3.toml.tmpl";
+/** The EROFS settings that template imports; the store's prepare daemon
+ *  imports the same file, so both daemons agree on the snapshotter. */
+const CONTAINERD_EROFS_CONFIG_PATH = "/etc/spectest/containerd-erofs.toml";
+/** The socket of the throwaway daemon that prepares the store. */
+const PREPARE_SOCKET = "/run/spectest-prepare.sock";
+/** containerd's CRI plugin lists only images carrying this label. */
+const CRI_MANAGED_LABEL = "io.cri-containerd.image=managed";
 /**
  * Written before the server starts and removed by the ready probe.
  *
@@ -22,36 +32,62 @@ const CLUSTER_STORE_TARGET = "/var/lib/rancher/k3s/agent/containerd";
  */
 const BOOT_MARKER = ".spectest-boot-incomplete";
 /**
- * Bring the cluster's image store back into service, or throw it away.
+ * The cluster's image store, and how it shares the environment's image
+ * cache (`CONTAINER_STORE.md` §12).
  *
- * The store is the cluster's containerd root on a directory that outlives
- * this container (`NESTED_STORE_ROOT`), so a build finds coredns, traefik,
- * the local-path provisioner and every image the project deployed already
- * pulled **and already extracted**. Extraction is 48x the I/O of the
- * download (`STICKY_DISKS.md` §7.2), which is why the zot mirror never
- * helped here and why an extracted store is the whole prize.
+ * The cluster's embedded containerd keeps its root on the **root disk**
+ * of the environment's image cache, in a directory of its own
+ * (`<root>/spectest-nested/<service>`, bind-mounted at
+ * `CLUSTER_STORE_TARGET`), and runs the **EROFS snapshotter** over the
+ * same read-only **layers disk** the guest's own containerd uses,
+ * mounted into the container at the same path. So a layer is one file
+ * on the host whichever runtime holds it, the cluster's pulls land in
+ * the lineage's root clone like a `docker pull` does, and the merge
+ * reads the cluster's indexes exactly as it reads dockerd's — at every
+ * VM teardown, whether the pull happened at env boot or in the middle
+ * of a test. Nothing distinguishes the two: a fork's root clone grew,
+ * so it is harvested. The next cold build's cluster **adopts** what the
+ * merge put on the layers disk under its own namespace (`k8s.io`) with
+ * the same helper dockerd's store uses, before the cluster starts.
  *
- * What it inherits with them is one build's crash state: the store was
- * captured from a running guest, and the teardown's `docker rm -f` killed
- * that cluster where it stood. **Deleting the dead files is not enough,
- * and the failure is not subtle.** containerd's index still lists the
- * previous build's containers, so the CRI plugin offers them to a kubelet
- * whose API server has never heard of them; the kubelet spends its startup
- * reconciling containers whose tasks are gone (`Failed to create existing
- * container … task not found`) and does not register its own Node in time,
- * and the CSI plugin — which waits on that Node with a fixed budget —
- * calls `Fatalf`. The kubelet dies, k3s exits 2, and what the author sees
- * is a rollout that timed out against a hostname that no longer resolves.
- * Measured: 2 of 2 delta restores, against 2 of 2 clean without the store.
+ * Extraction is 48x the I/O of the download (`STICKY_DISKS.md` §7.2),
+ * which is why a mirror never helped here and an extracted, shared
+ * store is the whole prize. Measured on the one project on the old
+ * per-environment store: 20.6 s of cluster bring-up per build.
  *
- * So the sweep removes those containers through **containerd itself**,
- * which is the only thing that can edit its index. The k3s image ships a
- * standalone `containerd` and `ctr`, so a throwaway daemon runs against
- * the store with the CRI plugin disabled, deletes the previous build's
- * containers and their tasks, and stops. Images, content blobs and
- * extracted snapshots — the whole cache — stay. It is the same discipline
- * `spectest-image-cache-up` applies to the guest's own store one level up,
- * where it sweeps the `moby` namespace before dockerd starts.
+ * Three things the container needs for it, all mounted by `k3s()`:
+ * golden's static `mkfs.erofs` (the EROFS differ needs it and no k3s
+ * image ships one); the adopt helper (POSIX sh, run by busybox); and a
+ * containerd config template that imports the EROFS settings — k3s
+ * generates its config from `config-v3.toml.tmpl` when present, and an
+ * `imports` line on top of the stock template is the one way to add
+ * plugin sections without colliding with the ones it writes. The
+ * EROFS snapshotter mounts every layer through a loop device, and a
+ * privileged container's `/dev` is a static copy that never sees the
+ * nodes the kernel allocates later, so the wrapper pre-creates 1024 of
+ * them (measured: 0 s).
+ *
+ * What the store inherits with the cache is one build's crash state:
+ * the root was captured from a running guest, and the teardown's
+ * `docker rm -f` killed that cluster where it stood. **Deleting the dead
+ * files is not enough, and the failure is not subtle.** containerd's
+ * index still lists the previous build's containers, so the CRI plugin
+ * offers them to a kubelet whose API server has never heard of them; the
+ * kubelet spends its startup reconciling containers whose tasks are gone
+ * (`Failed to create existing container … task not found`) and does not
+ * register its own Node in time, and the CSI plugin — which waits on that
+ * Node with a fixed budget — calls `Fatalf`. The kubelet dies, k3s exits
+ * 2, and what the author sees is a rollout that timed out against a
+ * hostname that no longer resolves. Measured: 2 of 2 delta restores,
+ * against 2 of 2 clean without the store.
+ *
+ * So a **throwaway daemon** on the store, with the CRI plugin disabled,
+ * does both jobs before k3s starts: it deletes the previous build's
+ * containers and their tasks through containerd itself (the only thing
+ * that can edit its index), and it adopts the inbox. Images, content
+ * blobs and extracted snapshots — the whole cache — stay. It is the same
+ * discipline `spectest-image-cache-up` applies to the guest's own store
+ * one level up.
  *
  * Two failures cost a cache and never a build:
  *
@@ -63,15 +99,17 @@ const BOOT_MARKER = ".spectest-boot-incomplete";
  *     so it survives only when a boot never reached a live API server.
  *     That is the backstop for the sweep being incomplete in some way we
  *     have not seen: the cost is one cold build's pulls, and it repairs
- *     itself. Blacksmith's rule from the other end — they refuse to commit
- *     a cache when the integrity check *did not run*, not only when it
- *     failed.
+ *     itself.
+ *
+ * Without an image cache (the fake backend, a server that predates it)
+ * the store is a per-environment volume as before, on overlayfs, with
+ * the same sweep.
  */
-/** Plugins the sweep daemon does not need. Every plugin is startup time,
- *  and the sweep sits on the critical path of the cluster's boot. CRI is
- *  off for a second reason: this daemon exists to edit an index, and a CRI
- *  plugin would set about being a container runtime on a store we are
- *  seconds from handing to the real one. */
+/** Plugins the prepare daemon does not need. Every plugin is startup
+ *  time, and the daemon sits on the critical path of the cluster's boot.
+ *  CRI is off for a second reason: this daemon exists to edit an index,
+ *  and a CRI plugin would set about being a container runtime on a store
+ *  we are seconds from handing to the real one. */
 const SWEEP_DISABLED_PLUGINS = [
     "io.containerd.grpc.v1.cri",
     "io.containerd.snapshotter.v1.btrfs",
@@ -82,42 +120,101 @@ const SWEEP_DISABLED_PLUGINS = [
     "io.containerd.snapshotter.v1.stargz",
     "io.containerd.snapshotter.v1.fuse-overlayfs",
 ];
-const STORE_SWEEP_SH = `S=${CLUSTER_STORE_TARGET}; M="$S/${BOOT_MARKER}"; A=/run/spectest-sweep.sock; ` +
-    `sweep_store() { ` +
-    `printf '%s\\n' 'version = 2' ` +
-    `'disabled_plugins = [${SWEEP_DISABLED_PLUGINS.map((p) => `"${p}"`).join(", ")}]' ` +
-    `> /run/spectest-sweep.toml || return 1; ` +
-    `containerd -c /run/spectest-sweep.toml --root "$S" --state /run/spectest-sweep ` +
-    `--address "$A" > /run/spectest-sweep.log 2>&1 & ` +
-    `cd_pid=$!; i=0; ` +
-    `while [ ! -S "$A" ] && [ $i -lt 600 ]; do sleep 0.1; i=$((i+1)); done; ` +
-    `if [ ! -S "$A" ]; then kill $cd_pid 2>/dev/null; return 1; fi; ` +
-    // One invocation for the whole set, not two per container. Each `ctr` is
-    // a Go binary start plus a gRPC round trip, and a build leaves ~20
-    // containers behind — the per-container form spent seconds of the
-    // cluster's boot on process starts alone (measured: ready 3.6s -> 7.0s).
-    `ids=$(timeout 30 ctr -a "$A" -n k8s.io containers ls -q 2>/dev/null); ` +
-    `if [ -n "$ids" ]; then ` +
-    `timeout 60 ctr -a "$A" -n k8s.io tasks rm -f $ids >/dev/null 2>&1 || true; ` +
-    `timeout 60 ctr -a "$A" -n k8s.io containers rm $ids >/dev/null 2>&1 || true; fi; ` +
-    // containerd 2.x keeps sandboxes in a store of their own; on the 1.7 k3s
-    // ships they are ordinary containers and this is a no-op.
-    `sb=$(timeout 30 ctr -a "$A" -n k8s.io sandboxes ls -q 2>/dev/null); ` +
-    `[ -n "$sb" ] && timeout 60 ctr -a "$A" -n k8s.io sandboxes rm $sb >/dev/null 2>&1; ` +
-    `kill $cd_pid 2>/dev/null; wait $cd_pid 2>/dev/null; ` +
-    `rm -rf "$S/io.containerd.runtime.v2.task" "$S/tmpmounts" /run/spectest-sweep "$A" 2>/dev/null || true; ` +
-    `return 0; }; ` +
-    `if [ -e "$M" ]; then ` +
-    `echo 'spectest: the previous cluster on this image store never became ready; starting from an empty one' >&2; ` +
-    `rm -rf "$S"/* "$S"/.[!.]* 2>/dev/null || true; ` +
-    `elif [ -d "$S/io.containerd.metadata.v1.bolt" ]; then ` +
-    // Timed and printed always: the sweep is inside the cluster's ready time,
-    // so it is the first thing to suspect if that time moves.
-    `T0=$(date +%s); ` +
-    `if sweep_store; then echo "spectest: reused the cluster image store at $S (swept in $(($(date +%s)-T0))s)"; ` +
-    `else echo 'spectest: could not sweep the inherited image store; starting from an empty one' >&2; ` +
-    `rm -rf "$S"/* "$S"/.[!.]* 2>/dev/null || true; fi; fi; ` +
-    `mkdir -p "$S" && : > "$M" 2>/dev/null || true; `;
+const DISABLED_PLUGINS_TOML = `disabled_plugins = [${SWEEP_DISABLED_PLUGINS.map((p) => `"${p}"`).join(", ")}]`;
+/** The containerd config template: the stock one, importing our EROFS
+ *  settings. `imports` is a top-level key and has to come first. */
+const CONTAINERD_TEMPLATE = `imports = ["${CONTAINERD_EROFS_CONFIG_PATH}"]\n{{ template "base" . }}\n`;
+/**
+ * containerd's EROFS settings, the same golden bakes for the guest's
+ * own daemon (`local-vms-rootfs.Dockerfile`): the differ first, the
+ * transfer service unpacking onto EROFS, and `set_immutable` off because
+ * a committed layer's file is a symlink into a read-only disk, on which
+ * the immutable ioctl would fail the commit.
+ */
+function erofsConfigToml() {
+    const arch = process.arch === "arm64" ? "arm64" : process.arch === "x64" ? "amd64" : process.arch;
+    return [
+        "version = 3",
+        '[plugins."io.containerd.snapshotter.v1.erofs"]',
+        "  set_immutable = false",
+        '[plugins."io.containerd.differ.v1.erofs"]',
+        '  mkfs_options = ["-T0", "--mkfs-time", "--sort=none"]',
+        '[plugins."io.containerd.service.v1.diff-service"]',
+        '  default = ["erofs","walking"]',
+        '[plugins."io.containerd.transfer.v1.local"]',
+        '  [[plugins."io.containerd.transfer.v1.local".unpack_config]]',
+        `    platform = "linux/${arch}"`,
+        '    snapshotter = "erofs"',
+        '    differ = "erofs"',
+        "",
+    ].join("\n");
+}
+/**
+ * The store-preparation half of the command wrapper: sweep the crash
+ * state, adopt the cache's inbox (image-cache mode), leave `SNAP` set to
+ * the snapshotter the cluster starts on.
+ *
+ * Runs under busybox `sh`. Every `ctr` call is one invocation for the
+ * whole set, not one per container: each is a Go binary start plus a
+ * gRPC round trip, and a build leaves ~20 containers behind — the
+ * per-container form spent seconds of the cluster's boot on process
+ * starts alone (measured: ready 3.6 s -> 7.0 s).
+ */
+function storePrepareSh(cache) {
+    const daemonConfig = cache
+        ? // containerd 2.x (v3 config, EROFS) when the image's containerd has
+            // the snapshotter; a 1.7 image (k3s < 1.33) gets the v2 shape it
+            // understands and stays on overlayfs.
+            `if [ "$EROFS" = 1 ]; then printf '%s\\n' 'version = 3' 'imports = ["${CONTAINERD_EROFS_CONFIG_PATH}"]' '${DISABLED_PLUGINS_TOML}' > /run/spectest-prepare.toml; ` +
+                `else printf '%s\\n' 'version = 2' '${DISABLED_PLUGINS_TOML}' > /run/spectest-prepare.toml; fi; `
+        : `printf '%s\\n' 'version = 2' '${DISABLED_PLUGINS_TOML}' > /run/spectest-prepare.toml; `;
+    const detect = cache
+        ? `EROFS=0; if containerd config default 2>/dev/null | grep -q 'snapshotter.v1.erofs'; then EROFS=1; fi; ` +
+            `if [ "$EROFS" = 1 ]; then SNAP=erofs; i=0; while [ $i -lt 1024 ]; do [ -e /dev/loop$i ] || mknod /dev/loop$i b 7 $i 2>/dev/null; i=$((i+1)); done; ` +
+            `else SNAP=overlayfs; echo 'spectest: this k3s image has no EROFS snapshotter; the cluster keeps its images on overlayfs and shares the image cache as a directory only' >&2; fi; `
+        : `EROFS=0; SNAP=overlayfs; `;
+    const adopt = cache
+        ? `adopt_store() { [ "$EROFS" = 1 ] && [ -f "$L/spectest-inbox/layers.txt" ] || return 0; ` +
+            `${STORE_ADOPT_PATH} "$S" "$L" -n k8s.io -a "$A" -l ${CRI_MANAGED_LABEL} 2>&1 | tail -1; }; `
+        : `adopt_store() { :; }; `;
+    return (`S=${CLUSTER_STORE_TARGET}; M="$S/${BOOT_MARKER}"; A=${PREPARE_SOCKET}; L=${cache ? cache.layers : "/nonexistent"}; ` +
+        detect +
+        `start_daemon() { ` +
+        daemonConfig +
+        `containerd -c /run/spectest-prepare.toml --root "$S" --state /run/spectest-prepare ` +
+        `--address "$A" > /run/spectest-prepare.log 2>&1 & ` +
+        `cd_pid=$!; i=0; ` +
+        `while [ ! -S "$A" ] && [ $i -lt 600 ]; do sleep 0.1; i=$((i+1)); done; ` +
+        `if [ ! -S "$A" ]; then kill $cd_pid 2>/dev/null; return 1; fi; return 0; }; ` +
+        `stop_daemon() { kill $cd_pid 2>/dev/null; wait $cd_pid 2>/dev/null; ` +
+        `rm -rf "$S/io.containerd.runtime.v2.task" "$S/tmpmounts" /run/spectest-prepare "$A" 2>/dev/null || true; }; ` +
+        `sweep_store() { ` +
+        `ids=$(timeout 30 ctr -a "$A" -n k8s.io containers ls -q 2>/dev/null); ` +
+        `if [ -n "$ids" ]; then ` +
+        `timeout 60 ctr -a "$A" -n k8s.io tasks rm -f $ids >/dev/null 2>&1 || true; ` +
+        `timeout 60 ctr -a "$A" -n k8s.io containers rm $ids >/dev/null 2>&1 || true; fi; ` +
+        // containerd 2.x keeps sandboxes in a store of their own; on the 1.7
+        // k3s ships they are ordinary containers and this is a no-op.
+        `sb=$(timeout 30 ctr -a "$A" -n k8s.io sandboxes ls -q 2>/dev/null); ` +
+        `[ -n "$sb" ] && timeout 60 ctr -a "$A" -n k8s.io sandboxes rm $sb >/dev/null 2>&1; return 0; }; ` +
+        adopt +
+        `if [ -e "$M" ]; then ` +
+        `echo 'spectest: the previous cluster on this image store never became ready; starting from an empty one' >&2; ` +
+        `rm -rf "$S"/* "$S"/.[!.]* 2>/dev/null || true; fi; ` +
+        `mkdir -p "$S"; ` +
+        `had_state=0; [ -d "$S/io.containerd.metadata.v1.bolt" ] && had_state=1; ` +
+        `T0=$(date +%s); ` +
+        `if [ "$had_state" = 1 ] || [ "$EROFS" = 1 ]; then ` +
+        `if start_daemon; then ` +
+        `[ "$had_state" = 1 ] && sweep_store; adopt_store; stop_daemon; ` +
+        // Timed and printed always: this is inside the cluster's ready time,
+        // so it is the first thing to suspect if that time moves.
+        `echo "spectest: cluster image store at $S ready on $SNAP (prepared in $(($(date +%s)-T0))s)"; ` +
+        `elif [ "$had_state" = 1 ]; then ` +
+        `echo 'spectest: could not sweep the inherited image store; starting from an empty one' >&2; ` +
+        `rm -rf "$S"/* "$S"/.[!.]* 2>/dev/null || true; fi; fi; ` +
+        `: > "$M" 2>/dev/null || true; `);
+}
 /**
  * `extraArgs` entries the component overrides anyway. Passing one of
  * these looks like it works — the flag really is appended to the k3s
@@ -1286,11 +1383,14 @@ async function collectDiagnostics(client, namespace) {
  * so neither cold-cache nor warm starts re-pull. Any `opts.version`
  * works — there's no base-snapshot release to keep in sync with.
  *
- * **Why v1.32.x:** kube-proxy's `nftables` proxy mode is GA in k8s 1.32
- * (beta in 1.31, alpha-gated in 1.30). The component runs kube-proxy in
- * that mode; dropping below 1.31 falls back to the iptables path.
+ * **Why v1.33.x:** k3s 1.33.5 is the first line embedding containerd
+ * 2.1 (2.1.4-k3s1), whose in-tree EROFS snapshotter is what lets the
+ * cluster share the environment's image cache (see the store section
+ * above); 1.32 ships containerd 1.7. kube-proxy's `nftables` proxy mode
+ * is GA since k8s 1.32 (beta in 1.31, alpha-gated in 1.30); dropping
+ * below 1.31 falls back to the iptables path.
  */
-const DEFAULT_K3S_VERSION = "v1.32.1-k3s1";
+const DEFAULT_K3S_VERSION = "v1.33.5-k3s1";
 export function k3s(opts = {}) {
     const version = opts.version ?? DEFAULT_K3S_VERSION;
     const extra = opts.extraArgs ?? [];
@@ -1306,6 +1406,10 @@ export function k3s(opts = {}) {
     // the in-cluster registry). Seeded via `files` because k3s reads it
     // only at startup, before any setup hook could run.
     const registriesYaml = buildRegistriesYaml(registryEnabled);
+    // The environment's image cache, when this VM carries one: the
+    // cluster's store goes on its root disk and its layers come off the
+    // shared layers disk (see the store section above).
+    const cache = imageCachePathsSync();
     const serverArgs = [
         "k3s",
         "server",
@@ -1387,8 +1491,10 @@ export function k3s(opts = {}) {
         // is affected.
         "mount --make-rshared / 2>/dev/null || " +
         "echo 'spectest: could not make / rshared; CSI node plugins may fail to publish volumes' >&2; " +
-        STORE_SWEEP_SH +
-        `exec ${serverArgs}`;
+        storePrepareSh(cache) +
+        // The snapshotter is decided by the wrapper: EROFS on the image cache
+        // when the image's containerd has it, overlayfs otherwise.
+        `exec ${serverArgs} --snapshotter=$SNAP`;
     // Plain /readyz probe. On a warm zot cache the cluster's images are
     // already local, so the first boot completes in seconds; the
     // first-ever boot on a cold-cache host pulls through the mirror and
@@ -1414,37 +1520,30 @@ export function k3s(opts = {}) {
         // the same netns and are reachable the same way, without appearing
         // in this list (it's documentation, not a firewall).
         ports: registryEnabled ? [80, 443, 6443, K3S_REGISTRY_PORT] : [80, 443, 6443],
-        // The cluster's image store, on a volume that outlives this
-        // container. Without it the store is the container's writable layer,
-        // which `docker rm -f` deletes at teardown — so every build re-pulls
-        // and, far more expensively, re-extracts coredns, traefik, the
-        // local-path provisioner and everything the project deploys.
-        // Measured on the one project on the image cache: 20.6 s per
-        // build, identical on the cold and delta tiers, i.e. the one term
-        // neither the delta restore nor the cache image store reached.
-        //
-        // `cache` names a cache disk, which is the ordinary way any project
-        // asks for one — there is nothing Kubernetes-shaped in the platform
-        // for this. Two clusters in one environment may name the same disk
-        // and still get separate containerd roots, because a volume is rooted
-        // per service on whatever disk it names; sharing a root would corrupt
-        // it, since two containerd daemons cannot share a store (bolt takes
-        // an exclusive lock).
-        //
-        // This reverses a NOTE that stood here for a year: a fresh k3s server
-        // over a store killed un-cleanly by `docker rm -f` wedged the
-        // apiserver minutes in, and nothing recovered. What was missing was
-        // the discipline `base.rs::IMAGE_CACHE_UP_SH` applies to the guest's own
-        // store — sweep the crash state, and never inherit a store that did
-        // not work. Both are in STORE_SWEEP_SH.
-        volumes: [{ target: CLUSTER_STORE_TARGET }],
-        ...(registriesYaml
-            ? {
-                files: [
-                    { path: "/etc/rancher/k3s/registries.yaml", content: registriesYaml },
-                ],
-            }
-            : {}),
+        // The cluster's image store (see the store section above): its
+        // containerd root on the image cache's root disk, the shared layers
+        // disk read-only at its guest path, golden's mkfs.erofs and the adopt
+        // helper — or, with no cache, a per-environment volume. Two clusters
+        // in one environment get separate roots either way (the service
+        // token names the directory), because two containerd daemons cannot
+        // share a store: bolt takes an exclusive lock.
+        volumes: cache
+            ? [
+                { source: nestedStoreDir(cache, SELF_SERVICE_TOKEN), target: CLUSTER_STORE_TARGET },
+                { source: cache.layers, target: cache.layers, readOnly: true },
+                { source: MKFS_EROFS_PATH, target: MKFS_EROFS_PATH, readOnly: true },
+                { source: STORE_ADOPT_PATH, target: STORE_ADOPT_PATH, readOnly: true },
+            ]
+            : [{ target: CLUSTER_STORE_TARGET }],
+        files: [
+            ...(registriesYaml ? [{ path: "/etc/rancher/k3s/registries.yaml", content: registriesYaml }] : []),
+            ...(cache
+                ? [
+                    { path: CONTAINERD_TEMPLATE_PATH, content: CONTAINERD_TEMPLATE },
+                    { path: CONTAINERD_EROFS_CONFIG_PATH, content: erofsConfigToml() },
+                ]
+                : []),
+        ],
         readyCheck: {
             type: "exec",
             command: readyCmd,

package/dist/daemon.js

@@ -50,6 +50,7 @@ import { certCovers as hostmatchCertCovers, hostWithoutPort, matchRoute, selectC
 import { INGRESS_HTTPS_PORT, INGRESS_HTTP_PORT, bindRoute, clearTables, emptyTables, planBind, registryTarget, routesFor, unbindRoute, } from "./harness/ingress-table.js";
 import { startTlsTerminator } from "./harness/tls-terminator.js";
 import { runContainerArgs } from "./harness/container-run.js";
+import { IMAGE_CACHE_MANIFEST, imageCachePathsSync, isOnImageCache } from "./harness/image-cache.js";
 import { assertAbsolute, certificateHostnames, defaultKeyMode, expandServiceToken, isNoopChown, mountFlag, needsIdTables, numericId, resolveChownIds, } from "./harness/file-mounts.js";
 import { conflict, notFound, requireString, } from "./harness/methods.js";
 import { openTerminal } from "./terminal.js";
@@ -448,7 +449,6 @@ async function hasBuildx() {
 // in-VM buildkitd keeps its exported cache there too, so a fresh VM finds
 // every layer it built before. Detected once; if the daemon will not
 // start, dockerd's own BuildKit builds instead.
-const IMAGE_CACHE_MANIFEST = "/run/spectest-image-cache.json";
 const LOCAL_BUILDER_NAME = "spectest-local";
 const LOCAL_BUILDKIT_ADDR = "tcp://127.0.0.1:1234";
 /** The bring-up script the cache base bakes (`base.rs::BUILDKITD_UP_SH`). */
@@ -457,16 +457,7 @@ const BUILDKITD_UP_PATH = "/usr/local/bin/spectest-buildkitd-up";
  *  root (read-write, this VM's own) and the layers disk (read-only, shared
  *  by every VM of a generation). `null` when the VM carries no cache. */
 async function imageCachePaths() {
-    try {
-        const raw = await fs.readFile(IMAGE_CACHE_MANIFEST, "utf8");
-        const parsed = JSON.parse(raw);
-        const root = (parsed.disks ?? []).find((d) => d.role === "root" && d.path)?.path;
-        const layers = (parsed.disks ?? []).find((d) => d.role === "layers" && d.path)?.path;
-        return root && layers ? { root, layers } : null;
-    }
-    catch {
-        return null;
-    }
+    return imageCachePathsSync(IMAGE_CACHE_MANIFEST);
 }
 let _localBuilder;
 /**
@@ -602,7 +593,10 @@ async function ensureVolumes(svc) {
             // serving older SDKs. Leaving it out of the manifest is what
             // protects a project running this SDK against a server whose
             // teardown guard predates it.
-            const durable = host.startsWith("/var/cache/spectest/");
+            // A directory on a cache disk is the same kind of thing: a nested
+            // runtime's containerd root (`k3s()`), kept as a cache by the
+            // lineage exactly as the container store one level up is.
+            const durable = host.startsWith("/var/cache/spectest/") || isOnImageCache(host, imageCachePathsSync(IMAGE_CACHE_MANIFEST));
             if (vol.source?.startsWith("/") && !durable) {
                 await recordAbsoluteVolumeDir(host);
             }
@@ -1121,12 +1115,18 @@ async function runServiceBuild(name, image, tag, caSuffix) {
             // intermediate layer, uncompressed so an import never inflates,
             // and one tag per service so exports do not replace each other.
             const cacheTag = name.replace(/[^a-z0-9-]/gi, "-").toLowerCase();
+            // LANDMINE: the local cache importer reads the `latest` entry of
+            // the directory's index unless told otherwise, and the exporter
+            // below writes this service's entry under `tag=<service>`. An
+            // import without the same tag misses every time and every
+            // `RUN` re-executes on a seeded cache (seen on the first deploy,
+            // 2026-09-05: the disk carried the blobs, the build used none).
             buildArgs = [
                 "buildx", "build",
                 "--builder", LOCAL_BUILDER_NAME,
                 "--progress=plain",
                 "--output", `type=image,name=${qualifyImageRef(tag)},unpack=true`,
-                ..._localCacheImports.flatMap((src) => ["--cache-from", `type=local,src=${src}`]),
+                ..._localCacheImports.flatMap((src) => ["--cache-from", `type=local,src=${src},tag=${cacheTag}`]),
                 "--cache-to", `type=local,dest=${_localCacheDir},mode=max,compression=uncompressed,force-compression=true,tag=${cacheTag}`,
                 ...argFlags,
                 "-f", dfPath, WORKSPACE,

package/dist/harness/image-cache.d.ts

@@ -0,0 +1,48 @@
+/**
+ * The image cache as the guest sees it (`CONTAINER_STORE.md`).
+ *
+ * The control plane writes one manifest per VM at start naming the two
+ * cache disks it attached: the **root** (containerd's own root,
+ * read-write, this VM's clone) and the **layers** disk (read-only, one
+ * EROFS file per layer, shared by every VM of a generation). Both paths
+ * are fixed by the control plane; the manifest is how a harness learns
+ * whether this VM carries a cache at all (the fake backend does not,
+ * and neither does a server older than the cache).
+ *
+ * Read synchronously as well as asynchronously: a component's service
+ * definition is built inside `defineEnvironment`, which is synchronous,
+ * and `k3s()` decides its mounts there.
+ */
+/** Written by `env.rs` before the harness starts. */
+export declare const IMAGE_CACHE_MANIFEST = "/run/spectest-image-cache.json";
+/** Where the cache's paths are, when this VM carries one. */
+export interface ImageCachePaths {
+    /** containerd's root: read-write, this VM's own clone. */
+    root: string;
+    /** The layers disk: read-only, shared by every VM of a generation. */
+    layers: string;
+}
+/**
+ * Directory under the root disk holding a nested runtime's containerd
+ * root, one per service: `<root>/spectest-nested/<service>`. The merge
+ * (`image_cache/merge.rs::NESTED_DIR`) reads every store it finds there
+ * exactly as it reads the disk's own.
+ */
+export declare const NESTED_STORES_DIR = "spectest-nested";
+/** The guest's static `mkfs.erofs`, which a nested runtime's EROFS
+ *  differ needs and no runtime image ships. */
+export declare const MKFS_EROFS_PATH = "/usr/local/bin/mkfs.erofs";
+/** The guest's adopt helper (`base.rs::STORE_ADOPT_SH`), POSIX sh so a
+ *  nested runtime's busybox can run the same file. */
+export declare const STORE_ADOPT_PATH = "/usr/local/bin/spectest-store-adopt";
+/** The cache's paths, or `null` when this VM carries none.
+ *  `SPECTEST_IMAGE_CACHE_MANIFEST` points a test at another file. */
+export declare function imageCachePathsSync(manifest?: string): ImageCachePaths | null;
+/** The host directory a nested runtime keeps its containerd root in. */
+export declare function nestedStoreDir(paths: ImageCachePaths, service: string): string;
+/**
+ * Is a volume's host path on a cache disk? Such a directory is a cache
+ * the lineage keeps — like the container store one level up — and the
+ * delta-restore teardown must not wipe it.
+ */
+export declare function isOnImageCache(hostPath: string, paths: ImageCachePaths | null): boolean;