@specific.dev/spectest 0.90.2 → 0.91.0

package/src/harness/k3s-image-loader.ts

@@ -0,0 +1,129 @@
+/** A short-lived, read-only OCI endpoint over the host containerd content store.
+ * k3s fetches only missing blobs and unpacks only missing snapshots. No Docker
+ * archive, staging registry, or second persistent copy of application blobs. */
+import { execFile, spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { createServer } from "node:http";
+import { promisify } from "node:util";
+import type { SpectestContext } from "../index.js";
+import { canonicalImageReference } from "./component-images.js";
+import { waitForK3sRuntime } from "./k3s-prepull.js";
+
+interface Descriptor { digest: string; size: number; mediaType: string; platform?: { os: string; architecture: string } }
+interface Manifest { schemaVersion: number; mediaType: string; config: Descriptor; layers: Descriptor[]; annotations?: Record<string, string> }
+const digest = (bytes: Uint8Array) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+const descriptor = (bytes: Uint8Array, mediaType: string): Descriptor => ({ digest: digest(bytes), size: bytes.length, mediaType });
+const run = promisify(execFile);
+const HOST_CTR = ["--address", "/run/containerd/containerd.sock", "--namespace", "moby"];
+
+async function hostContent(id: string): Promise<Buffer> {
+  if (!/^sha256:[a-f0-9]{64}$/.test(id)) throw new Error(`invalid content digest ${id}`);
+  const result = await run("ctr", [...HOST_CTR, "content", "get", id], { encoding: "buffer", maxBuffer: 16 * 1024 * 1024, timeout: 30_000 });
+  if (digest(result.stdout) !== id) throw new Error(`corrupt image metadata ${id}`);
+  return result.stdout;
+}
+
+/** Preserve the original index/manifest bytes and all application descriptors. */
+async function hostImage(source: string) {
+  const name = canonicalImageReference(source);
+  const { stdout } = await run("ctr", [...HOST_CTR, "images", "ls", `name==${name}`], { timeout: 30_000 });
+  const found = stdout.split("\n").map(line => line.trim().split(/\s+/)).find(row => row[0] === name)?.[2];
+  if (!found) throw new Error(`built image ${name} is missing from the VM container store`);
+  const metadata = new Map<string, Buffer>();
+  const allowed = new Map<string, Descriptor>();
+  let configDigest: string | undefined;
+  async function visit(id: string, selected: boolean, depth = 0): Promise<Descriptor> {
+    if (depth > 4) throw new Error(`built image ${name} has a recursive manifest index`);
+    const bytes = await hostContent(id);
+    const obj = JSON.parse(bytes.toString());
+    const desc = descriptor(bytes, obj.mediaType);
+    metadata.set(id, bytes);
+    allowed.set(id, desc);
+    if (obj.config && Array.isArray(obj.layers)) {
+      const manifest = obj as Manifest;
+      for (const blob of [manifest.config, ...manifest.layers]) allowed.set(blob.digest, blob);
+      if (selected) configDigest = manifest.config.digest;
+    } else if (Array.isArray(obj.manifests)) {
+      for (const entry of obj.manifests as Descriptor[]) {
+        const matches = !entry.platform || (entry.platform.os === "linux" && entry.platform.architecture === (process.arch === "arm64" ? "arm64" : "amd64"));
+        await visit(entry.digest, selected && matches, depth + 1);
+      }
+    } else throw new Error(`built image ${name} has invalid image metadata`);
+    return desc;
+  }
+  const target = await visit(found, true);
+  if (!configDigest) throw new Error(`built image ${name} has no image for this VM's platform`);
+  return { target, metadata, allowed, configDigest };
+}
+
+export async function loadK3sBuildImages(
+  service: string, images: readonly { source: string; reference: string }[],
+  gateway: string, exec: SpectestContext["exec"],
+): Promise<void> {
+  if (!images.length) return;
+  await waitForK3sRuntime(service, exec);
+  const checked = async (args: string[], timeoutMs = 30_000) => {
+    const result = await exec(service, args, { timeoutMs });
+    if (result.exitCode !== 0) throw new Error(`k3s(${service}): image load failed: ${result.stderr || result.stdout}`);
+    return result.stdout;
+  };
+  const config = await checked(["cat", "/var/lib/rancher/k3s/agent/etc/containerd/config.toml"]);
+  const snapshotter = config.match(/^\s*snapshotter\s*=\s*['"]([^'"]+)['"]/m)?.[1];
+  if (!snapshotter) throw new Error(`k3s(${service}): cannot determine kubelet's snapshotter`);
+  const ctr = ["ctr", "--address", "/run/k3s/containerd/containerd.sock", "--namespace", "k8s.io"];
+  const help = await checked([...ctr, "images", "pull", "--help"]);
+  const local = help.includes("--local") ? ["--local"] : [];
+  for (const image of images) {
+    const reference = canonicalImageReference(image.reference);
+    const { target, metadata: blobs, allowed, configDigest } = await hostImage(image.source);
+    const streams = new Set<ReturnType<typeof spawn>>();
+    let transferred = 0;
+    const server = createServer((req, res) => {
+      const pathname = new URL(req.url ?? "/", "http://local").pathname;
+      const match = pathname.match(/^\/v2\/.+\/(manifests|blobs)\/([^/]+)$/);
+      if (!["GET", "HEAD"].includes(req.method ?? "") || !match) { res.writeHead(404).end(); return; }
+      const id = match[1] === "manifests" && !match[2].startsWith("sha256:") ? target.digest : match[2];
+      const desc = allowed.get(id);
+      if (!desc) { res.writeHead(404).end(); return; }
+      res.writeHead(200, { "Content-Type": desc.mediaType, "Content-Length": desc.size, "Docker-Content-Digest": id });
+      if (req.method === "HEAD") { res.end(); return; }
+      transferred += desc.size;
+      const bytes = blobs.get(id);
+      if (bytes) { res.end(bytes); return; }
+      const proc = spawn("ctr", [...HOST_CTR, "content", "get", id], { stdio: ["ignore", "pipe", "ignore"] });
+      streams.add(proc);
+      proc.stdout!.pipe(res);
+      proc.on("error", err => res.destroy(err));
+      proc.on("exit", code => { streams.delete(proc); if (code !== 0) res.destroy(new Error(`content read failed: ${id}`)); });
+      res.on("close", () => { if (proc.exitCode === null) proc.kill(); });
+    });
+    const directory = `/run/spectest-image-load-${createHash("sha256").update(reference).digest("hex").slice(0, 16)}`;
+    try {
+      await new Promise<void>((resolve, reject) => { server.once("error", reject); server.listen(0, gateway, resolve); });
+      const port = (server.address() as { port: number }).port;
+      // A private hosts directory scopes this override to this single ctr call;
+      // it cannot change the cluster's configured registries or pull policy.
+      const hosts = `server = "http://${gateway}:${port}"\n`;
+      await checked(["sh", "-c", 'mkdir -p "$1/_default"; printf "%s" "$2" > "$1/_default/hosts.toml"', "sh", directory, hosts]);
+      await checked([...ctr, "images", "pull", ...local, "--hosts-dir", directory,
+        "--snapshotter", snapshotter, "--label", "io.cri-containerd.image=managed", reference], 600_000);
+      const repo = reference.slice(0, reference.lastIndexOf(":"));
+      await checked([...ctr, "images", "tag", "--force", reference, `${repo}@${target.digest}`]);
+      // CRI learns about ctr image updates asynchronously. Confirm the current
+      // build's config, not merely that an older tag already exists.
+      const deadline = Date.now() + 30_000;
+      for (;;) {
+        const result = await exec(service, ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock", "inspecti", reference], { timeoutMs: 5_000 });
+        if (result.exitCode === 0 && JSON.parse(result.stdout).status?.id === configDigest) break;
+        if (Date.now() >= deadline) throw new Error(`k3s(${service}): current build is not visible to kubelet: ${reference}`);
+        await new Promise(resolve => setTimeout(resolve, 100));
+      }
+      console.log(`[k3s-images] ${service}: ${reference} @ ${target.digest}, transferred ${transferred} bytes`);
+    } finally {
+      for (const proc of streams) proc.kill();
+      server.closeAllConnections();
+      await new Promise<void>(resolve => server.close(() => resolve()));
+      await exec(service, ["rm", "-rf", directory], { timeoutMs: 5_000 }).catch(() => undefined);
+    }
+  }
+}

package/src/harness/k3s-prepull.test.ts

@@ -0,0 +1,47 @@
+import { expect, test } from "bun:test";
+import type { SpectestContext } from "../index.js";
+import { prePullK3sImages } from "./k3s-prepull";
+
+const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
+const digest = `registry.test/app@sha256:${"a".repeat(64)}`;
+const mockExec = (fn: (args: string[]) => Promise<{ exitCode: number; stdout?: string; stderr?: string }>) =>
+  (async (_name: string, args: string[]) => ({ stdout: "", stderr: "", ...await fn(args) })) as SpectestContext["exec"];
+
+test("pre-pulls reuse immutable cached images, but resolve tags on every fresh setup", async () => {
+  const calls: string[][] = [];
+  await prePullK3sImages("cluster", [digest, "registry.test/app:latest"], mockExec(async args => {
+    calls.push(args); return { exitCode: 0 };
+  }));
+  expect(calls).toEqual([[...cri, "info"], [...cri, "inspecti", digest], [...cri, "pull", "registry.test/app:latest"]]);
+  calls.length = 0;
+  await prePullK3sImages("cluster", [digest], mockExec(async args => {
+    calls.push(args); return { exitCode: args[3] === "inspecti" ? 1 : 0 };
+  }));
+  expect(calls.at(-1)).toEqual([...cri, "pull", digest]);
+});
+
+test("failed pre-pulls fail setup after outstanding pulls finish, with bounded concurrency", async () => {
+  let active = 0, peak = 0, finished = 0;
+  const work = prePullK3sImages("cluster", ["a:1", "b:1", "c:1"], mockExec(async args => {
+    if (args.at(-1) === "info") return { exitCode: 0 };
+    peak = Math.max(peak, ++active);
+    await Bun.sleep(args.at(-1) === "a:1" ? 1 : 20);
+    --active; ++finished;
+    return { exitCode: args.at(-1) === "a:1" ? 1 : 0, stderr: "registry unavailable" };
+  }));
+  await expect(work).rejects.toThrow("pre-pull failed for a:1: registry unavailable");
+  expect(active).toBe(0);
+  expect(peak).toBe(2);
+  expect(finished).toBe(3);
+});
+
+test("pre-pulls wait for the runtime even when the API server is already ready", async () => {
+  const calls: string[] = [];
+  let ready = false;
+  await prePullK3sImages("cluster", ["app:1"], mockExec(async args => {
+    calls.push(args[3]!);
+    if (args[3] === "info" && !ready) { ready = true; return { exitCode: 1, stderr: "connection refused" }; }
+    return { exitCode: 0 };
+  }));
+  expect(calls).toEqual(["info", "info", "pull"]);
+});

package/src/harness/k3s-prepull.ts

@@ -0,0 +1,36 @@
+import type { SpectestContext } from "../index.js";
+
+/** CRI PullImage waits for unpacking and uses k3s's registry configuration.
+ * Always resolve tags on a fresh setup. Immutable digests can be used offline.
+ * Drain both workers on failure so no pulls outlive the setup hook.
+ */
+export async function prePullK3sImages(name: string, images: readonly string[], exec: SpectestContext["exec"]): Promise<void> {
+  if (!images.length) return;
+  await waitForK3sRuntime(name, exec);
+  const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
+  const queue = [...images];
+  const results = await Promise.allSettled(Array.from({ length: Math.min(2, queue.length) }, async () => {
+    for (let image = queue.shift(); image !== undefined; image = queue.shift()) {
+      if (/@sha256:[a-f0-9]{64}$/.test(image)) {
+        const existing = await exec(name, [...cri, "inspecti", image], { timeoutMs: 30_000 });
+        if (existing.exitCode === 0) continue;
+      }
+      const result = await exec(name, [...cri, "pull", image], { timeoutMs: 600_000 });
+      if (result.exitCode !== 0) throw new Error(`k3s(${name}): pre-pull failed for ${image}: ${result.stderr || result.stdout}`);
+    }
+  }));
+  const failed = results.find(result => result.status === "rejected");
+  if (failed?.status === "rejected") throw failed.reason;
+}
+
+export async function waitForK3sRuntime(name: string, exec: SpectestContext["exec"]): Promise<void> {
+  const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
+  // /readyz belongs to the API server and can precede the node runtime.
+  const deadline = Date.now() + 60_000;
+  for (;;) {
+    const result = await exec(name, [...cri, "info"], { timeoutMs: 5_000 });
+    if (result.exitCode === 0) break;
+    if (Date.now() >= deadline) throw new Error(`k3s(${name}): containerd did not become ready for pre-pulls: ${result.stderr || result.stdout}`);
+    await new Promise(resolve => setTimeout(resolve, 250));
+  }
+}

package/src/harness/sticky-consumer.test.ts

@@ -43,3 +43,31 @@ test("healthy cache keeps recent images and a cache that cannot shrink is not pu
   await expect(collectStickyCache(config, run, async () => ({ totalBytes: 256 * GiB, freeBytes: 5 * GiB })))
     .rejects.toThrow("cannot meet filesystem retention target");
 });
+
+test("nested Kubernetes stores share the filesystem budget and are pruned before recent Docker images", async () => {
+  let used = 100 * GiB;
+  const calls: string[][] = [];
+  let evictions = 0;
+  const run = async (command: string, args: string[]) => { calls.push([command, ...args]); return ""; };
+  await collectStickyCache(config, run, async () => ({ totalBytes: 256 * GiB, freeBytes: 256 * GiB - used }), {
+    protectedImages: new Set(), close: async () => {},
+    pruneOldest: async () => { used -= 40 * GiB; evictions++; return true; },
+  });
+  expect(evictions).toBe(1);
+  expect(calls.some(c => c[0] === "docker" && c[1] === "image" && c[2] === "ls")).toBe(false);
+});
+
+test("GC does not remove an image while it supplies the Kubernetes maintenance daemon", async () => {
+  const removed: string[] = [];
+  const run = async (command: string, args: string[]) => {
+    if (command === "docker" && args[1] === "ls") return "runtime\nold";
+    if (command === "docker" && args[1] === "inspect") return JSON.stringify([
+      { Id: "runtime", Created: "2020" }, { Id: "old", Created: "2021" }]);
+    if (command === "docker" && args[1] === "rm") removed.push(args.at(-1)!);
+    return "";
+  };
+  await collectStickyCache(config, run, async () => ({ totalBytes: 256 * GiB, freeBytes: (removed.length ? 220 : 150) * GiB }), {
+    protectedImages: new Set(["runtime"]), close: async () => {}, pruneOldest: async () => false,
+  });
+  expect(removed).toEqual(["old"]);
+});

package/src/harness/sticky-consumer.ts

@@ -1,10 +1,13 @@
-/** Docker/BuildKit policy for the generic sticky disk. No host publication API. */
+/** Docker, BuildKit and nested Kubernetes policy for the generic sticky disk. */
 import { execFile } from "node:child_process";
 import { mkdir, readFile, statfs, writeFile } from "node:fs/promises";
 
+import { STICKY_RUNTIME } from "./sticky-paths.js";
+import { openKubernetesCacheMaintenance, type KubernetesCacheMaintenance } from "./sticky-kubernetes.js";
+
 export const STICKY_CAPABILITY = "sticky-disks-v1";
 export const STICKY_COMPATIBILITY = "docker29-ctr21-buildkit032-overlayfs-256g-v2";
-export const STICKY_RUNTIME = "/run/spectest-sticky";
+export { STICKY_RUNTIME } from "./sticky-paths.js";
 export interface StickyConsumerConfig {
   path: string;
   capacityBytes: number;
@@ -115,6 +118,7 @@ export async function collectStickyCache(
     const fs = await statfs(config.path);
     return { freeBytes: fs.bavail * fs.bsize, totalBytes: fs.blocks * fs.bsize };
   },
+  kubernetes?: KubernetesCacheMaintenance,
 ): Promise<void> {
   validateStickyConfig(config);
   const buildctl = ["--addr", "tcp://127.0.0.1:1234", "prune"];
@@ -133,6 +137,10 @@ export async function collectStickyCache(
   };
   await pruneBuildkit();
   if (withinTarget(fs)) return;
+  while (await kubernetes?.pruneOldest()) {
+    fs = await usage();
+    if (withinTarget(fs)) return;
+  }
   // Pressure can be entirely in recent Docker images. Drop oldest images
   // first, even within the age window, until the whole filesystem fits.
   const ids = [...new Set((await run("docker", ["image", "ls", "-aq", "--no-trunc"])).split(/\s+/).filter(Boolean))];
@@ -140,6 +148,7 @@ export async function collectStickyCache(
     ? JSON.parse(await run("docker", ["image", "inspect", ...ids])) : [];
   images.sort((a, b) => a.Created.localeCompare(b.Created) || a.Id.localeCompare(b.Id));
   for (const image of images) {
+    if (kubernetes?.protectedImages.has(image.Id)) continue;
     await run("docker", ["image", "rm", "-f", image.Id], 120_000);
     await pruneBuildkit();
     if (withinTarget(fs)) return;
@@ -158,7 +167,9 @@ export async function prepareStickyConsumer(artifact = false): Promise<{ prepare
     if (containers.length) await exec("docker", ["rm", "-f", ...containers]);
     const volumes = (await exec("docker", ["volume", "ls", "-q"])).split(/\s+/).filter(Boolean);
     if (volumes.length) await exec("docker", ["volume", "rm", ...volumes]);
-    await collectStickyCache(config);
+    const kubernetes = await openKubernetesCacheMaintenance(exec);
+    try { await collectStickyCache(config, exec, undefined, kubernetes); }
+    finally { await kubernetes.close(); }
   }
   const units = ["spectest-sticky-buildkit", "spectest-sticky-docker", "spectest-sticky-containerd"];
   for (const unit of units) {

package/src/harness/sticky-kubernetes.test.ts

@@ -0,0 +1,88 @@
+import { afterEach, expect, test } from "bun:test";
+import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { openKubernetesCacheMaintenance, stickyKubernetesDirectory, type CacheCommand } from "./sticky-kubernetes";
+import { isOnStickyDisk, stickyDiskPathSync, STICKY_ROOT } from "./sticky-paths";
+
+const roots: string[] = [];
+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+async function temporary() { const root = await mkdtemp(join(tmpdir(), "sticky-k8s-test-")); roots.push(root); return root; }
+const image = `sha256:${"a".repeat(64)}`;
+async function store(root: string, service: string, age = 0) {
+  const dir = `${root}/${service}/0123456789abcdef`;
+  await mkdir(dir, { recursive: true });
+  await writeFile(`${dir}/store.json`, JSON.stringify({ schema: 1, image, updatedAt: Date.now() - age }));
+  return dir;
+}
+function runner(calls: string[][]): CacheCommand {
+  return async (program, args) => {
+    calls.push([program, ...args]);
+    if (program === "docker" && args[0] === "inspect") return JSON.stringify({ ExitCode: 0, OOMKilled: false });
+    return "";
+  };
+}
+
+test("sticky volume preservation requires an active manifest and a path inside the mount", async () => {
+  const manifest = `${await temporary()}/consumer.json`;
+  expect(stickyDiskPathSync(manifest)).toBeNull();
+  await writeFile(manifest, JSON.stringify({ path: STICKY_ROOT }));
+  expect(stickyDiskPathSync(manifest)).toBe(STICKY_ROOT);
+  expect(isOnStickyDisk(`${STICKY_ROOT}/kubernetes/cluster`, STICKY_ROOT)).toBe(true);
+  for (const path of [`${STICKY_ROOT}-other/store`, `${STICKY_ROOT}/../other`, "/workspace/data"]) {
+    expect(isOnStickyDisk(path, STICKY_ROOT)).toBe(false);
+  }
+  expect(isOnStickyDisk(`${STICKY_ROOT}/kubernetes/cluster`, null)).toBe(false);
+  await writeFile(manifest, JSON.stringify({ path: "/" }));
+  expect(stickyDiskPathSync(manifest)).toBeNull();
+});
+
+test("Kubernetes stores are isolated by service and k3s version", () => {
+  const original = stickyKubernetesDirectory("cluster", "v1.33.5-k3s1");
+  expect(stickyKubernetesDirectory("cluster", "v1.32.1-k3s1")).not.toBe(original);
+  expect(stickyKubernetesDirectory("second", "v1.33.5-k3s1")).not.toBe(original);
+});
+
+test("maintenance prunes oldest stores and releases runtime-image protection only after the last writer stops", async () => {
+  const root = await temporary();
+  const old = await store(root, "old", 1000);
+  const recent = await store(root, "recent");
+  const calls: string[][] = [];
+  const cache = await openKubernetesCacheMaintenance(runner(calls), root);
+  expect(cache.protectedImages.has(image)).toBe(true);
+  expect(await cache.pruneOldest()).toBe(true);
+  expect(calls).toContainEqual(["rm", "-rf", old]);
+  expect(cache.protectedImages.has(image)).toBe(true);
+  expect(await cache.pruneOldest()).toBe(true);
+  expect(calls).toContainEqual(["rm", "-rf", recent]);
+  expect(cache.protectedImages.size).toBe(0);
+  expect(await cache.pruneOldest()).toBe(false);
+  await cache.close();
+  expect(calls.filter(c => c[0] === "docker" && c[1] === "stop")).toHaveLength(2);
+});
+
+test("failed maintenance startup closes every started store and refuses publication", async () => {
+  const root = await temporary();
+  await store(root, "first"); await store(root, "second");
+  const calls: string[][] = [];
+  const run = runner(calls);
+  let checks = 0;
+  await expect(openKubernetesCacheMaintenance(async (program, args, timeout) => {
+    const result = await run(program, args, timeout);
+    if (args[0] === "exec" && ++checks === 2) throw new Error("containerd cannot open store");
+    return result;
+  }, root)).rejects.toThrow("Kubernetes cache preparation failed");
+  expect(calls.filter(c => c[0] === "docker" && c[1] === "stop")).toHaveLength(2);
+});
+
+test("unclean nested containerd shutdown rejects sealing instead of publishing its store", async () => {
+  const root = await temporary(); await store(root, "cluster");
+  const calls: string[][] = [];
+  const run = runner(calls);
+  const cache = await openKubernetesCacheMaintenance(async (program, args, timeout) => {
+    const result = await run(program, args, timeout);
+    return args[0] === "inspect" ? JSON.stringify({ ExitCode: 137, OOMKilled: false }) : result;
+  }, root);
+  await expect(cache.close()).rejects.toThrow("Kubernetes cache shutdown failed");
+  expect(calls.some(c => c[0] === "rm")).toBe(false);
+});

package/src/harness/sticky-kubernetes.ts

@@ -0,0 +1,165 @@
+/** Kubernetes owns a separate containerd store; Docker/BuildKit GC cannot
+ * collect it. Maintenance runs only on the isolated publication clone. */
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, readFile, readdir, rename, writeFile } from "node:fs/promises";
+import { STICKY_ROOT } from "./sticky-paths.js";
+
+export type CacheCommand = (program: string, args: string[], timeout?: number) => Promise<string>;
+const STORE_TARGET = "/var/lib/rancher/k3s/agent/containerd";
+const SOCKET = "/run/spectest-cache.sock";
+const ROOT = `${STICKY_ROOT}/kubernetes`;
+const IMAGE_ID = /^sha256:[a-f0-9]{64}$/;
+const SERVICE_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/;
+interface StoreDescriptor { schema: 1; image: string; updatedAt: number }
+
+export function stickyKubernetesDirectory(service: string, version: string): string {
+  // Version changes must not open old metadata with a different containerd.
+  const layout = createHash("sha256").update(`overlayfs-v1:${version}`).digest("hex").slice(0, 16);
+  return `${ROOT}/${service}/${layout}`;
+}
+
+const runCommand: CacheCommand = async (program, args, timeout = 30_000) =>
+  (await promisify(execFile)(program, args, { timeout })).stdout.trim();
+
+export async function registerStickyKubernetesStore(
+  service: string, version: string, run: CacheCommand = runCommand,
+): Promise<void> {
+  if (!SERVICE_NAME.test(service)) throw new Error(`invalid Kubernetes cache service name: ${service}`);
+  // Respect custom Dockerfile overrides of the component's image.
+  const image = (await run("docker", ["inspect", "--format", "{{.Image}}", service])).trim();
+  if (!IMAGE_ID.test(image)) throw new Error(`cannot identify Kubernetes runtime image for ${service}`);
+  const directory = stickyKubernetesDirectory(service, version);
+  await mkdir(directory, { recursive: true });
+  const temporary = `${directory}/store.${randomUUID()}.tmp`;
+  await writeFile(temporary, JSON.stringify({ schema: 1, image, updatedAt: Date.now() } satisfies StoreDescriptor));
+  await rename(temporary, `${directory}/store.json`);
+}
+
+/** Standalone containerd never starts CRI/kubelet or any inherited workload.
+ * Native API deletion preserves content and committed image snapshots while
+ * releasing sandbox, container, lease and writable snapshot references. */
+export const KUBERNETES_CACHE_MAINTENANCE_SH = `set -eu
+exec 2>&1
+cat > /run/cache.toml <<'CONFIG'
+version = 2
+disabled_plugins = ["io.containerd.grpc.v1.cri", "io.containerd.cri.v1.runtime", "io.containerd.cri.v1.images"]
+CONFIG
+containerd --config /run/cache.toml --root ${STORE_TARGET} --state /run/cache-state --address ${SOCKET} > /run/cache.log 2>&1 &
+pid=$!
+stop() { kill "$pid"; wait "$pid"; }
+trap 'stop; exit 0' TERM INT
+trap 'kill "$pid" 2>/dev/null || true' EXIT
+c() { ctr --address ${SOCKET} --namespace k8s.io "$@"; }
+i=0
+until c version >/dev/null 2>&1; do
+  kill -0 "$pid" || { cat /run/cache.log; exit 1; }
+  i=$((i+1)); [ "$i" -lt 300 ] || { cat /run/cache.log; exit 1; }
+  sleep 0.1
+done
+ids=$(c containers ls -q)
+if [ -n "$ids" ]; then c containers rm $ids; fi
+# 1.7 has no sandbox store; 2.x does.
+case "$(containerd --version)" in
+  *v1.*) ;;
+  *) sandboxes=$(c sandboxes ls)
+     ids=$(printf '%s\\n' "$sandboxes" | awk 'NR > 1 {print $1}')
+     if [ -n "$ids" ]; then c sandboxes rm --force $ids; fi ;;
+esac
+ids=$(c leases ls -q)
+if [ -n "$ids" ]; then c leases rm $ids; fi
+# Remove children before parents, and tolerate asynchronous GC winning a race.
+previous=''
+while :; do
+  snapshots=$(c snapshots --snapshotter overlayfs ls)
+  ids=$(printf '%s\\n' "$snapshots" | awk 'NR > 1 && $NF != "Committed" {print $1}')
+  [ -n "$ids" ] || break
+  [ "$ids" != "$previous" ] || { echo 'cannot clean Kubernetes writable snapshots' >&2; exit 1; }
+  previous="$ids"
+  for id in $ids; do c snapshots --snapshotter overlayfs rm "$id" >/dev/null 2>&1 || true; done
+done
+touch /run/cache-ready
+while :; do sleep 1 & wait $!; done
+`;
+
+interface Store { directory: string; descriptor: StoreDescriptor; container: string; active: boolean }
+export interface KubernetesCacheMaintenance {
+  /** Outer images needed by live maintenance containers must not be evicted. */
+  protectedImages: Set<string>;
+  /** Evicts one complete image store, oldest service/version first. */
+  pruneOldest(): Promise<boolean>;
+  close(): Promise<void>;
+}
+
+export async function openKubernetesCacheMaintenance(run: CacheCommand, root = ROOT): Promise<KubernetesCacheMaintenance> {
+  const stores: Store[] = [];
+  const protectedImages = new Set<string>();
+  const closeStore = async (store: Store) => {
+    if (!store.active) return;
+    await run("docker", ["stop", "--time", "30", store.container], 45_000);
+    const status = JSON.parse(await run("docker", ["inspect", "--format", "{{json .State}}", store.container]));
+    if (status.ExitCode !== 0 || status.OOMKilled) throw new Error(`unclean Kubernetes cache shutdown: ${store.container}`);
+    await run("docker", ["rm", store.container]);
+    store.active = false;
+    if (!stores.some(other => other.active && other.descriptor.image === store.descriptor.image)) protectedImages.delete(store.descriptor.image);
+    // These contain runtime state, not image content or unpacked layers.
+    await run("rm", ["-rf", ...["io.containerd.runtime.v2.task", "io.containerd.grpc.v1.cri", "io.containerd.cri.v1.runtime", "tmpmounts"]
+      .map(child => `${store.directory}/containerd/${child}`)]);
+  };
+  const close = async () => {
+    const failures: unknown[] = [];
+    for (const store of stores) {
+      try { await closeStore(store); } catch (error) { failures.push(error); }
+    }
+    if (failures.length) throw new AggregateError(failures, "Kubernetes cache shutdown failed");
+  };
+  const entries = await readdir(root, { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => {
+    if (error.code === "ENOENT") return [];
+    throw error;
+  });
+  try {
+    for (const service of entries) {
+      if (!service.isDirectory() || !SERVICE_NAME.test(service.name)) continue;
+      for (const layout of await readdir(`${root}/${service.name}`, { withFileTypes: true })) {
+        if (!layout.isDirectory() || !/^[a-f0-9]{16}$/.test(layout.name)) continue;
+        const directory = `${root}/${service.name}/${layout.name}`;
+        const descriptor: StoreDescriptor = JSON.parse(await readFile(`${directory}/store.json`, "utf8"));
+        if (descriptor.schema !== 1 || !IMAGE_ID.test(descriptor.image) || !Number.isSafeInteger(descriptor.updatedAt)) {
+          throw new Error(`invalid Kubernetes cache descriptor: ${directory}`);
+        }
+        const container = `spectest-k8s-cache-${randomUUID()}`;
+        await run("docker", ["run", "-d", "--name", container, "--network", "none", "--privileged", "--stop-signal", "SIGTERM", "--tmpfs", "/run", "--tmpfs", "/var/run",
+          "--mount", `type=bind,source=${directory}/containerd,target=${STORE_TARGET}`,
+          "--entrypoint", "/bin/sh", descriptor.image, "-c", KUBERNETES_CACHE_MAINTENANCE_SH]);
+        stores.push({ directory, descriptor, container, active: true });
+        protectedImages.add(descriptor.image);
+        try {
+          await run("docker", ["exec", container, "sh", "-c", "i=0; until [ -f /run/cache-ready ]; do i=$((i+1)); [ $i -lt 900 ] || exit 1; sleep 0.1; done"], 100_000);
+        } catch (error) {
+          const logs = await run("docker", ["logs", "--tail", "40", container]).catch(() => "");
+          throw new Error(`Kubernetes cache preparation failed for ${directory}: ${logs}`, { cause: error });
+        }
+      }
+    }
+    stores.sort((a, b) => a.descriptor.updatedAt - b.descriptor.updatedAt || a.directory.localeCompare(b.directory));
+    let next = 0;
+    const pruneOldest = async () => {
+      const store = stores[next++];
+      if (!store) return false;
+      await run("docker", ["exec", store.container, "sh", "-ec",
+        `ids=$(ctr -a ${SOCKET} -n k8s.io images ls -q); if [ -n "$ids" ]; then ctr -a ${SOCKET} -n k8s.io images rm --sync $ids; fi`], 120_000);
+      await closeStore(store);
+      // No writer remains. Discard the now-empty cache and its descriptor so
+      // obsolete k3s binaries can also be collected by the outer Docker GC.
+      await run("rm", ["-rf", store.directory]);
+      return true;
+    };
+    // Abandoned services/versions should not occupy the disk indefinitely.
+    while (stores[next] && stores[next]!.descriptor.updatedAt < Date.now() - 7 * 24 * 60 * 60 * 1000) await pruneOldest();
+    return { protectedImages, pruneOldest, close };
+  } catch (error) {
+    await close().catch(() => {});
+    throw error;
+  }
+}

package/src/harness/sticky-paths.ts

@@ -0,0 +1,18 @@
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+
+export const STICKY_ROOT = "/var/lib/spectest-sticky";
+export const STICKY_RUNTIME = "/run/spectest-sticky";
+
+/** Written only after the supervisor mounts this project's private disk. */
+export function stickyDiskPathSync(manifest = `${STICKY_RUNTIME}/consumer.json`): string | null {
+  try {
+    const config = JSON.parse(readFileSync(manifest, "utf8"));
+    return config.path === STICKY_ROOT ? STICKY_ROOT : null;
+  } catch { return null; }
+}
+
+export function isOnStickyDisk(path: string, root = stickyDiskPathSync()): boolean {
+  const absolute = resolve(path);
+  return root !== null && (absolute === root || absolute.startsWith(`${root}/`));
+}