@specific.dev/spectest 0.90.3 → 0.91.1

package/src/harness/component-images.test.ts

@@ -0,0 +1,44 @@
+import { expect, test } from "bun:test";
+import { canonicalImageReference, componentBuildTasks, kubernetesImages } from "./component-images";
+import { validateServiceImage, type BuildImage } from "../index";
+
+test("services and Kubernetes share all Dockerfile build options without a type discriminator", () => {
+  const image: BuildImage & { reference: string } = {
+    reference: "k8s.internal:5000/runtime:spectest", content: "FROM scratch AS test",
+    target: "test", context: ".", exclude: ["fixtures"], buildArgs: { MODE: "test" },
+  };
+  expect(validateServiceImage("app", { ...image, context: undefined })).toEqual({ ...image, context: undefined, type: "dockerfile" });
+  expect(validateServiceImage("redis", "redis:7.4")).toEqual({ type: "registry", reference: "redis:7.4" });
+  const parsed = kubernetesImages(["redis:7.4", image]);
+  expect(parsed.pulls).toEqual(["redis:7.4"]);
+  expect(parsed.builds).toEqual([{ reference: image.reference, image }]);
+  const tasks = componentBuildTasks([{ name: "cluster", image: "k3s:latest", imageBuilds: parsed.builds }]);
+  expect(tasks[0].image).toBe(image);
+  expect(tasks[0].owner).toBe("cluster");
+  expect(tasks[0].name).toMatch(/^spectest-image-[a-f0-9]+$/);
+});
+
+test("image identity validates references and rejects conflicting definitions", () => {
+  expect(canonicalImageReference("redis:7.4")).toBe("docker.io/library/redis:7.4");
+  expect(canonicalImageReference("docker.io/redis:7.4")).toBe("docker.io/library/redis:7.4");
+  expect(canonicalImageReference("company/api")).toBe("docker.io/company/api:latest");
+  expect(canonicalImageReference("localhost:5000/api")).toBe("localhost:5000/api:latest");
+  expect(kubernetesImages(["redis:7.4", "redis:7.4"]).pulls).toHaveLength(1);
+  for (const value of ["", "--help", "https://example.com/api", "has space", "bad:tag:extra"]) {
+    expect(() => kubernetesImages([value])).toThrow();
+  }
+  expect(() => kubernetesImages([{ content: "FROM scratch" } as never])).toThrow("reference");
+  expect(() => kubernetesImages([{ type: "registry", reference: "redis:7" } as never])).toThrow();
+  expect(() => kubernetesImages([{ reference: `api@sha256:${"a".repeat(64)}`, content: "FROM scratch" }])).toThrow("digest");
+  expect(() => kubernetesImages(["api:1", { reference: "docker.io/library/api:1", content: "FROM scratch" }])).toThrow("conflicting");
+});
+
+test("build task identity survives list reordering but is scoped to the owning cluster", () => {
+  const a = { reference: "app:1", image: { content: "FROM scratch" } };
+  const b = { reference: "app:2", image: { content: "FROM scratch" } };
+  const tasks = (name: string, imageBuilds: typeof a[]) => componentBuildTasks([{ name, image: "k3s:1", imageBuilds }]);
+  expect(tasks("cluster", [a, b])[0].name).toBe(tasks("cluster", [b, a])[1].name);
+  expect(tasks("cluster", [a])[0].name).not.toBe(tasks("other", [a])[0].name);
+  expect(() => validateServiceImage("app", { content: "FROM scratch", buildArgs: { "bad-key": "x" } })).toThrow("ARG");
+  expect(() => validateServiceImage("app", { content: "FROM scratch", exclude: [1] as never })).toThrow("exclude");
+});

package/src/harness/component-images.ts

@@ -0,0 +1,79 @@
+import { createHash } from "node:crypto";
+import type { BuildImage, ComponentImageBuild, ServiceConfig } from "../index.js";
+
+export type KubernetesImage = string | (BuildImage & { reference: string });
+
+/** containerd/CRI use fully qualified names, including the default tag. */
+export function canonicalImageReference(reference: string): string {
+  if (typeof reference !== "string" || !reference || reference.startsWith("-") || /[\s\x00]/.test(reference)) {
+    throw new Error("image reference must be a non-empty image name without whitespace");
+  }
+  const slash = reference.indexOf("/");
+  const first = slash < 0 ? "" : reference.slice(0, slash);
+  let result = slash < 0 ? `docker.io/library/${reference}`
+    : first.includes(".") || first.includes(":") || first === "localhost" ? reference : `docker.io/${reference}`;
+  if (result.startsWith("index.docker.io/")) result = result.replace("index.docker.io/", "docker.io/");
+  if (result.startsWith("docker.io/") && !result.slice(10).includes("/")) result = `docker.io/library/${result.slice(10)}`;
+  const [name, digest] = result.split("@");
+  const tagAt = name.lastIndexOf(":") > name.lastIndexOf("/") ? name.lastIndexOf(":") : -1;
+  const repository = tagAt < 0 ? name : name.slice(0, tagAt);
+  if (!/^[a-z0-9.-]+(?::[0-9]+)?\/[a-z0-9]+(?:(?:[._]|__|[-]+)[a-z0-9]+)*(?:\/[a-z0-9]+(?:(?:[._]|__|[-]+)[a-z0-9]+)*)*$/.test(repository)
+      || (tagAt >= 0 && !/^[\w][\w.-]{0,127}$/.test(name.slice(tagAt + 1)))
+      || (result.includes("@") && (!digest || !/^sha256:[a-f0-9]{64}$/.test(digest) || result.split("@").length !== 2))) {
+    throw new Error(`invalid image reference: ${reference}`);
+  }
+  if (tagAt < 0 && !digest) result += ":latest";
+  return result;
+}
+
+export function kubernetesImages(images: readonly KubernetesImage[]): { pulls: string[]; builds: ComponentImageBuild[] } {
+  if (!Array.isArray(images)) throw new Error("k3s(): images must be an array of registry references or Dockerfile builds");
+  const pulls: string[] = [];
+  const builds: ComponentImageBuild[] = [];
+  const seen = new Map<string, string>();
+  for (const image of images) {
+    if (typeof image !== "string" && (!image || typeof image !== "object" || "type" in image || !image.reference)) {
+      throw new Error("k3s(): images expects registry strings or Dockerfile builds with a reference");
+    }
+    const reference = canonicalImageReference(typeof image === "string" ? image : image.reference);
+    if (typeof image !== "string" && reference.includes("@")) throw new Error("k3s(): a build reference must be an image name/tag, not a digest");
+    const identity = JSON.stringify(image);
+    const previous = seen.get(reference);
+    if (previous !== undefined) {
+      if (previous !== identity) throw new Error(`k3s(): conflicting images for ${reference}`);
+      continue;
+    }
+    seen.set(reference, identity);
+    if (typeof image === "string") pulls.push(image);
+    else builds.push({ reference, image });
+  }
+  return { pulls, builds };
+}
+
+/** Stable internal task names, separate from the service graph. */
+export function componentBuildTasks<T extends ServiceConfig & { name: string }>(services: readonly T[]) {
+  return services.flatMap(svc => (svc.imageBuilds ?? []).map(build => ({
+    name: `spectest-image-${createHash("sha256").update(`${svc.name}\0${build.reference}`).digest("hex")}`,
+    owner: svc.name,
+    reference: build.reference,
+    image: build.image,
+  })));
+}
+
+/** Private handoff to a component's setup hook. Builds stay generic; the
+ * component decides how its runtime consumes them and establishes trust. */
+export const COMPONENT_RUNTIME = Symbol.for("spectest.component-runtime");
+export interface PreparedComponentImage {
+  source: string;
+  reference: string;
+  /** A component can load each result as soon as its own build finishes. */
+  ready?: Promise<void>;
+}
+export interface ComponentRuntime {
+  images: readonly PreparedComponentImage[];
+  gateway: string;
+  caBundle: string | null;
+}
+export function componentRuntime(context: object): ComponentRuntime | undefined {
+  return (context as { [COMPONENT_RUNTIME]?: ComponentRuntime })[COMPONENT_RUNTIME];
+}

package/src/harness/concurrency.ts

@@ -0,0 +1,8 @@
+/** Wait for every writer to stop before reporting a failure to setup/cleanup. */
+export async function waitForAll(work: readonly Promise<unknown>[]): Promise<void> {
+  const results = await Promise.allSettled(work);
+  const failures = results.flatMap(result => result.status === "rejected" ? [result.reason] : []);
+  if (failures.length === 1) throw failures[0];
+  if (failures.length > 1) throw new AggregateError(failures,
+    `Concurrent environment preparation failed: ${failures.map(error => error instanceof Error ? error.message : String(error)).join("; ")}`);
+}

package/src/harness/k3s-image-loader.ts

@@ -0,0 +1,131 @@
+/** A short-lived, read-only OCI endpoint over the host containerd content store.
+ * k3s fetches only missing blobs and unpacks only missing snapshots. No Docker
+ * archive, staging registry, or second persistent copy of application blobs. */
+import { execFile, spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { createServer } from "node:http";
+import { promisify } from "node:util";
+import type { SpectestContext } from "../index.js";
+import { canonicalImageReference, type PreparedComponentImage } from "./component-images.js";
+import { waitForK3sRuntime } from "./k3s-prepull.js";
+import { waitForAll } from "./concurrency.js";
+
+interface Descriptor { digest: string; size: number; mediaType: string; platform?: { os: string; architecture: string } }
+interface Manifest { schemaVersion: number; mediaType: string; config: Descriptor; layers: Descriptor[]; annotations?: Record<string, string> }
+const digest = (bytes: Uint8Array) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+const descriptor = (bytes: Uint8Array, mediaType: string): Descriptor => ({ digest: digest(bytes), size: bytes.length, mediaType });
+const run = promisify(execFile);
+const HOST_CTR = ["--address", "/run/containerd/containerd.sock", "--namespace", "moby"];
+
+async function hostContent(id: string): Promise<Buffer> {
+  if (!/^sha256:[a-f0-9]{64}$/.test(id)) throw new Error(`invalid content digest ${id}`);
+  const result = await run("ctr", [...HOST_CTR, "content", "get", id], { encoding: "buffer", maxBuffer: 16 * 1024 * 1024, timeout: 30_000 });
+  if (digest(result.stdout) !== id) throw new Error(`corrupt image metadata ${id}`);
+  return result.stdout;
+}
+
+/** Preserve the original index/manifest bytes and all application descriptors. */
+async function hostImage(source: string) {
+  const name = canonicalImageReference(source);
+  const { stdout } = await run("ctr", [...HOST_CTR, "images", "ls", `name==${name}`], { timeout: 30_000 });
+  const found = stdout.split("\n").map(line => line.trim().split(/\s+/)).find(row => row[0] === name)?.[2];
+  if (!found) throw new Error(`built image ${name} is missing from the VM container store`);
+  const metadata = new Map<string, Buffer>();
+  const allowed = new Map<string, Descriptor>();
+  let configDigest: string | undefined;
+  async function visit(id: string, selected: boolean, depth = 0): Promise<Descriptor> {
+    if (depth > 4) throw new Error(`built image ${name} has a recursive manifest index`);
+    const bytes = await hostContent(id);
+    const obj = JSON.parse(bytes.toString());
+    const desc = descriptor(bytes, obj.mediaType);
+    metadata.set(id, bytes);
+    allowed.set(id, desc);
+    if (obj.config && Array.isArray(obj.layers)) {
+      const manifest = obj as Manifest;
+      for (const blob of [manifest.config, ...manifest.layers]) allowed.set(blob.digest, blob);
+      if (selected) configDigest = manifest.config.digest;
+    } else if (Array.isArray(obj.manifests)) {
+      for (const entry of obj.manifests as Descriptor[]) {
+        const matches = !entry.platform || (entry.platform.os === "linux" && entry.platform.architecture === (process.arch === "arm64" ? "arm64" : "amd64"));
+        await visit(entry.digest, selected && matches, depth + 1);
+      }
+    } else throw new Error(`built image ${name} has invalid image metadata`);
+    return desc;
+  }
+  const target = await visit(found, true);
+  if (!configDigest) throw new Error(`built image ${name} has no image for this VM's platform`);
+  return { target, metadata, allowed, configDigest };
+}
+
+export async function loadK3sBuildImages(
+  service: string, images: readonly PreparedComponentImage[],
+  gateway: string, exec: SpectestContext["exec"],
+): Promise<void> {
+  if (!images.length) return;
+  await waitForK3sRuntime(service, exec);
+  const checked = async (args: string[], timeoutMs = 30_000) => {
+    const result = await exec(service, args, { timeoutMs });
+    if (result.exitCode !== 0) throw new Error(`k3s(${service}): image load failed: ${result.stderr || result.stdout}`);
+    return result.stdout;
+  };
+  const config = await checked(["cat", "/var/lib/rancher/k3s/agent/etc/containerd/config.toml"]);
+  const snapshotter = config.match(/^\s*snapshotter\s*=\s*['"]([^'"]+)['"]/m)?.[1];
+  if (!snapshotter) throw new Error(`k3s(${service}): cannot determine kubelet's snapshotter`);
+  const ctr = ["ctr", "--address", "/run/k3s/containerd/containerd.sock", "--namespace", "k8s.io"];
+  const help = await checked([...ctr, "images", "pull", "--help"]);
+  const local = help.includes("--local") ? ["--local"] : [];
+  await waitForAll(images.map(async image => {
+    await image.ready;
+    const reference = canonicalImageReference(image.reference);
+    const { target, metadata: blobs, allowed, configDigest } = await hostImage(image.source);
+    const streams = new Set<ReturnType<typeof spawn>>();
+    let transferred = 0;
+    const server = createServer((req, res) => {
+      const pathname = new URL(req.url ?? "/", "http://local").pathname;
+      const match = pathname.match(/^\/v2\/.+\/(manifests|blobs)\/([^/]+)$/);
+      if (!["GET", "HEAD"].includes(req.method ?? "") || !match) { res.writeHead(404).end(); return; }
+      const id = match[1] === "manifests" && !match[2].startsWith("sha256:") ? target.digest : match[2];
+      const desc = allowed.get(id);
+      if (!desc) { res.writeHead(404).end(); return; }
+      res.writeHead(200, { "Content-Type": desc.mediaType, "Content-Length": desc.size, "Docker-Content-Digest": id });
+      if (req.method === "HEAD") { res.end(); return; }
+      transferred += desc.size;
+      const bytes = blobs.get(id);
+      if (bytes) { res.end(bytes); return; }
+      const proc = spawn("ctr", [...HOST_CTR, "content", "get", id], { stdio: ["ignore", "pipe", "ignore"] });
+      streams.add(proc);
+      proc.stdout!.pipe(res);
+      proc.on("error", err => res.destroy(err));
+      proc.on("exit", code => { streams.delete(proc); if (code !== 0) res.destroy(new Error(`content read failed: ${id}`)); });
+      res.on("close", () => { if (proc.exitCode === null) proc.kill(); });
+    });
+    const directory = `/run/spectest-image-load-${createHash("sha256").update(reference).digest("hex").slice(0, 16)}`;
+    try {
+      await new Promise<void>((resolve, reject) => { server.once("error", reject); server.listen(0, gateway, resolve); });
+      const port = (server.address() as { port: number }).port;
+      // A private hosts directory scopes this override to this single ctr call;
+      // it cannot change the cluster's configured registries or pull policy.
+      const hosts = `server = "http://${gateway}:${port}"\n`;
+      await checked(["sh", "-c", 'mkdir -p "$1/_default"; printf "%s" "$2" > "$1/_default/hosts.toml"', "sh", directory, hosts]);
+      await checked([...ctr, "images", "pull", ...local, "--hosts-dir", directory,
+        "--snapshotter", snapshotter, "--label", "io.cri-containerd.image=managed", reference], 600_000);
+      const repo = reference.slice(0, reference.lastIndexOf(":"));
+      await checked([...ctr, "images", "tag", "--force", reference, `${repo}@${target.digest}`]);
+      // CRI learns about ctr image updates asynchronously. Confirm the current
+      // build's config, not merely that an older tag already exists.
+      const deadline = Date.now() + 30_000;
+      for (;;) {
+        const result = await exec(service, ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock", "inspecti", reference], { timeoutMs: 5_000 });
+        if (result.exitCode === 0 && JSON.parse(result.stdout).status?.id === configDigest) break;
+        if (Date.now() >= deadline) throw new Error(`k3s(${service}): current build is not visible to kubelet: ${reference}`);
+        await new Promise(resolve => setTimeout(resolve, 100));
+      }
+      console.log(`[k3s-images] ${service}: ${reference} @ ${target.digest}, transferred ${transferred} bytes`);
+    } finally {
+      for (const proc of streams) proc.kill();
+      server.closeAllConnections();
+      await new Promise<void>(resolve => server.close(() => resolve()));
+      await exec(service, ["rm", "-rf", directory], { timeoutMs: 5_000 }).catch(() => undefined);
+    }
+  }));
+}

package/src/harness/k3s-prepull.test.ts

@@ -0,0 +1,57 @@
+import { expect, test } from "bun:test";
+import type { SpectestContext } from "../index.js";
+import { prePullK3sImages } from "./k3s-prepull";
+
+const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
+const digest = `registry.test/app@sha256:${"a".repeat(64)}`;
+const mockExec = (fn: (args: string[]) => Promise<{ exitCode: number; stdout?: string; stderr?: string }>) =>
+  (async (_name: string, args: string[]) => ({ stdout: "", stderr: "", ...await fn(args) })) as SpectestContext["exec"];
+
+test("pre-pulls reuse immutable cached images, but resolve tags on every fresh setup", async () => {
+  const calls: string[][] = [];
+  await prePullK3sImages("cluster", [digest, "registry.test/app:latest"], mockExec(async args => {
+    calls.push(args); return { exitCode: 0 };
+  }));
+  expect(calls).toEqual([[...cri, "info"], [...cri, "inspecti", digest], [...cri, "pull", "registry.test/app:latest"]]);
+  calls.length = 0;
+  await prePullK3sImages("cluster", [digest], mockExec(async args => {
+    calls.push(args); return { exitCode: args[3] === "inspecti" ? 1 : 0 };
+  }));
+  expect(calls.at(-1)).toEqual([...cri, "pull", digest]);
+});
+
+test("failed pre-pulls fail setup after all concurrent pulls finish", async () => {
+  let active = 0, peak = 0, finished = 0;
+  const work = prePullK3sImages("cluster", ["a:1", "b:1", "c:1"], mockExec(async args => {
+    if (args.at(-1) === "info") return { exitCode: 0 };
+    peak = Math.max(peak, ++active);
+    await Bun.sleep(args.at(-1) === "a:1" ? 1 : 20);
+    --active; ++finished;
+    return { exitCode: args.at(-1) === "a:1" ? 1 : 0, stderr: "registry unavailable" };
+  }));
+  await expect(work).rejects.toThrow("pre-pull failed for a:1: registry unavailable");
+  expect(active).toBe(0);
+  expect(peak).toBe(3);
+  expect(finished).toBe(3);
+});
+
+test("pre-pulls wait for the runtime even when the API server is already ready", async () => {
+  const calls: string[] = [];
+  let ready = false;
+  await prePullK3sImages("cluster", ["app:1"], mockExec(async args => {
+    calls.push(args[3]!);
+    if (args[3] === "info" && !ready) { ready = true; return { exitCode: 1, stderr: "connection refused" }; }
+    return { exitCode: 0 };
+  }));
+  expect(calls).toEqual(["info", "info", "pull"]);
+});
+
+test("simultaneous pull failures retain every failing image in the setup error", async () => {
+  const error = await prePullK3sImages("cluster", ["a:1", "b:1"], mockExec(async args => ({
+    exitCode: args[3] === "pull" ? 1 : 0, stderr: "unavailable",
+  }))).catch(error => error);
+  expect(error).toBeInstanceOf(AggregateError);
+  expect(error.errors).toHaveLength(2);
+  expect(error.message).toContain("pre-pull failed for a:1");
+  expect(error.message).toContain("pre-pull failed for b:1");
+});

package/src/harness/k3s-prepull.ts

@@ -0,0 +1,33 @@
+import type { SpectestContext } from "../index.js";
+import { waitForAll } from "./concurrency.js";
+
+/** CRI PullImage waits for unpacking and uses k3s's registry configuration.
+ * Always resolve tags on a fresh setup. Immutable digests can be used offline.
+ * Submit every image concurrently; containerd manages downloads and unpacking.
+ * Drain all pulls on failure so none outlive the setup hook.
+ */
+export async function prePullK3sImages(name: string, images: readonly string[], exec: SpectestContext["exec"]): Promise<void> {
+  if (!images.length) return;
+  await waitForK3sRuntime(name, exec);
+  const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
+  await waitForAll(images.map(async image => {
+    if (/@sha256:[a-f0-9]{64}$/.test(image)) {
+      const existing = await exec(name, [...cri, "inspecti", image], { timeoutMs: 30_000 });
+      if (existing.exitCode === 0) return;
+    }
+    const result = await exec(name, [...cri, "pull", image], { timeoutMs: 600_000 });
+    if (result.exitCode !== 0) throw new Error(`k3s(${name}): pre-pull failed for ${image}: ${result.stderr || result.stdout}`);
+  }));
+}
+
+export async function waitForK3sRuntime(name: string, exec: SpectestContext["exec"]): Promise<void> {
+  const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
+  // /readyz belongs to the API server and can precede the node runtime.
+  const deadline = Date.now() + 60_000;
+  for (;;) {
+    const result = await exec(name, [...cri, "info"], { timeoutMs: 5_000 });
+    if (result.exitCode === 0) break;
+    if (Date.now() >= deadline) throw new Error(`k3s(${name}): containerd did not become ready for pre-pulls: ${result.stderr || result.stdout}`);
+    await new Promise(resolve => setTimeout(resolve, 250));
+  }
+}

package/src/harness/sticky-consumer.test.ts

@@ -43,3 +43,31 @@ test("healthy cache keeps recent images and a cache that cannot shrink is not pu
   await expect(collectStickyCache(config, run, async () => ({ totalBytes: 256 * GiB, freeBytes: 5 * GiB })))
     .rejects.toThrow("cannot meet filesystem retention target");
 });
+
+test("nested Kubernetes stores share the filesystem budget and are pruned before recent Docker images", async () => {
+  let used = 100 * GiB;
+  const calls: string[][] = [];
+  let evictions = 0;
+  const run = async (command: string, args: string[]) => { calls.push([command, ...args]); return ""; };
+  await collectStickyCache(config, run, async () => ({ totalBytes: 256 * GiB, freeBytes: 256 * GiB - used }), {
+    protectedImages: new Set(), close: async () => {},
+    pruneOldest: async () => { used -= 40 * GiB; evictions++; return true; },
+  });
+  expect(evictions).toBe(1);
+  expect(calls.some(c => c[0] === "docker" && c[1] === "image" && c[2] === "ls")).toBe(false);
+});
+
+test("GC does not remove an image while it supplies the Kubernetes maintenance daemon", async () => {
+  const removed: string[] = [];
+  const run = async (command: string, args: string[]) => {
+    if (command === "docker" && args[1] === "ls") return "runtime\nold";
+    if (command === "docker" && args[1] === "inspect") return JSON.stringify([
+      { Id: "runtime", Created: "2020" }, { Id: "old", Created: "2021" }]);
+    if (command === "docker" && args[1] === "rm") removed.push(args.at(-1)!);
+    return "";
+  };
+  await collectStickyCache(config, run, async () => ({ totalBytes: 256 * GiB, freeBytes: (removed.length ? 220 : 150) * GiB }), {
+    protectedImages: new Set(["runtime"]), close: async () => {}, pruneOldest: async () => false,
+  });
+  expect(removed).toEqual(["old"]);
+});

package/src/harness/sticky-consumer.ts

@@ -1,10 +1,13 @@
-/** Docker/BuildKit policy for the generic sticky disk. No host publication API. */
+/** Docker, BuildKit and nested Kubernetes policy for the generic sticky disk. */
 import { execFile } from "node:child_process";
 import { mkdir, readFile, statfs, writeFile } from "node:fs/promises";
 
+import { STICKY_RUNTIME } from "./sticky-paths.js";
+import { openKubernetesCacheMaintenance, type KubernetesCacheMaintenance } from "./sticky-kubernetes.js";
+
 export const STICKY_CAPABILITY = "sticky-disks-v1";
 export const STICKY_COMPATIBILITY = "docker29-ctr21-buildkit032-overlayfs-256g-v2";
-export const STICKY_RUNTIME = "/run/spectest-sticky";
+export { STICKY_RUNTIME } from "./sticky-paths.js";
 export interface StickyConsumerConfig {
   path: string;
   capacityBytes: number;
@@ -115,6 +118,7 @@ export async function collectStickyCache(
     const fs = await statfs(config.path);
     return { freeBytes: fs.bavail * fs.bsize, totalBytes: fs.blocks * fs.bsize };
   },
+  kubernetes?: KubernetesCacheMaintenance,
 ): Promise<void> {
   validateStickyConfig(config);
   const buildctl = ["--addr", "tcp://127.0.0.1:1234", "prune"];
@@ -133,6 +137,10 @@ export async function collectStickyCache(
   };
   await pruneBuildkit();
   if (withinTarget(fs)) return;
+  while (await kubernetes?.pruneOldest()) {
+    fs = await usage();
+    if (withinTarget(fs)) return;
+  }
   // Pressure can be entirely in recent Docker images. Drop oldest images
   // first, even within the age window, until the whole filesystem fits.
   const ids = [...new Set((await run("docker", ["image", "ls", "-aq", "--no-trunc"])).split(/\s+/).filter(Boolean))];
@@ -140,6 +148,7 @@ export async function collectStickyCache(
     ? JSON.parse(await run("docker", ["image", "inspect", ...ids])) : [];
   images.sort((a, b) => a.Created.localeCompare(b.Created) || a.Id.localeCompare(b.Id));
   for (const image of images) {
+    if (kubernetes?.protectedImages.has(image.Id)) continue;
     await run("docker", ["image", "rm", "-f", image.Id], 120_000);
     await pruneBuildkit();
     if (withinTarget(fs)) return;
@@ -158,7 +167,9 @@ export async function prepareStickyConsumer(artifact = false): Promise<{ prepare
     if (containers.length) await exec("docker", ["rm", "-f", ...containers]);
     const volumes = (await exec("docker", ["volume", "ls", "-q"])).split(/\s+/).filter(Boolean);
     if (volumes.length) await exec("docker", ["volume", "rm", ...volumes]);
-    await collectStickyCache(config);
+    const kubernetes = await openKubernetesCacheMaintenance(exec);
+    try { await collectStickyCache(config, exec, undefined, kubernetes); }
+    finally { await kubernetes.close(); }
   }
   const units = ["spectest-sticky-buildkit", "spectest-sticky-docker", "spectest-sticky-containerd"];
   for (const unit of units) {

package/src/harness/sticky-kubernetes.test.ts

@@ -0,0 +1,88 @@
+import { afterEach, expect, test } from "bun:test";
+import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { openKubernetesCacheMaintenance, stickyKubernetesDirectory, type CacheCommand } from "./sticky-kubernetes";
+import { isOnStickyDisk, stickyDiskPathSync, STICKY_ROOT } from "./sticky-paths";
+
+const roots: string[] = [];
+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+async function temporary() { const root = await mkdtemp(join(tmpdir(), "sticky-k8s-test-")); roots.push(root); return root; }
+const image = `sha256:${"a".repeat(64)}`;
+async function store(root: string, service: string, age = 0) {
+  const dir = `${root}/${service}/0123456789abcdef`;
+  await mkdir(dir, { recursive: true });
+  await writeFile(`${dir}/store.json`, JSON.stringify({ schema: 1, image, updatedAt: Date.now() - age }));
+  return dir;
+}
+function runner(calls: string[][]): CacheCommand {
+  return async (program, args) => {
+    calls.push([program, ...args]);
+    if (program === "docker" && args[0] === "inspect") return JSON.stringify({ ExitCode: 0, OOMKilled: false });
+    return "";
+  };
+}
+
+test("sticky volume preservation requires an active manifest and a path inside the mount", async () => {
+  const manifest = `${await temporary()}/consumer.json`;
+  expect(stickyDiskPathSync(manifest)).toBeNull();
+  await writeFile(manifest, JSON.stringify({ path: STICKY_ROOT }));
+  expect(stickyDiskPathSync(manifest)).toBe(STICKY_ROOT);
+  expect(isOnStickyDisk(`${STICKY_ROOT}/kubernetes/cluster`, STICKY_ROOT)).toBe(true);
+  for (const path of [`${STICKY_ROOT}-other/store`, `${STICKY_ROOT}/../other`, "/workspace/data"]) {
+    expect(isOnStickyDisk(path, STICKY_ROOT)).toBe(false);
+  }
+  expect(isOnStickyDisk(`${STICKY_ROOT}/kubernetes/cluster`, null)).toBe(false);
+  await writeFile(manifest, JSON.stringify({ path: "/" }));
+  expect(stickyDiskPathSync(manifest)).toBeNull();
+});
+
+test("Kubernetes stores are isolated by service and k3s version", () => {
+  const original = stickyKubernetesDirectory("cluster", "v1.33.5-k3s1");
+  expect(stickyKubernetesDirectory("cluster", "v1.32.1-k3s1")).not.toBe(original);
+  expect(stickyKubernetesDirectory("second", "v1.33.5-k3s1")).not.toBe(original);
+});
+
+test("maintenance prunes oldest stores and releases runtime-image protection only after the last writer stops", async () => {
+  const root = await temporary();
+  const old = await store(root, "old", 1000);
+  const recent = await store(root, "recent");
+  const calls: string[][] = [];
+  const cache = await openKubernetesCacheMaintenance(runner(calls), root);
+  expect(cache.protectedImages.has(image)).toBe(true);
+  expect(await cache.pruneOldest()).toBe(true);
+  expect(calls).toContainEqual(["rm", "-rf", old]);
+  expect(cache.protectedImages.has(image)).toBe(true);
+  expect(await cache.pruneOldest()).toBe(true);
+  expect(calls).toContainEqual(["rm", "-rf", recent]);
+  expect(cache.protectedImages.size).toBe(0);
+  expect(await cache.pruneOldest()).toBe(false);
+  await cache.close();
+  expect(calls.filter(c => c[0] === "docker" && c[1] === "stop")).toHaveLength(2);
+});
+
+test("failed maintenance startup closes every started store and refuses publication", async () => {
+  const root = await temporary();
+  await store(root, "first"); await store(root, "second");
+  const calls: string[][] = [];
+  const run = runner(calls);
+  let checks = 0;
+  await expect(openKubernetesCacheMaintenance(async (program, args, timeout) => {
+    const result = await run(program, args, timeout);
+    if (args[0] === "exec" && ++checks === 2) throw new Error("containerd cannot open store");
+    return result;
+  }, root)).rejects.toThrow("Kubernetes cache preparation failed");
+  expect(calls.filter(c => c[0] === "docker" && c[1] === "stop")).toHaveLength(2);
+});
+
+test("unclean nested containerd shutdown rejects sealing instead of publishing its store", async () => {
+  const root = await temporary(); await store(root, "cluster");
+  const calls: string[][] = [];
+  const run = runner(calls);
+  const cache = await openKubernetesCacheMaintenance(async (program, args, timeout) => {
+    const result = await run(program, args, timeout);
+    return args[0] === "inspect" ? JSON.stringify({ ExitCode: 137, OOMKilled: false }) : result;
+  }, root);
+  await expect(cache.close()).rejects.toThrow("Kubernetes cache shutdown failed");
+  expect(calls.some(c => c[0] === "rm")).toBe(false);
+});