@specific.dev/spectest 0.90.3 → 0.91.0

package/src/daemon.ts

@@ -1,3 +1,4 @@
+import { componentBuildTasks, COMPONENT_RUNTIME } from "./harness/component-images.js";
 import { androidScope, androidRecordingChunk, releaseAndroidRecording, isAndroidApp, type AndroidRecording } from "./android.js";
 // Long-running HTTP daemon. Runs as a systemd unit on the VM host (not in
 // a container). Owns:
@@ -35,6 +36,7 @@ import {
   dnsName as makeDnsDecl,
   isWildcard,
   validateServiceImage,
+  normalizeServiceImage,
   proxy as makeProxyDecl,
 } from "./index.js";
 import type { DnsTarget, LoweredIngress } from "./index.js";
@@ -154,6 +156,7 @@ import {
 } from "./harness/ingress-table.js";
 import { startTlsTerminator, type TlsTerminator } from "./harness/tls-terminator.js";
 import { runContainerArgs } from "./harness/container-run.js";
+import { isOnStickyDisk } from "./harness/sticky-paths.js";
 import {
   BUILDKIT_CACHE_DIR,
   BUILDKIT_CACHE_UNION,
@@ -239,7 +242,7 @@ import type {
   ServiceConfig,
   ServiceDefinition,
   ServiceHandles,
-  ServiceImage,
+  NormalizedServiceImage,
   SpectestContext,
   Terminal,
   TerminalOpts,
@@ -256,10 +259,21 @@ import type { InternalTerminal, TerminalFrameSink } from "./terminal.js";
  * name (container name, image tag, ready-probe host). Widened to
  * accept any helpers shape so a typed `setup({ helpers })` call from
  * a component (e.g. K3sHelpers) lines up at the call site. */
-type NamedService = ServiceDefinition<Record<string, unknown>> & { name: string };
+type NamedService = Omit<ServiceDefinition<Record<string, unknown>>, "image"> & { name: string; image: NormalizedServiceImage };
 
 function namedServices(cfg: EnvironmentConfig): NamedService[] {
-  return Object.entries(cfg.services).map(([name, def]) => ({ name, ...def }));
+  return Object.entries(cfg.services).map(([name, def]) => ({ name, ...def, image: normalizeServiceImage(def.image) }));
+}
+
+/** Build tasks participate in image preparation, never container startup. */
+function imageTasks(services: NamedService[]): NamedService[] {
+  const tasks = componentBuildTasks(services).map(task => ({ ...task, image: validateServiceImage(task.name, task.image) }));
+  const names = new Set(services.map(s => s.name));
+  for (const task of tasks) {
+    if (names.has(task.name)) throw new Error(`image task name conflicts with service: ${task.name}`);
+    names.add(task.name);
+  }
+  return [...services, ...tasks];
 }
 
 const DEFAULT_TEST_TIMEOUT_MS = 60_000;
@@ -977,7 +991,7 @@ async function ensureVolumes(svc: NamedService): Promise<string[]> {
       // A directory on a cache disk is the same kind of thing: a nested
       // runtime's containerd root (`k3s()`), kept as a cache by the
       // lineage exactly as the container store one level up is.
-      const durable = host.startsWith("/var/cache/spectest/") || isOnImageCache(host, imageCachePathsSync(IMAGE_CACHE_MANIFEST));
+      const durable = host.startsWith("/var/cache/spectest/") || isOnStickyDisk(host) || isOnImageCache(host, imageCachePathsSync(IMAGE_CACHE_MANIFEST));
       if (vol.source?.startsWith("/") && !durable) {
         await recordAbsoluteVolumeDir(host);
       }
@@ -1365,6 +1379,18 @@ async function resolveDockerfileBuild(svc: NamedService): Promise<DockerfileBuil
 async function prepareServiceImage(
   svc: NamedService,
   opts?: { dedup?: boolean },
+): Promise<{ tag: string; buildSteps?: BuildStep[] }> {
+  const result = await prepareServiceImageInner(svc, opts);
+  if (svc.image.type === "dockerfile" && svc.image.reference) {
+    const tagged = await docker(["tag", result.tag, svc.image.reference], 30_000);
+    if (tagged.code !== 0) throw new Error(`cannot name built image ${svc.image.reference}: ${tagged.stderr}`);
+  }
+  return result;
+}
+
+async function prepareServiceImageInner(
+  svc: NamedService,
+  opts?: { dedup?: boolean },
 ): Promise<{ tag: string; buildSteps?: BuildStep[] }> {
   const tag = imageTag(svc.name);
   if (svc.image.type === "registry") {
@@ -1414,7 +1440,7 @@ async function prepareServiceImage(
         progressService(svc.name, { status: "failed" });
         throw err;
       }
-      // first.tag is already CA-layered; tagging it covers this service too.
+      // Reuse the same application image; runtime trust is added separately.
       const tagr = await docker(["tag", first.tag, tag], 30_000);
       if (tagr.code !== 0) {
         throw new Error(`docker tag ${first.tag} ${tag} failed: ${tagr.stderr.trim()}`);
@@ -3175,7 +3201,8 @@ function specToNamedService(spec: RuntimeServiceSpec): NamedService {
   // run here: the image is checked (a `path` Dockerfile and a `context`
   // must exist), and coverage adapters get their configure step.
   const svc = { ...(rest as ServiceConfig), image: validateServiceImage(name, rest.image) };
-  return { name, ...applyCoverageAdapters(name, svc) };
+  const configured = applyCoverageAdapters(name, svc);
+  return { name, ...configured, image: normalizeServiceImage(configured.image) };
 }
 
 /** Implementation behind `ctx.startService` / a fake's `ctx.startService`.
@@ -3507,12 +3534,14 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
   const bootStart = Date.now();
   const cfg: EnvironmentConfig = requireLoaded().project.environment;
   const services = namedServices(cfg);
+  const tasks = imageTasks(services);
+  const componentTasks = componentBuildTasks(services);
   const timings = new Map<string, ServiceTiming>();
   if (prepared && (prepared.version !== 1 || !prepared.images ||
-      Object.keys(prepared.images).sort().join("\n") !== services.map(s => s.name).sort().join("\n"))) {
+      Object.keys(prepared.images).sort().join("\n") !== tasks.map(s => s.name).sort().join("\n"))) {
     throw new Error("prepared image manifest does not match the environment's services");
   }
-  progressInit(services);
+  progressInit(tasks);
   // Build dedup is only valid within one workspace generation — a fresh
   // bootstrap may follow a workspace re-upload with the same dockerfile
   // text but different build-context content.
@@ -3535,7 +3564,7 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
       if ((await readIgnoreFile(path.join(WORKSPACE, ".dockerignore"))) !== null) return;
       await fs.writeFile(
         path.join(WORKSPACE, ".dockerignore"),
-        unionDockerignore(services),
+        unionDockerignore(tasks),
       );
     })(),
   ]);
@@ -3572,11 +3601,14 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
     if (found.code !== 0 || found.stdout.trim() !== entry.image) {
       throw new Error(`prepared image missing for ${svc.name}: ${entry.image}`);
     }
-    return { tag: entry.image, buildSteps: entry.buildSteps };
+    const tag = imageTag(svc.name);
+    const tagged = await docker(["tag", entry.image, tag], 30_000);
+    if (tagged.code !== 0) throw new Error(`cannot restore prepared image name for ${svc.name}: ${tagged.stderr}`);
+    return { tag, buildSteps: entry.buildSteps };
   };
   const prep = new Map<string, Promise<void>>();
   const prepOffsets = new Map<string, number>();
-  for (const svc of services) {
+  for (const svc of tasks) {
     const p = (async () => {
       const t0 = Date.now();
       prepOffsets.set(svc.name, Date.now() - bootStart);
@@ -3602,7 +3634,7 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
     // No application services, setup hooks, ingress, or runtime state in the builder.
     await Promise.all(prep.values());
     const images: PreparedImages["images"] = Object.create(null);
-    for (const svc of services) {
+    for (const svc of tasks) {
       const out = await docker(["image", "inspect", "--format", "{{.Id}}", tags.get(svc.name)!], 30_000);
       const image = out.stdout.trim();
       if (out.code !== 0 || !/^sha256:[a-f0-9]{64}$/.test(image)) {
@@ -3654,6 +3686,8 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
     progressService(svc.name, { status: "probing", detail: "ready check" });
     await waitForReady(svc);
     const tSetup = Date.now();
+    const builds = componentTasks.filter(task => task.owner === svc.name);
+    await Promise.all(builds.map(task => prep.get(task.name)!));
     if (svc.setup) {
       progressService(svc.name, { status: "probing", detail: "running setup" });
       const helpers = await ensureHelpers(svc.name, svc);
@@ -3662,11 +3696,17 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
           name: svc.name,
           helpers,
           ...(await spectestContext({ service: svc.name, includeSelf: true })),
+          ...{ [COMPONENT_RUNTIME]: {
+            images: builds.map(task => ({ source: tags.get(task.name)!, reference: task.reference })),
+            gateway: await bridgeGatewayIp(),
+            caBundle: await ensureCaBundle(),
+          } },
         });
       } catch (err) {
         throw await withContainerPostMortem(svc.name, err);
       }
     }
+    for (const task of builds) progressService(task.name, { status: "ready", detail: `loaded ${task.reference}` });
     progressService(svc.name, { status: "ready", detail: undefined });
     const ti = timings.get(svc.name);
     if (ti) {
@@ -4223,9 +4263,10 @@ async function computeProjectReach(all: NamedService[]): Promise<ReachRules> {
 async function computeProjectReachInner(all: NamedService[]): Promise<ReachRules> {
   const files = await walkProjectFiles(WORKSPACE, [".git"]);
   const sources: ReachSource[] = [];
-  for (const svc of all) {
+  const owners = new Map(componentBuildTasks(all).map(t => [t.name, t.owner]));
+  for (const svc of imageTasks(all)) {
     const src: ReachSource = {
-      service: svc.name,
+      service: owners.get(svc.name) ?? svc.name,
       volumeSources: (svc.volumes ?? []).flatMap((v) => (v.source && !v.reachExempt ? [v.source] : [])),
       ...(svc.inputs && svc.inputs.length > 0 ? { inputs: [...svc.inputs] } : {}),
     };
@@ -6720,7 +6761,7 @@ export function harnessMethods(state: RouteState): MethodTable {
     bootstrap: async (params) =>
       exclusive(state, "inFlightBootstrap", "bootstrap already in progress", async () => {
         const prepared = params.prepared === undefined ? undefined : preparedImages(
-          params.prepared, params.identity, namedServices(requireLoaded().project.environment).map(s => s.name),
+          params.prepared, params.identity, imageTasks(namedServices(requireLoaded().project.environment)).map(s => s.name),
         );
         return { ok: true, timings: await bootstrap(prepared) };
       }),

package/src/harness/component-images.test.ts

@@ -0,0 +1,44 @@
+import { expect, test } from "bun:test";
+import { canonicalImageReference, componentBuildTasks, kubernetesImages } from "./component-images";
+import { validateServiceImage, type BuildImage } from "../index";
+
+test("services and Kubernetes share all Dockerfile build options without a type discriminator", () => {
+  const image: BuildImage & { reference: string } = {
+    reference: "k8s.internal:5000/runtime:spectest", content: "FROM scratch AS test",
+    target: "test", context: ".", exclude: ["fixtures"], buildArgs: { MODE: "test" },
+  };
+  expect(validateServiceImage("app", { ...image, context: undefined })).toEqual({ ...image, context: undefined, type: "dockerfile" });
+  expect(validateServiceImage("redis", "redis:7.4")).toEqual({ type: "registry", reference: "redis:7.4" });
+  const parsed = kubernetesImages(["redis:7.4", image]);
+  expect(parsed.pulls).toEqual(["redis:7.4"]);
+  expect(parsed.builds).toEqual([{ reference: image.reference, image }]);
+  const tasks = componentBuildTasks([{ name: "cluster", image: "k3s:latest", imageBuilds: parsed.builds }]);
+  expect(tasks[0].image).toBe(image);
+  expect(tasks[0].owner).toBe("cluster");
+  expect(tasks[0].name).toMatch(/^spectest-image-[a-f0-9]+$/);
+});
+
+test("image identity validates references and rejects conflicting definitions", () => {
+  expect(canonicalImageReference("redis:7.4")).toBe("docker.io/library/redis:7.4");
+  expect(canonicalImageReference("docker.io/redis:7.4")).toBe("docker.io/library/redis:7.4");
+  expect(canonicalImageReference("company/api")).toBe("docker.io/company/api:latest");
+  expect(canonicalImageReference("localhost:5000/api")).toBe("localhost:5000/api:latest");
+  expect(kubernetesImages(["redis:7.4", "redis:7.4"]).pulls).toHaveLength(1);
+  for (const value of ["", "--help", "https://example.com/api", "has space", "bad:tag:extra"]) {
+    expect(() => kubernetesImages([value])).toThrow();
+  }
+  expect(() => kubernetesImages([{ content: "FROM scratch" } as never])).toThrow("reference");
+  expect(() => kubernetesImages([{ type: "registry", reference: "redis:7" } as never])).toThrow();
+  expect(() => kubernetesImages([{ reference: `api@sha256:${"a".repeat(64)}`, content: "FROM scratch" }])).toThrow("digest");
+  expect(() => kubernetesImages(["api:1", { reference: "docker.io/library/api:1", content: "FROM scratch" }])).toThrow("conflicting");
+});
+
+test("build task identity survives list reordering but is scoped to the owning cluster", () => {
+  const a = { reference: "app:1", image: { content: "FROM scratch" } };
+  const b = { reference: "app:2", image: { content: "FROM scratch" } };
+  const tasks = (name: string, imageBuilds: typeof a[]) => componentBuildTasks([{ name, image: "k3s:1", imageBuilds }]);
+  expect(tasks("cluster", [a, b])[0].name).toBe(tasks("cluster", [b, a])[1].name);
+  expect(tasks("cluster", [a])[0].name).not.toBe(tasks("other", [a])[0].name);
+  expect(() => validateServiceImage("app", { content: "FROM scratch", buildArgs: { "bad-key": "x" } })).toThrow("ARG");
+  expect(() => validateServiceImage("app", { content: "FROM scratch", exclude: [1] as never })).toThrow("exclude");
+});

package/src/harness/component-images.ts

@@ -0,0 +1,73 @@
+import { createHash } from "node:crypto";
+import type { BuildImage, ComponentImageBuild, ServiceConfig } from "../index.js";
+
+export type KubernetesImage = string | (BuildImage & { reference: string });
+
+/** containerd/CRI use fully qualified names, including the default tag. */
+export function canonicalImageReference(reference: string): string {
+  if (typeof reference !== "string" || !reference || reference.startsWith("-") || /[\s\x00]/.test(reference)) {
+    throw new Error("image reference must be a non-empty image name without whitespace");
+  }
+  const slash = reference.indexOf("/");
+  const first = slash < 0 ? "" : reference.slice(0, slash);
+  let result = slash < 0 ? `docker.io/library/${reference}`
+    : first.includes(".") || first.includes(":") || first === "localhost" ? reference : `docker.io/${reference}`;
+  if (result.startsWith("index.docker.io/")) result = result.replace("index.docker.io/", "docker.io/");
+  if (result.startsWith("docker.io/") && !result.slice(10).includes("/")) result = `docker.io/library/${result.slice(10)}`;
+  const [name, digest] = result.split("@");
+  const tagAt = name.lastIndexOf(":") > name.lastIndexOf("/") ? name.lastIndexOf(":") : -1;
+  const repository = tagAt < 0 ? name : name.slice(0, tagAt);
+  if (!/^[a-z0-9.-]+(?::[0-9]+)?\/[a-z0-9]+(?:(?:[._]|__|[-]+)[a-z0-9]+)*(?:\/[a-z0-9]+(?:(?:[._]|__|[-]+)[a-z0-9]+)*)*$/.test(repository)
+      || (tagAt >= 0 && !/^[\w][\w.-]{0,127}$/.test(name.slice(tagAt + 1)))
+      || (result.includes("@") && (!digest || !/^sha256:[a-f0-9]{64}$/.test(digest) || result.split("@").length !== 2))) {
+    throw new Error(`invalid image reference: ${reference}`);
+  }
+  if (tagAt < 0 && !digest) result += ":latest";
+  return result;
+}
+
+export function kubernetesImages(images: readonly KubernetesImage[]): { pulls: string[]; builds: ComponentImageBuild[] } {
+  if (!Array.isArray(images)) throw new Error("k3s(): images must be an array of registry references or Dockerfile builds");
+  const pulls: string[] = [];
+  const builds: ComponentImageBuild[] = [];
+  const seen = new Map<string, string>();
+  for (const image of images) {
+    if (typeof image !== "string" && (!image || typeof image !== "object" || "type" in image || !image.reference)) {
+      throw new Error("k3s(): images expects registry strings or Dockerfile builds with a reference");
+    }
+    const reference = canonicalImageReference(typeof image === "string" ? image : image.reference);
+    if (typeof image !== "string" && reference.includes("@")) throw new Error("k3s(): a build reference must be an image name/tag, not a digest");
+    const identity = JSON.stringify(image);
+    const previous = seen.get(reference);
+    if (previous !== undefined) {
+      if (previous !== identity) throw new Error(`k3s(): conflicting images for ${reference}`);
+      continue;
+    }
+    seen.set(reference, identity);
+    if (typeof image === "string") pulls.push(image);
+    else builds.push({ reference, image });
+  }
+  return { pulls, builds };
+}
+
+/** Stable internal task names, separate from the service graph. */
+export function componentBuildTasks<T extends ServiceConfig & { name: string }>(services: readonly T[]) {
+  return services.flatMap(svc => (svc.imageBuilds ?? []).map(build => ({
+    name: `spectest-image-${createHash("sha256").update(`${svc.name}\0${build.reference}`).digest("hex")}`,
+    owner: svc.name,
+    reference: build.reference,
+    image: build.image,
+  })));
+}
+
+/** Private handoff to a component's setup hook. Builds stay generic; the
+ * component decides how its runtime consumes them and establishes trust. */
+export const COMPONENT_RUNTIME = Symbol.for("spectest.component-runtime");
+export interface ComponentRuntime {
+  images: readonly { source: string; reference: string }[];
+  gateway: string;
+  caBundle: string | null;
+}
+export function componentRuntime(context: object): ComponentRuntime | undefined {
+  return (context as { [COMPONENT_RUNTIME]?: ComponentRuntime })[COMPONENT_RUNTIME];
+}

package/src/harness/k3s-image-loader.ts

@@ -0,0 +1,129 @@
+/** A short-lived, read-only OCI endpoint over the host containerd content store.
+ * k3s fetches only missing blobs and unpacks only missing snapshots. No Docker
+ * archive, staging registry, or second persistent copy of application blobs. */
+import { execFile, spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { createServer } from "node:http";
+import { promisify } from "node:util";
+import type { SpectestContext } from "../index.js";
+import { canonicalImageReference } from "./component-images.js";
+import { waitForK3sRuntime } from "./k3s-prepull.js";
+
+interface Descriptor { digest: string; size: number; mediaType: string; platform?: { os: string; architecture: string } }
+interface Manifest { schemaVersion: number; mediaType: string; config: Descriptor; layers: Descriptor[]; annotations?: Record<string, string> }
+const digest = (bytes: Uint8Array) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+const descriptor = (bytes: Uint8Array, mediaType: string): Descriptor => ({ digest: digest(bytes), size: bytes.length, mediaType });
+const run = promisify(execFile);
+const HOST_CTR = ["--address", "/run/containerd/containerd.sock", "--namespace", "moby"];
+
+async function hostContent(id: string): Promise<Buffer> {
+  if (!/^sha256:[a-f0-9]{64}$/.test(id)) throw new Error(`invalid content digest ${id}`);
+  const result = await run("ctr", [...HOST_CTR, "content", "get", id], { encoding: "buffer", maxBuffer: 16 * 1024 * 1024, timeout: 30_000 });
+  if (digest(result.stdout) !== id) throw new Error(`corrupt image metadata ${id}`);
+  return result.stdout;
+}
+
+/** Preserve the original index/manifest bytes and all application descriptors. */
+async function hostImage(source: string) {
+  const name = canonicalImageReference(source);
+  const { stdout } = await run("ctr", [...HOST_CTR, "images", "ls", `name==${name}`], { timeout: 30_000 });
+  const found = stdout.split("\n").map(line => line.trim().split(/\s+/)).find(row => row[0] === name)?.[2];
+  if (!found) throw new Error(`built image ${name} is missing from the VM container store`);
+  const metadata = new Map<string, Buffer>();
+  const allowed = new Map<string, Descriptor>();
+  let configDigest: string | undefined;
+  async function visit(id: string, selected: boolean, depth = 0): Promise<Descriptor> {
+    if (depth > 4) throw new Error(`built image ${name} has a recursive manifest index`);
+    const bytes = await hostContent(id);
+    const obj = JSON.parse(bytes.toString());
+    const desc = descriptor(bytes, obj.mediaType);
+    metadata.set(id, bytes);
+    allowed.set(id, desc);
+    if (obj.config && Array.isArray(obj.layers)) {
+      const manifest = obj as Manifest;
+      for (const blob of [manifest.config, ...manifest.layers]) allowed.set(blob.digest, blob);
+      if (selected) configDigest = manifest.config.digest;
+    } else if (Array.isArray(obj.manifests)) {
+      for (const entry of obj.manifests as Descriptor[]) {
+        const matches = !entry.platform || (entry.platform.os === "linux" && entry.platform.architecture === (process.arch === "arm64" ? "arm64" : "amd64"));
+        await visit(entry.digest, selected && matches, depth + 1);
+      }
+    } else throw new Error(`built image ${name} has invalid image metadata`);
+    return desc;
+  }
+  const target = await visit(found, true);
+  if (!configDigest) throw new Error(`built image ${name} has no image for this VM's platform`);
+  return { target, metadata, allowed, configDigest };
+}
+
+export async function loadK3sBuildImages(
+  service: string, images: readonly { source: string; reference: string }[],
+  gateway: string, exec: SpectestContext["exec"],
+): Promise<void> {
+  if (!images.length) return;
+  await waitForK3sRuntime(service, exec);
+  const checked = async (args: string[], timeoutMs = 30_000) => {
+    const result = await exec(service, args, { timeoutMs });
+    if (result.exitCode !== 0) throw new Error(`k3s(${service}): image load failed: ${result.stderr || result.stdout}`);
+    return result.stdout;
+  };
+  const config = await checked(["cat", "/var/lib/rancher/k3s/agent/etc/containerd/config.toml"]);
+  const snapshotter = config.match(/^\s*snapshotter\s*=\s*['"]([^'"]+)['"]/m)?.[1];
+  if (!snapshotter) throw new Error(`k3s(${service}): cannot determine kubelet's snapshotter`);
+  const ctr = ["ctr", "--address", "/run/k3s/containerd/containerd.sock", "--namespace", "k8s.io"];
+  const help = await checked([...ctr, "images", "pull", "--help"]);
+  const local = help.includes("--local") ? ["--local"] : [];
+  for (const image of images) {
+    const reference = canonicalImageReference(image.reference);
+    const { target, metadata: blobs, allowed, configDigest } = await hostImage(image.source);
+    const streams = new Set<ReturnType<typeof spawn>>();
+    let transferred = 0;
+    const server = createServer((req, res) => {
+      const pathname = new URL(req.url ?? "/", "http://local").pathname;
+      const match = pathname.match(/^\/v2\/.+\/(manifests|blobs)\/([^/]+)$/);
+      if (!["GET", "HEAD"].includes(req.method ?? "") || !match) { res.writeHead(404).end(); return; }
+      const id = match[1] === "manifests" && !match[2].startsWith("sha256:") ? target.digest : match[2];
+      const desc = allowed.get(id);
+      if (!desc) { res.writeHead(404).end(); return; }
+      res.writeHead(200, { "Content-Type": desc.mediaType, "Content-Length": desc.size, "Docker-Content-Digest": id });
+      if (req.method === "HEAD") { res.end(); return; }
+      transferred += desc.size;
+      const bytes = blobs.get(id);
+      if (bytes) { res.end(bytes); return; }
+      const proc = spawn("ctr", [...HOST_CTR, "content", "get", id], { stdio: ["ignore", "pipe", "ignore"] });
+      streams.add(proc);
+      proc.stdout!.pipe(res);
+      proc.on("error", err => res.destroy(err));
+      proc.on("exit", code => { streams.delete(proc); if (code !== 0) res.destroy(new Error(`content read failed: ${id}`)); });
+      res.on("close", () => { if (proc.exitCode === null) proc.kill(); });
+    });
+    const directory = `/run/spectest-image-load-${createHash("sha256").update(reference).digest("hex").slice(0, 16)}`;
+    try {
+      await new Promise<void>((resolve, reject) => { server.once("error", reject); server.listen(0, gateway, resolve); });
+      const port = (server.address() as { port: number }).port;
+      // A private hosts directory scopes this override to this single ctr call;
+      // it cannot change the cluster's configured registries or pull policy.
+      const hosts = `server = "http://${gateway}:${port}"\n`;
+      await checked(["sh", "-c", 'mkdir -p "$1/_default"; printf "%s" "$2" > "$1/_default/hosts.toml"', "sh", directory, hosts]);
+      await checked([...ctr, "images", "pull", ...local, "--hosts-dir", directory,
+        "--snapshotter", snapshotter, "--label", "io.cri-containerd.image=managed", reference], 600_000);
+      const repo = reference.slice(0, reference.lastIndexOf(":"));
+      await checked([...ctr, "images", "tag", "--force", reference, `${repo}@${target.digest}`]);
+      // CRI learns about ctr image updates asynchronously. Confirm the current
+      // build's config, not merely that an older tag already exists.
+      const deadline = Date.now() + 30_000;
+      for (;;) {
+        const result = await exec(service, ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock", "inspecti", reference], { timeoutMs: 5_000 });
+        if (result.exitCode === 0 && JSON.parse(result.stdout).status?.id === configDigest) break;
+        if (Date.now() >= deadline) throw new Error(`k3s(${service}): current build is not visible to kubelet: ${reference}`);
+        await new Promise(resolve => setTimeout(resolve, 100));
+      }
+      console.log(`[k3s-images] ${service}: ${reference} @ ${target.digest}, transferred ${transferred} bytes`);
+    } finally {
+      for (const proc of streams) proc.kill();
+      server.closeAllConnections();
+      await new Promise<void>(resolve => server.close(() => resolve()));
+      await exec(service, ["rm", "-rf", directory], { timeoutMs: 5_000 }).catch(() => undefined);
+    }
+  }
+}

package/src/harness/k3s-prepull.test.ts

@@ -0,0 +1,47 @@
+import { expect, test } from "bun:test";
+import type { SpectestContext } from "../index.js";
+import { prePullK3sImages } from "./k3s-prepull";
+
+const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
+const digest = `registry.test/app@sha256:${"a".repeat(64)}`;
+const mockExec = (fn: (args: string[]) => Promise<{ exitCode: number; stdout?: string; stderr?: string }>) =>
+  (async (_name: string, args: string[]) => ({ stdout: "", stderr: "", ...await fn(args) })) as SpectestContext["exec"];
+
+test("pre-pulls reuse immutable cached images, but resolve tags on every fresh setup", async () => {
+  const calls: string[][] = [];
+  await prePullK3sImages("cluster", [digest, "registry.test/app:latest"], mockExec(async args => {
+    calls.push(args); return { exitCode: 0 };
+  }));
+  expect(calls).toEqual([[...cri, "info"], [...cri, "inspecti", digest], [...cri, "pull", "registry.test/app:latest"]]);
+  calls.length = 0;
+  await prePullK3sImages("cluster", [digest], mockExec(async args => {
+    calls.push(args); return { exitCode: args[3] === "inspecti" ? 1 : 0 };
+  }));
+  expect(calls.at(-1)).toEqual([...cri, "pull", digest]);
+});
+
+test("failed pre-pulls fail setup after outstanding pulls finish, with bounded concurrency", async () => {
+  let active = 0, peak = 0, finished = 0;
+  const work = prePullK3sImages("cluster", ["a:1", "b:1", "c:1"], mockExec(async args => {
+    if (args.at(-1) === "info") return { exitCode: 0 };
+    peak = Math.max(peak, ++active);
+    await Bun.sleep(args.at(-1) === "a:1" ? 1 : 20);
+    --active; ++finished;
+    return { exitCode: args.at(-1) === "a:1" ? 1 : 0, stderr: "registry unavailable" };
+  }));
+  await expect(work).rejects.toThrow("pre-pull failed for a:1: registry unavailable");
+  expect(active).toBe(0);
+  expect(peak).toBe(2);
+  expect(finished).toBe(3);
+});
+
+test("pre-pulls wait for the runtime even when the API server is already ready", async () => {
+  const calls: string[] = [];
+  let ready = false;
+  await prePullK3sImages("cluster", ["app:1"], mockExec(async args => {
+    calls.push(args[3]!);
+    if (args[3] === "info" && !ready) { ready = true; return { exitCode: 1, stderr: "connection refused" }; }
+    return { exitCode: 0 };
+  }));
+  expect(calls).toEqual(["info", "info", "pull"]);
+});

package/src/harness/k3s-prepull.ts

@@ -0,0 +1,36 @@
+import type { SpectestContext } from "../index.js";
+
+/** CRI PullImage waits for unpacking and uses k3s's registry configuration.
+ * Always resolve tags on a fresh setup. Immutable digests can be used offline.
+ * Drain both workers on failure so no pulls outlive the setup hook.
+ */
+export async function prePullK3sImages(name: string, images: readonly string[], exec: SpectestContext["exec"]): Promise<void> {
+  if (!images.length) return;
+  await waitForK3sRuntime(name, exec);
+  const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
+  const queue = [...images];
+  const results = await Promise.allSettled(Array.from({ length: Math.min(2, queue.length) }, async () => {
+    for (let image = queue.shift(); image !== undefined; image = queue.shift()) {
+      if (/@sha256:[a-f0-9]{64}$/.test(image)) {
+        const existing = await exec(name, [...cri, "inspecti", image], { timeoutMs: 30_000 });
+        if (existing.exitCode === 0) continue;
+      }
+      const result = await exec(name, [...cri, "pull", image], { timeoutMs: 600_000 });
+      if (result.exitCode !== 0) throw new Error(`k3s(${name}): pre-pull failed for ${image}: ${result.stderr || result.stdout}`);
+    }
+  }));
+  const failed = results.find(result => result.status === "rejected");
+  if (failed?.status === "rejected") throw failed.reason;
+}
+
+export async function waitForK3sRuntime(name: string, exec: SpectestContext["exec"]): Promise<void> {
+  const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
+  // /readyz belongs to the API server and can precede the node runtime.
+  const deadline = Date.now() + 60_000;
+  for (;;) {
+    const result = await exec(name, [...cri, "info"], { timeoutMs: 5_000 });
+    if (result.exitCode === 0) break;
+    if (Date.now() >= deadline) throw new Error(`k3s(${name}): containerd did not become ready for pre-pulls: ${result.stderr || result.stdout}`);
+    await new Promise(resolve => setTimeout(resolve, 250));
+  }
+}

package/src/harness/sticky-consumer.test.ts

@@ -43,3 +43,31 @@ test("healthy cache keeps recent images and a cache that cannot shrink is not pu
   await expect(collectStickyCache(config, run, async () => ({ totalBytes: 256 * GiB, freeBytes: 5 * GiB })))
     .rejects.toThrow("cannot meet filesystem retention target");
 });
+
+test("nested Kubernetes stores share the filesystem budget and are pruned before recent Docker images", async () => {
+  let used = 100 * GiB;
+  const calls: string[][] = [];
+  let evictions = 0;
+  const run = async (command: string, args: string[]) => { calls.push([command, ...args]); return ""; };
+  await collectStickyCache(config, run, async () => ({ totalBytes: 256 * GiB, freeBytes: 256 * GiB - used }), {
+    protectedImages: new Set(), close: async () => {},
+    pruneOldest: async () => { used -= 40 * GiB; evictions++; return true; },
+  });
+  expect(evictions).toBe(1);
+  expect(calls.some(c => c[0] === "docker" && c[1] === "image" && c[2] === "ls")).toBe(false);
+});
+
+test("GC does not remove an image while it supplies the Kubernetes maintenance daemon", async () => {
+  const removed: string[] = [];
+  const run = async (command: string, args: string[]) => {
+    if (command === "docker" && args[1] === "ls") return "runtime\nold";
+    if (command === "docker" && args[1] === "inspect") return JSON.stringify([
+      { Id: "runtime", Created: "2020" }, { Id: "old", Created: "2021" }]);
+    if (command === "docker" && args[1] === "rm") removed.push(args.at(-1)!);
+    return "";
+  };
+  await collectStickyCache(config, run, async () => ({ totalBytes: 256 * GiB, freeBytes: (removed.length ? 220 : 150) * GiB }), {
+    protectedImages: new Set(["runtime"]), close: async () => {}, pruneOldest: async () => false,
+  });
+  expect(removed).toEqual(["old"]);
+});

package/src/harness/sticky-consumer.ts

@@ -1,10 +1,13 @@
-/** Docker/BuildKit policy for the generic sticky disk. No host publication API. */
+/** Docker, BuildKit and nested Kubernetes policy for the generic sticky disk. */
 import { execFile } from "node:child_process";
 import { mkdir, readFile, statfs, writeFile } from "node:fs/promises";
 
+import { STICKY_RUNTIME } from "./sticky-paths.js";
+import { openKubernetesCacheMaintenance, type KubernetesCacheMaintenance } from "./sticky-kubernetes.js";
+
 export const STICKY_CAPABILITY = "sticky-disks-v1";
 export const STICKY_COMPATIBILITY = "docker29-ctr21-buildkit032-overlayfs-256g-v2";
-export const STICKY_RUNTIME = "/run/spectest-sticky";
+export { STICKY_RUNTIME } from "./sticky-paths.js";
 export interface StickyConsumerConfig {
   path: string;
   capacityBytes: number;
@@ -115,6 +118,7 @@ export async function collectStickyCache(
     const fs = await statfs(config.path);
     return { freeBytes: fs.bavail * fs.bsize, totalBytes: fs.blocks * fs.bsize };
   },
+  kubernetes?: KubernetesCacheMaintenance,
 ): Promise<void> {
   validateStickyConfig(config);
   const buildctl = ["--addr", "tcp://127.0.0.1:1234", "prune"];
@@ -133,6 +137,10 @@ export async function collectStickyCache(
   };
   await pruneBuildkit();
   if (withinTarget(fs)) return;
+  while (await kubernetes?.pruneOldest()) {
+    fs = await usage();
+    if (withinTarget(fs)) return;
+  }
   // Pressure can be entirely in recent Docker images. Drop oldest images
   // first, even within the age window, until the whole filesystem fits.
   const ids = [...new Set((await run("docker", ["image", "ls", "-aq", "--no-trunc"])).split(/\s+/).filter(Boolean))];
@@ -140,6 +148,7 @@ export async function collectStickyCache(
     ? JSON.parse(await run("docker", ["image", "inspect", ...ids])) : [];
   images.sort((a, b) => a.Created.localeCompare(b.Created) || a.Id.localeCompare(b.Id));
   for (const image of images) {
+    if (kubernetes?.protectedImages.has(image.Id)) continue;
     await run("docker", ["image", "rm", "-f", image.Id], 120_000);
     await pruneBuildkit();
     if (withinTarget(fs)) return;
@@ -158,7 +167,9 @@ export async function prepareStickyConsumer(artifact = false): Promise<{ prepare
     if (containers.length) await exec("docker", ["rm", "-f", ...containers]);
     const volumes = (await exec("docker", ["volume", "ls", "-q"])).split(/\s+/).filter(Boolean);
     if (volumes.length) await exec("docker", ["volume", "rm", ...volumes]);
-    await collectStickyCache(config);
+    const kubernetes = await openKubernetesCacheMaintenance(exec);
+    try { await collectStickyCache(config, exec, undefined, kubernetes); }
+    finally { await kubernetes.close(); }
   }
   const units = ["spectest-sticky-buildkit", "spectest-sticky-docker", "spectest-sticky-containerd"];
   for (const unit of units) {