@specific.dev/spectest 0.90.2 → 0.91.0

package/dist/daemon.js

@@ -1,3 +1,4 @@
+import { componentBuildTasks, COMPONENT_RUNTIME } from "./harness/component-images.js";
 import { androidScope, androidRecordingChunk, releaseAndroidRecording, isAndroidApp } from "./android.js";
 // Long-running HTTP daemon. Runs as a systemd unit on the VM host (not in
 // a container). Owns:
@@ -24,7 +25,7 @@ import net from "node:net";
 import path from "node:path";
 import { pathToFileURL } from "node:url";
 import { ENVIRONMENT_CA_PATH as CA_PATH } from "./environment-ca.js";
-import { assert, expect, expectRaw, lowerIngress, dnsName as makeDnsDecl, isWildcard, validateServiceImage, proxy as makeProxyDecl, } from "./index.js";
+import { assert, expect, expectRaw, lowerIngress, dnsName as makeDnsDecl, isWildcard, validateServiceImage, normalizeServiceImage, proxy as makeProxyDecl, } from "./index.js";
 import { COVERAGE_CONTAINER_DIR, coverageBundleRef, coverageHostDir, encodeCoverageBundle, isEmptyReport, readCoverageDir, applyCoverageDelta, walkProjectFiles, ENVIRONMENT_SERVICE, REACH_REPORT, READS_REPORT, } from "./harness/coverage.js";
 import { configureBrowserCoverage } from "./browser-coverage.js";
 import { INVENTORY_REPORT, NODE_COVERAGE_MARKERS_SUBDIR, NODE_COVERAGE_SCRIPTS_SUBDIR, applyCoverageAdapters, coverageAdapters, coverageReportsMode, validateCoverage, } from "./coverage.js";
@@ -57,6 +58,7 @@ import { certCovers as hostmatchCertCovers, hostWithoutPort, matchRoute, selectC
 import { INGRESS_HTTPS_PORT, INGRESS_HTTP_PORT, bindRoute, clearTables, emptyTables, planBind, registryTarget, routesFor, unbindRoute, } from "./harness/ingress-table.js";
 import { startTlsTerminator } from "./harness/tls-terminator.js";
 import { runContainerArgs } from "./harness/container-run.js";
+import { isOnStickyDisk } from "./harness/sticky-paths.js";
 import { BUILDKIT_CACHE_DIR, BUILDKIT_CACHE_UNION, IMAGE_CACHE_MANIFEST, imageCachePathsSync, isOnImageCache, mergeCacheIndex, } from "./harness/image-cache.js";
 import { assertAbsolute, certificateHostnames, defaultKeyMode, expandServiceToken, isNoopChown, mountFlag, needsIdTables, numericId, resolveChownIds, } from "./harness/file-mounts.js";
 import { badRequest, conflict, notFound, requireString, } from "./harness/methods.js";
@@ -70,7 +72,18 @@ import { deepUnwrap, readRaw, wrap } from "./inspect.js";
 import { setRecordSecrets } from "./record-secrets.js";
 import { encodeReplayBundle, replayChunk } from "./replay-bundle.js";
 function namedServices(cfg) {
-    return Object.entries(cfg.services).map(([name, def]) => ({ name, ...def }));
+    return Object.entries(cfg.services).map(([name, def]) => ({ name, ...def, image: normalizeServiceImage(def.image) }));
+}
+/** Build tasks participate in image preparation, never container startup. */
+function imageTasks(services) {
+    const tasks = componentBuildTasks(services).map(task => ({ ...task, image: validateServiceImage(task.name, task.image) }));
+    const names = new Set(services.map(s => s.name));
+    for (const task of tasks) {
+        if (names.has(task.name))
+            throw new Error(`image task name conflicts with service: ${task.name}`);
+        names.add(task.name);
+    }
+    return [...services, ...tasks];
 }
 const DEFAULT_TEST_TIMEOUT_MS = 60_000;
 const NETWORK_NAME = process.env.SPECTEST_NETWORK ?? "spectest-net";
@@ -677,7 +690,7 @@ async function ensureVolumes(svc) {
             // A directory on a cache disk is the same kind of thing: a nested
             // runtime's containerd root (`k3s()`), kept as a cache by the
             // lineage exactly as the container store one level up is.
-            const durable = host.startsWith("/var/cache/spectest/") || isOnImageCache(host, imageCachePathsSync(IMAGE_CACHE_MANIFEST));
+            const durable = host.startsWith("/var/cache/spectest/") || isOnStickyDisk(host) || isOnImageCache(host, imageCachePathsSync(IMAGE_CACHE_MANIFEST));
             if (vol.source?.startsWith("/") && !durable) {
                 await recordAbsoluteVolumeDir(host);
             }
@@ -1013,6 +1026,15 @@ async function resolveDockerfileBuild(svc) {
     };
 }
 async function prepareServiceImage(svc, opts) {
+    const result = await prepareServiceImageInner(svc, opts);
+    if (svc.image.type === "dockerfile" && svc.image.reference) {
+        const tagged = await docker(["tag", result.tag, svc.image.reference], 30_000);
+        if (tagged.code !== 0)
+            throw new Error(`cannot name built image ${svc.image.reference}: ${tagged.stderr}`);
+    }
+    return result;
+}
+async function prepareServiceImageInner(svc, opts) {
     const tag = imageTag(svc.name);
     if (svc.image.type === "registry") {
         const ref = svc.image.reference;
@@ -1062,7 +1084,7 @@ async function prepareServiceImage(svc, opts) {
                 progressService(svc.name, { status: "failed" });
                 throw err;
             }
-            // first.tag is already CA-layered; tagging it covers this service too.
+            // Reuse the same application image; runtime trust is added separately.
             const tagr = await docker(["tag", first.tag, tag], 30_000);
             if (tagr.code !== 0) {
                 throw new Error(`docker tag ${first.tag} ${tag} failed: ${tagr.stderr.trim()}`);
@@ -2640,7 +2662,8 @@ function specToNamedService(spec) {
     // run here: the image is checked (a `path` Dockerfile and a `context`
     // must exist), and coverage adapters get their configure step.
     const svc = { ...rest, image: validateServiceImage(name, rest.image) };
-    return { name, ...applyCoverageAdapters(name, svc) };
+    const configured = applyCoverageAdapters(name, svc);
+    return { name, ...configured, image: normalizeServiceImage(configured.image) };
 }
 /** Implementation behind `ctx.startService` / a fake's `ctx.startService`.
  *  Prepares the image (pulling on first use through the host cache), runs
@@ -2931,12 +2954,14 @@ async function bootstrapInner(prepared, identity) {
     const bootStart = Date.now();
     const cfg = requireLoaded().project.environment;
     const services = namedServices(cfg);
+    const tasks = imageTasks(services);
+    const componentTasks = componentBuildTasks(services);
     const timings = new Map();
     if (prepared && (prepared.version !== 1 || !prepared.images ||
-        Object.keys(prepared.images).sort().join("\n") !== services.map(s => s.name).sort().join("\n"))) {
+        Object.keys(prepared.images).sort().join("\n") !== tasks.map(s => s.name).sort().join("\n"))) {
         throw new Error("prepared image manifest does not match the environment's services");
     }
-    progressInit(services);
+    progressInit(tasks);
     // Build dedup is only valid within one workspace generation — a fresh
     // bootstrap may follow a workspace re-upload with the same dockerfile
     // text but different build-context content.
@@ -2957,7 +2982,7 @@ async function bootstrapInner(prepared, identity) {
             // that reads it.
             if ((await readIgnoreFile(path.join(WORKSPACE, ".dockerignore"))) !== null)
                 return;
-            await fs.writeFile(path.join(WORKSPACE, ".dockerignore"), unionDockerignore(services));
+            await fs.writeFile(path.join(WORKSPACE, ".dockerignore"), unionDockerignore(tasks));
         })(),
     ]);
     // Image prep is DECOUPLED from container start: every service's image
@@ -2993,11 +3018,15 @@ async function bootstrapInner(prepared, identity) {
         if (found.code !== 0 || found.stdout.trim() !== entry.image) {
             throw new Error(`prepared image missing for ${svc.name}: ${entry.image}`);
         }
-        return { tag: entry.image, buildSteps: entry.buildSteps };
+        const tag = imageTag(svc.name);
+        const tagged = await docker(["tag", entry.image, tag], 30_000);
+        if (tagged.code !== 0)
+            throw new Error(`cannot restore prepared image name for ${svc.name}: ${tagged.stderr}`);
+        return { tag, buildSteps: entry.buildSteps };
     };
     const prep = new Map();
     const prepOffsets = new Map();
-    for (const svc of services) {
+    for (const svc of tasks) {
         const p = (async () => {
             const t0 = Date.now();
             prepOffsets.set(svc.name, Date.now() - bootStart);
@@ -3022,7 +3051,7 @@ async function bootstrapInner(prepared, identity) {
         // No application services, setup hooks, ingress, or runtime state in the builder.
         await Promise.all(prep.values());
         const images = Object.create(null);
-        for (const svc of services) {
+        for (const svc of tasks) {
             const out = await docker(["image", "inspect", "--format", "{{.Id}}", tags.get(svc.name)], 30_000);
             const image = out.stdout.trim();
             if (out.code !== 0 || !/^sha256:[a-f0-9]{64}$/.test(image)) {
@@ -3073,6 +3102,8 @@ async function bootstrapInner(prepared, identity) {
         progressService(svc.name, { status: "probing", detail: "ready check" });
         await waitForReady(svc);
         const tSetup = Date.now();
+        const builds = componentTasks.filter(task => task.owner === svc.name);
+        await Promise.all(builds.map(task => prep.get(task.name)));
         if (svc.setup) {
             progressService(svc.name, { status: "probing", detail: "running setup" });
             const helpers = await ensureHelpers(svc.name, svc);
@@ -3081,12 +3112,19 @@ async function bootstrapInner(prepared, identity) {
                     name: svc.name,
                     helpers,
                     ...(await spectestContext({ service: svc.name, includeSelf: true })),
+                    ...{ [COMPONENT_RUNTIME]: {
+                            images: builds.map(task => ({ source: tags.get(task.name), reference: task.reference })),
+                            gateway: await bridgeGatewayIp(),
+                            caBundle: await ensureCaBundle(),
+                        } },
                 });
             }
             catch (err) {
                 throw await withContainerPostMortem(svc.name, err);
             }
         }
+        for (const task of builds)
+            progressService(task.name, { status: "ready", detail: `loaded ${task.reference}` });
         progressService(svc.name, { status: "ready", detail: undefined });
         const ti = timings.get(svc.name);
         if (ti) {
@@ -3545,9 +3583,10 @@ async function computeProjectReach(all) {
 async function computeProjectReachInner(all) {
     const files = await walkProjectFiles(WORKSPACE, [".git"]);
     const sources = [];
-    for (const svc of all) {
+    const owners = new Map(componentBuildTasks(all).map(t => [t.name, t.owner]));
+    for (const svc of imageTasks(all)) {
         const src = {
-            service: svc.name,
+            service: owners.get(svc.name) ?? svc.name,
             volumeSources: (svc.volumes ?? []).flatMap((v) => (v.source && !v.reachExempt ? [v.source] : [])),
             ...(svc.inputs && svc.inputs.length > 0 ? { inputs: [...svc.inputs] } : {}),
         };
@@ -5611,7 +5650,7 @@ export function harnessMethods(state) {
             }
         }),
         bootstrap: async (params) => exclusive(state, "inFlightBootstrap", "bootstrap already in progress", async () => {
-            const prepared = params.prepared === undefined ? undefined : preparedImages(params.prepared, params.identity, namedServices(requireLoaded().project.environment).map(s => s.name));
+            const prepared = params.prepared === undefined ? undefined : preparedImages(params.prepared, params.identity, imageTasks(namedServices(requireLoaded().project.environment)).map(s => s.name));
             return { ok: true, timings: await bootstrap(prepared) };
         }),
         projectSetup: async () => exclusive(state, "inFlightProjectSetup", "project-setup already in progress", () => runProjectSetup()),

package/dist/harness/component-images.d.ts

@@ -0,0 +1,31 @@
+import type { BuildImage, ComponentImageBuild, ServiceConfig } from "../index.js";
+export type KubernetesImage = string | (BuildImage & {
+    reference: string;
+});
+/** containerd/CRI use fully qualified names, including the default tag. */
+export declare function canonicalImageReference(reference: string): string;
+export declare function kubernetesImages(images: readonly KubernetesImage[]): {
+    pulls: string[];
+    builds: ComponentImageBuild[];
+};
+/** Stable internal task names, separate from the service graph. */
+export declare function componentBuildTasks<T extends ServiceConfig & {
+    name: string;
+}>(services: readonly T[]): {
+    name: string;
+    owner: string;
+    reference: string;
+    image: BuildImage;
+}[];
+/** Private handoff to a component's setup hook. Builds stay generic; the
+ * component decides how its runtime consumes them and establishes trust. */
+export declare const COMPONENT_RUNTIME: unique symbol;
+export interface ComponentRuntime {
+    images: readonly {
+        source: string;
+        reference: string;
+    }[];
+    gateway: string;
+    caBundle: string | null;
+}
+export declare function componentRuntime(context: object): ComponentRuntime | undefined;

package/dist/harness/component-images.js

@@ -0,0 +1,69 @@
+import { createHash } from "node:crypto";
+/** containerd/CRI use fully qualified names, including the default tag. */
+export function canonicalImageReference(reference) {
+    if (typeof reference !== "string" || !reference || reference.startsWith("-") || /[\s\x00]/.test(reference)) {
+        throw new Error("image reference must be a non-empty image name without whitespace");
+    }
+    const slash = reference.indexOf("/");
+    const first = slash < 0 ? "" : reference.slice(0, slash);
+    let result = slash < 0 ? `docker.io/library/${reference}`
+        : first.includes(".") || first.includes(":") || first === "localhost" ? reference : `docker.io/${reference}`;
+    if (result.startsWith("index.docker.io/"))
+        result = result.replace("index.docker.io/", "docker.io/");
+    if (result.startsWith("docker.io/") && !result.slice(10).includes("/"))
+        result = `docker.io/library/${result.slice(10)}`;
+    const [name, digest] = result.split("@");
+    const tagAt = name.lastIndexOf(":") > name.lastIndexOf("/") ? name.lastIndexOf(":") : -1;
+    const repository = tagAt < 0 ? name : name.slice(0, tagAt);
+    if (!/^[a-z0-9.-]+(?::[0-9]+)?\/[a-z0-9]+(?:(?:[._]|__|[-]+)[a-z0-9]+)*(?:\/[a-z0-9]+(?:(?:[._]|__|[-]+)[a-z0-9]+)*)*$/.test(repository)
+        || (tagAt >= 0 && !/^[\w][\w.-]{0,127}$/.test(name.slice(tagAt + 1)))
+        || (result.includes("@") && (!digest || !/^sha256:[a-f0-9]{64}$/.test(digest) || result.split("@").length !== 2))) {
+        throw new Error(`invalid image reference: ${reference}`);
+    }
+    if (tagAt < 0 && !digest)
+        result += ":latest";
+    return result;
+}
+export function kubernetesImages(images) {
+    if (!Array.isArray(images))
+        throw new Error("k3s(): images must be an array of registry references or Dockerfile builds");
+    const pulls = [];
+    const builds = [];
+    const seen = new Map();
+    for (const image of images) {
+        if (typeof image !== "string" && (!image || typeof image !== "object" || "type" in image || !image.reference)) {
+            throw new Error("k3s(): images expects registry strings or Dockerfile builds with a reference");
+        }
+        const reference = canonicalImageReference(typeof image === "string" ? image : image.reference);
+        if (typeof image !== "string" && reference.includes("@"))
+            throw new Error("k3s(): a build reference must be an image name/tag, not a digest");
+        const identity = JSON.stringify(image);
+        const previous = seen.get(reference);
+        if (previous !== undefined) {
+            if (previous !== identity)
+                throw new Error(`k3s(): conflicting images for ${reference}`);
+            continue;
+        }
+        seen.set(reference, identity);
+        if (typeof image === "string")
+            pulls.push(image);
+        else
+            builds.push({ reference, image });
+    }
+    return { pulls, builds };
+}
+/** Stable internal task names, separate from the service graph. */
+export function componentBuildTasks(services) {
+    return services.flatMap(svc => (svc.imageBuilds ?? []).map(build => ({
+        name: `spectest-image-${createHash("sha256").update(`${svc.name}\0${build.reference}`).digest("hex")}`,
+        owner: svc.name,
+        reference: build.reference,
+        image: build.image,
+    })));
+}
+/** Private handoff to a component's setup hook. Builds stay generic; the
+ * component decides how its runtime consumes them and establishes trust. */
+export const COMPONENT_RUNTIME = Symbol.for("spectest.component-runtime");
+export function componentRuntime(context) {
+    return context[COMPONENT_RUNTIME];
+}

package/dist/harness/k3s-image-loader.d.ts

@@ -0,0 +1,5 @@
+import type { SpectestContext } from "../index.js";
+export declare function loadK3sBuildImages(service: string, images: readonly {
+    source: string;
+    reference: string;
+}[], gateway: string, exec: SpectestContext["exec"]): Promise<void>;

package/dist/harness/k3s-image-loader.js

@@ -0,0 +1,150 @@
+/** A short-lived, read-only OCI endpoint over the host containerd content store.
+ * k3s fetches only missing blobs and unpacks only missing snapshots. No Docker
+ * archive, staging registry, or second persistent copy of application blobs. */
+import { execFile, spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { createServer } from "node:http";
+import { promisify } from "node:util";
+import { canonicalImageReference } from "./component-images.js";
+import { waitForK3sRuntime } from "./k3s-prepull.js";
+const digest = (bytes) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+const descriptor = (bytes, mediaType) => ({ digest: digest(bytes), size: bytes.length, mediaType });
+const run = promisify(execFile);
+const HOST_CTR = ["--address", "/run/containerd/containerd.sock", "--namespace", "moby"];
+async function hostContent(id) {
+    if (!/^sha256:[a-f0-9]{64}$/.test(id))
+        throw new Error(`invalid content digest ${id}`);
+    const result = await run("ctr", [...HOST_CTR, "content", "get", id], { encoding: "buffer", maxBuffer: 16 * 1024 * 1024, timeout: 30_000 });
+    if (digest(result.stdout) !== id)
+        throw new Error(`corrupt image metadata ${id}`);
+    return result.stdout;
+}
+/** Preserve the original index/manifest bytes and all application descriptors. */
+async function hostImage(source) {
+    const name = canonicalImageReference(source);
+    const { stdout } = await run("ctr", [...HOST_CTR, "images", "ls", `name==${name}`], { timeout: 30_000 });
+    const found = stdout.split("\n").map(line => line.trim().split(/\s+/)).find(row => row[0] === name)?.[2];
+    if (!found)
+        throw new Error(`built image ${name} is missing from the VM container store`);
+    const metadata = new Map();
+    const allowed = new Map();
+    let configDigest;
+    async function visit(id, selected, depth = 0) {
+        if (depth > 4)
+            throw new Error(`built image ${name} has a recursive manifest index`);
+        const bytes = await hostContent(id);
+        const obj = JSON.parse(bytes.toString());
+        const desc = descriptor(bytes, obj.mediaType);
+        metadata.set(id, bytes);
+        allowed.set(id, desc);
+        if (obj.config && Array.isArray(obj.layers)) {
+            const manifest = obj;
+            for (const blob of [manifest.config, ...manifest.layers])
+                allowed.set(blob.digest, blob);
+            if (selected)
+                configDigest = manifest.config.digest;
+        }
+        else if (Array.isArray(obj.manifests)) {
+            for (const entry of obj.manifests) {
+                const matches = !entry.platform || (entry.platform.os === "linux" && entry.platform.architecture === (process.arch === "arm64" ? "arm64" : "amd64"));
+                await visit(entry.digest, selected && matches, depth + 1);
+            }
+        }
+        else
+            throw new Error(`built image ${name} has invalid image metadata`);
+        return desc;
+    }
+    const target = await visit(found, true);
+    if (!configDigest)
+        throw new Error(`built image ${name} has no image for this VM's platform`);
+    return { target, metadata, allowed, configDigest };
+}
+export async function loadK3sBuildImages(service, images, gateway, exec) {
+    if (!images.length)
+        return;
+    await waitForK3sRuntime(service, exec);
+    const checked = async (args, timeoutMs = 30_000) => {
+        const result = await exec(service, args, { timeoutMs });
+        if (result.exitCode !== 0)
+            throw new Error(`k3s(${service}): image load failed: ${result.stderr || result.stdout}`);
+        return result.stdout;
+    };
+    const config = await checked(["cat", "/var/lib/rancher/k3s/agent/etc/containerd/config.toml"]);
+    const snapshotter = config.match(/^\s*snapshotter\s*=\s*['"]([^'"]+)['"]/m)?.[1];
+    if (!snapshotter)
+        throw new Error(`k3s(${service}): cannot determine kubelet's snapshotter`);
+    const ctr = ["ctr", "--address", "/run/k3s/containerd/containerd.sock", "--namespace", "k8s.io"];
+    const help = await checked([...ctr, "images", "pull", "--help"]);
+    const local = help.includes("--local") ? ["--local"] : [];
+    for (const image of images) {
+        const reference = canonicalImageReference(image.reference);
+        const { target, metadata: blobs, allowed, configDigest } = await hostImage(image.source);
+        const streams = new Set();
+        let transferred = 0;
+        const server = createServer((req, res) => {
+            const pathname = new URL(req.url ?? "/", "http://local").pathname;
+            const match = pathname.match(/^\/v2\/.+\/(manifests|blobs)\/([^/]+)$/);
+            if (!["GET", "HEAD"].includes(req.method ?? "") || !match) {
+                res.writeHead(404).end();
+                return;
+            }
+            const id = match[1] === "manifests" && !match[2].startsWith("sha256:") ? target.digest : match[2];
+            const desc = allowed.get(id);
+            if (!desc) {
+                res.writeHead(404).end();
+                return;
+            }
+            res.writeHead(200, { "Content-Type": desc.mediaType, "Content-Length": desc.size, "Docker-Content-Digest": id });
+            if (req.method === "HEAD") {
+                res.end();
+                return;
+            }
+            transferred += desc.size;
+            const bytes = blobs.get(id);
+            if (bytes) {
+                res.end(bytes);
+                return;
+            }
+            const proc = spawn("ctr", [...HOST_CTR, "content", "get", id], { stdio: ["ignore", "pipe", "ignore"] });
+            streams.add(proc);
+            proc.stdout.pipe(res);
+            proc.on("error", err => res.destroy(err));
+            proc.on("exit", code => { streams.delete(proc); if (code !== 0)
+                res.destroy(new Error(`content read failed: ${id}`)); });
+            res.on("close", () => { if (proc.exitCode === null)
+                proc.kill(); });
+        });
+        const directory = `/run/spectest-image-load-${createHash("sha256").update(reference).digest("hex").slice(0, 16)}`;
+        try {
+            await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, gateway, resolve); });
+            const port = server.address().port;
+            // A private hosts directory scopes this override to this single ctr call;
+            // it cannot change the cluster's configured registries or pull policy.
+            const hosts = `server = "http://${gateway}:${port}"\n`;
+            await checked(["sh", "-c", 'mkdir -p "$1/_default"; printf "%s" "$2" > "$1/_default/hosts.toml"', "sh", directory, hosts]);
+            await checked([...ctr, "images", "pull", ...local, "--hosts-dir", directory,
+                "--snapshotter", snapshotter, "--label", "io.cri-containerd.image=managed", reference], 600_000);
+            const repo = reference.slice(0, reference.lastIndexOf(":"));
+            await checked([...ctr, "images", "tag", "--force", reference, `${repo}@${target.digest}`]);
+            // CRI learns about ctr image updates asynchronously. Confirm the current
+            // build's config, not merely that an older tag already exists.
+            const deadline = Date.now() + 30_000;
+            for (;;) {
+                const result = await exec(service, ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock", "inspecti", reference], { timeoutMs: 5_000 });
+                if (result.exitCode === 0 && JSON.parse(result.stdout).status?.id === configDigest)
+                    break;
+                if (Date.now() >= deadline)
+                    throw new Error(`k3s(${service}): current build is not visible to kubelet: ${reference}`);
+                await new Promise(resolve => setTimeout(resolve, 100));
+            }
+            console.log(`[k3s-images] ${service}: ${reference} @ ${target.digest}, transferred ${transferred} bytes`);
+        }
+        finally {
+            for (const proc of streams)
+                proc.kill();
+            server.closeAllConnections();
+            await new Promise(resolve => server.close(() => resolve()));
+            await exec(service, ["rm", "-rf", directory], { timeoutMs: 5_000 }).catch(() => undefined);
+        }
+    }
+}

package/dist/harness/k3s-prepull.d.ts

@@ -0,0 +1,7 @@
+import type { SpectestContext } from "../index.js";
+/** CRI PullImage waits for unpacking and uses k3s's registry configuration.
+ * Always resolve tags on a fresh setup. Immutable digests can be used offline.
+ * Drain both workers on failure so no pulls outlive the setup hook.
+ */
+export declare function prePullK3sImages(name: string, images: readonly string[], exec: SpectestContext["exec"]): Promise<void>;
+export declare function waitForK3sRuntime(name: string, exec: SpectestContext["exec"]): Promise<void>;

package/dist/harness/k3s-prepull.js

@@ -0,0 +1,39 @@
+/** CRI PullImage waits for unpacking and uses k3s's registry configuration.
+ * Always resolve tags on a fresh setup. Immutable digests can be used offline.
+ * Drain both workers on failure so no pulls outlive the setup hook.
+ */
+export async function prePullK3sImages(name, images, exec) {
+    if (!images.length)
+        return;
+    await waitForK3sRuntime(name, exec);
+    const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
+    const queue = [...images];
+    const results = await Promise.allSettled(Array.from({ length: Math.min(2, queue.length) }, async () => {
+        for (let image = queue.shift(); image !== undefined; image = queue.shift()) {
+            if (/@sha256:[a-f0-9]{64}$/.test(image)) {
+                const existing = await exec(name, [...cri, "inspecti", image], { timeoutMs: 30_000 });
+                if (existing.exitCode === 0)
+                    continue;
+            }
+            const result = await exec(name, [...cri, "pull", image], { timeoutMs: 600_000 });
+            if (result.exitCode !== 0)
+                throw new Error(`k3s(${name}): pre-pull failed for ${image}: ${result.stderr || result.stdout}`);
+        }
+    }));
+    const failed = results.find(result => result.status === "rejected");
+    if (failed?.status === "rejected")
+        throw failed.reason;
+}
+export async function waitForK3sRuntime(name, exec) {
+    const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
+    // /readyz belongs to the API server and can precede the node runtime.
+    const deadline = Date.now() + 60_000;
+    for (;;) {
+        const result = await exec(name, [...cri, "info"], { timeoutMs: 5_000 });
+        if (result.exitCode === 0)
+            break;
+        if (Date.now() >= deadline)
+            throw new Error(`k3s(${name}): containerd did not become ready for pre-pulls: ${result.stderr || result.stdout}`);
+        await new Promise(resolve => setTimeout(resolve, 250));
+    }
+}

package/dist/harness/sticky-consumer.d.ts

@@ -1,6 +1,7 @@
+import { type KubernetesCacheMaintenance } from "./sticky-kubernetes.js";
 export declare const STICKY_CAPABILITY = "sticky-disks-v1";
 export declare const STICKY_COMPATIBILITY = "docker29-ctr21-buildkit032-overlayfs-256g-v2";
-export declare const STICKY_RUNTIME = "/run/spectest-sticky";
+export { STICKY_RUNTIME } from "./sticky-paths.js";
 export interface StickyConsumerConfig {
     path: string;
     capacityBytes: number;
@@ -19,10 +20,9 @@ export declare function initializeStickyConsumer(config: StickyConsumerConfig):
 export declare function collectStickyCache(config: StickyConsumerConfig, run?: typeof exec, usage?: () => Promise<{
     freeBytes: number;
     totalBytes: number;
-}>): Promise<void>;
+}>, kubernetes?: KubernetesCacheMaintenance): Promise<void>;
 /** Only on an isolated maintenance clone; never on the live ready environment. */
 export declare function prepareStickyConsumer(artifact?: boolean): Promise<{
     prepared: true;
     freeBytes: number;
 }>;
-export {};

package/dist/harness/sticky-consumer.js

@@ -1,9 +1,11 @@
-/** Docker/BuildKit policy for the generic sticky disk. No host publication API. */
+/** Docker, BuildKit and nested Kubernetes policy for the generic sticky disk. */
 import { execFile } from "node:child_process";
 import { mkdir, readFile, statfs, writeFile } from "node:fs/promises";
+import { STICKY_RUNTIME } from "./sticky-paths.js";
+import { openKubernetesCacheMaintenance } from "./sticky-kubernetes.js";
 export const STICKY_CAPABILITY = "sticky-disks-v1";
 export const STICKY_COMPATIBILITY = "docker29-ctr21-buildkit032-overlayfs-256g-v2";
-export const STICKY_RUNTIME = "/run/spectest-sticky";
+export { STICKY_RUNTIME } from "./sticky-paths.js";
 function exec(program, args, timeout = 60_000) {
     return new Promise((resolve, reject) => {
         execFile(program, args, { timeout, maxBuffer: 8 * 1024 * 1024 }, (error, stdout, stderr) => {
@@ -110,7 +112,7 @@ disabled_plugins = ["io.containerd.grpc.v1.cri"]
 export async function collectStickyCache(config, run = exec, usage = async () => {
     const fs = await statfs(config.path);
     return { freeBytes: fs.bavail * fs.bsize, totalBytes: fs.blocks * fs.bsize };
-}) {
+}, kubernetes) {
     validateStickyConfig(config);
     const buildctl = ["--addr", "tcp://127.0.0.1:1234", "prune"];
     await run("buildctl", [...buildctl, "--keep-duration", "168h"], 120_000);
@@ -128,6 +130,11 @@ export async function collectStickyCache(config, run = exec, usage = async () =>
     await pruneBuildkit();
     if (withinTarget(fs))
         return;
+    while (await kubernetes?.pruneOldest()) {
+        fs = await usage();
+        if (withinTarget(fs))
+            return;
+    }
     // Pressure can be entirely in recent Docker images. Drop oldest images
     // first, even within the age window, until the whole filesystem fits.
     const ids = [...new Set((await run("docker", ["image", "ls", "-aq", "--no-trunc"])).split(/\s+/).filter(Boolean))];
@@ -135,6 +142,8 @@ export async function collectStickyCache(config, run = exec, usage = async () =>
         ? JSON.parse(await run("docker", ["image", "inspect", ...ids])) : [];
     images.sort((a, b) => a.Created.localeCompare(b.Created) || a.Id.localeCompare(b.Id));
     for (const image of images) {
+        if (kubernetes?.protectedImages.has(image.Id))
+            continue;
         await run("docker", ["image", "rm", "-f", image.Id], 120_000);
         await pruneBuildkit();
         if (withinTarget(fs))
@@ -157,7 +166,13 @@ export async function prepareStickyConsumer(artifact = false) {
         const volumes = (await exec("docker", ["volume", "ls", "-q"])).split(/\s+/).filter(Boolean);
         if (volumes.length)
             await exec("docker", ["volume", "rm", ...volumes]);
-        await collectStickyCache(config);
+        const kubernetes = await openKubernetesCacheMaintenance(exec);
+        try {
+            await collectStickyCache(config, exec, undefined, kubernetes);
+        }
+        finally {
+            await kubernetes.close();
+        }
     }
     const units = ["spectest-sticky-buildkit", "spectest-sticky-docker", "spectest-sticky-containerd"];
     for (const unit of units) {

package/dist/harness/sticky-kubernetes.d.ts

@@ -0,0 +1,15 @@
+export type CacheCommand = (program: string, args: string[], timeout?: number) => Promise<string>;
+export declare function stickyKubernetesDirectory(service: string, version: string): string;
+export declare function registerStickyKubernetesStore(service: string, version: string, run?: CacheCommand): Promise<void>;
+/** Standalone containerd never starts CRI/kubelet or any inherited workload.
+ * Native API deletion preserves content and committed image snapshots while
+ * releasing sandbox, container, lease and writable snapshot references. */
+export declare const KUBERNETES_CACHE_MAINTENANCE_SH = "set -eu\nexec 2>&1\ncat > /run/cache.toml <<'CONFIG'\nversion = 2\ndisabled_plugins = [\"io.containerd.grpc.v1.cri\", \"io.containerd.cri.v1.runtime\", \"io.containerd.cri.v1.images\"]\nCONFIG\ncontainerd --config /run/cache.toml --root /var/lib/rancher/k3s/agent/containerd --state /run/cache-state --address /run/spectest-cache.sock > /run/cache.log 2>&1 &\npid=$!\nstop() { kill \"$pid\"; wait \"$pid\"; }\ntrap 'stop; exit 0' TERM INT\ntrap 'kill \"$pid\" 2>/dev/null || true' EXIT\nc() { ctr --address /run/spectest-cache.sock --namespace k8s.io \"$@\"; }\ni=0\nuntil c version >/dev/null 2>&1; do\n  kill -0 \"$pid\" || { cat /run/cache.log; exit 1; }\n  i=$((i+1)); [ \"$i\" -lt 300 ] || { cat /run/cache.log; exit 1; }\n  sleep 0.1\ndone\nids=$(c containers ls -q)\nif [ -n \"$ids\" ]; then c containers rm $ids; fi\n# 1.7 has no sandbox store; 2.x does.\ncase \"$(containerd --version)\" in\n  *v1.*) ;;\n  *) sandboxes=$(c sandboxes ls)\n     ids=$(printf '%s\\n' \"$sandboxes\" | awk 'NR > 1 {print $1}')\n     if [ -n \"$ids\" ]; then c sandboxes rm --force $ids; fi ;;\nesac\nids=$(c leases ls -q)\nif [ -n \"$ids\" ]; then c leases rm $ids; fi\n# Remove children before parents, and tolerate asynchronous GC winning a race.\nprevious=''\nwhile :; do\n  snapshots=$(c snapshots --snapshotter overlayfs ls)\n  ids=$(printf '%s\\n' \"$snapshots\" | awk 'NR > 1 && $NF != \"Committed\" {print $1}')\n  [ -n \"$ids\" ] || break\n  [ \"$ids\" != \"$previous\" ] || { echo 'cannot clean Kubernetes writable snapshots' >&2; exit 1; }\n  previous=\"$ids\"\n  for id in $ids; do c snapshots --snapshotter overlayfs rm \"$id\" >/dev/null 2>&1 || true; done\ndone\ntouch /run/cache-ready\nwhile :; do sleep 1 & wait $!; done\n";
+export interface KubernetesCacheMaintenance {
+    /** Outer images needed by live maintenance containers must not be evicted. */
+    protectedImages: Set<string>;
+    /** Evicts one complete image store, oldest service/version first. */
+    pruneOldest(): Promise<boolean>;
+    close(): Promise<void>;
+}
+export declare function openKubernetesCacheMaintenance(run: CacheCommand, root?: string): Promise<KubernetesCacheMaintenance>;