@specific.dev/spectest 0.90.3 → 0.91.0

package/dist/harness/component-images.js

@@ -0,0 +1,69 @@
+import { createHash } from "node:crypto";
+/** containerd/CRI use fully qualified names, including the default tag. */
+export function canonicalImageReference(reference) {
+    if (typeof reference !== "string" || !reference || reference.startsWith("-") || /[\s\x00]/.test(reference)) {
+        throw new Error("image reference must be a non-empty image name without whitespace");
+    }
+    const slash = reference.indexOf("/");
+    const first = slash < 0 ? "" : reference.slice(0, slash);
+    let result = slash < 0 ? `docker.io/library/${reference}`
+        : first.includes(".") || first.includes(":") || first === "localhost" ? reference : `docker.io/${reference}`;
+    if (result.startsWith("index.docker.io/"))
+        result = result.replace("index.docker.io/", "docker.io/");
+    if (result.startsWith("docker.io/") && !result.slice(10).includes("/"))
+        result = `docker.io/library/${result.slice(10)}`;
+    const [name, digest] = result.split("@");
+    const tagAt = name.lastIndexOf(":") > name.lastIndexOf("/") ? name.lastIndexOf(":") : -1;
+    const repository = tagAt < 0 ? name : name.slice(0, tagAt);
+    if (!/^[a-z0-9.-]+(?::[0-9]+)?\/[a-z0-9]+(?:(?:[._]|__|[-]+)[a-z0-9]+)*(?:\/[a-z0-9]+(?:(?:[._]|__|[-]+)[a-z0-9]+)*)*$/.test(repository)
+        || (tagAt >= 0 && !/^[\w][\w.-]{0,127}$/.test(name.slice(tagAt + 1)))
+        || (result.includes("@") && (!digest || !/^sha256:[a-f0-9]{64}$/.test(digest) || result.split("@").length !== 2))) {
+        throw new Error(`invalid image reference: ${reference}`);
+    }
+    if (tagAt < 0 && !digest)
+        result += ":latest";
+    return result;
+}
+export function kubernetesImages(images) {
+    if (!Array.isArray(images))
+        throw new Error("k3s(): images must be an array of registry references or Dockerfile builds");
+    const pulls = [];
+    const builds = [];
+    const seen = new Map();
+    for (const image of images) {
+        if (typeof image !== "string" && (!image || typeof image !== "object" || "type" in image || !image.reference)) {
+            throw new Error("k3s(): images expects registry strings or Dockerfile builds with a reference");
+        }
+        const reference = canonicalImageReference(typeof image === "string" ? image : image.reference);
+        if (typeof image !== "string" && reference.includes("@"))
+            throw new Error("k3s(): a build reference must be an image name/tag, not a digest");
+        const identity = JSON.stringify(image);
+        const previous = seen.get(reference);
+        if (previous !== undefined) {
+            if (previous !== identity)
+                throw new Error(`k3s(): conflicting images for ${reference}`);
+            continue;
+        }
+        seen.set(reference, identity);
+        if (typeof image === "string")
+            pulls.push(image);
+        else
+            builds.push({ reference, image });
+    }
+    return { pulls, builds };
+}
+/** Stable internal task names, separate from the service graph. */
+export function componentBuildTasks(services) {
+    return services.flatMap(svc => (svc.imageBuilds ?? []).map(build => ({
+        name: `spectest-image-${createHash("sha256").update(`${svc.name}\0${build.reference}`).digest("hex")}`,
+        owner: svc.name,
+        reference: build.reference,
+        image: build.image,
+    })));
+}
+/** Private handoff to a component's setup hook. Builds stay generic; the
+ * component decides how its runtime consumes them and establishes trust. */
+export const COMPONENT_RUNTIME = Symbol.for("spectest.component-runtime");
+export function componentRuntime(context) {
+    return context[COMPONENT_RUNTIME];
+}

package/dist/harness/k3s-image-loader.d.ts

@@ -0,0 +1,5 @@
+import type { SpectestContext } from "../index.js";
+export declare function loadK3sBuildImages(service: string, images: readonly {
+    source: string;
+    reference: string;
+}[], gateway: string, exec: SpectestContext["exec"]): Promise<void>;

package/dist/harness/k3s-image-loader.js

@@ -0,0 +1,150 @@
+/** A short-lived, read-only OCI endpoint over the host containerd content store.
+ * k3s fetches only missing blobs and unpacks only missing snapshots. No Docker
+ * archive, staging registry, or second persistent copy of application blobs. */
+import { execFile, spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { createServer } from "node:http";
+import { promisify } from "node:util";
+import { canonicalImageReference } from "./component-images.js";
+import { waitForK3sRuntime } from "./k3s-prepull.js";
+const digest = (bytes) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+const descriptor = (bytes, mediaType) => ({ digest: digest(bytes), size: bytes.length, mediaType });
+const run = promisify(execFile);
+const HOST_CTR = ["--address", "/run/containerd/containerd.sock", "--namespace", "moby"];
+async function hostContent(id) {
+    if (!/^sha256:[a-f0-9]{64}$/.test(id))
+        throw new Error(`invalid content digest ${id}`);
+    const result = await run("ctr", [...HOST_CTR, "content", "get", id], { encoding: "buffer", maxBuffer: 16 * 1024 * 1024, timeout: 30_000 });
+    if (digest(result.stdout) !== id)
+        throw new Error(`corrupt image metadata ${id}`);
+    return result.stdout;
+}
+/** Preserve the original index/manifest bytes and all application descriptors. */
+async function hostImage(source) {
+    const name = canonicalImageReference(source);
+    const { stdout } = await run("ctr", [...HOST_CTR, "images", "ls", `name==${name}`], { timeout: 30_000 });
+    const found = stdout.split("\n").map(line => line.trim().split(/\s+/)).find(row => row[0] === name)?.[2];
+    if (!found)
+        throw new Error(`built image ${name} is missing from the VM container store`);
+    const metadata = new Map();
+    const allowed = new Map();
+    let configDigest;
+    async function visit(id, selected, depth = 0) {
+        if (depth > 4)
+            throw new Error(`built image ${name} has a recursive manifest index`);
+        const bytes = await hostContent(id);
+        const obj = JSON.parse(bytes.toString());
+        const desc = descriptor(bytes, obj.mediaType);
+        metadata.set(id, bytes);
+        allowed.set(id, desc);
+        if (obj.config && Array.isArray(obj.layers)) {
+            const manifest = obj;
+            for (const blob of [manifest.config, ...manifest.layers])
+                allowed.set(blob.digest, blob);
+            if (selected)
+                configDigest = manifest.config.digest;
+        }
+        else if (Array.isArray(obj.manifests)) {
+            for (const entry of obj.manifests) {
+                const matches = !entry.platform || (entry.platform.os === "linux" && entry.platform.architecture === (process.arch === "arm64" ? "arm64" : "amd64"));
+                await visit(entry.digest, selected && matches, depth + 1);
+            }
+        }
+        else
+            throw new Error(`built image ${name} has invalid image metadata`);
+        return desc;
+    }
+    const target = await visit(found, true);
+    if (!configDigest)
+        throw new Error(`built image ${name} has no image for this VM's platform`);
+    return { target, metadata, allowed, configDigest };
+}
+export async function loadK3sBuildImages(service, images, gateway, exec) {
+    if (!images.length)
+        return;
+    await waitForK3sRuntime(service, exec);
+    const checked = async (args, timeoutMs = 30_000) => {
+        const result = await exec(service, args, { timeoutMs });
+        if (result.exitCode !== 0)
+            throw new Error(`k3s(${service}): image load failed: ${result.stderr || result.stdout}`);
+        return result.stdout;
+    };
+    const config = await checked(["cat", "/var/lib/rancher/k3s/agent/etc/containerd/config.toml"]);
+    const snapshotter = config.match(/^\s*snapshotter\s*=\s*['"]([^'"]+)['"]/m)?.[1];
+    if (!snapshotter)
+        throw new Error(`k3s(${service}): cannot determine kubelet's snapshotter`);
+    const ctr = ["ctr", "--address", "/run/k3s/containerd/containerd.sock", "--namespace", "k8s.io"];
+    const help = await checked([...ctr, "images", "pull", "--help"]);
+    const local = help.includes("--local") ? ["--local"] : [];
+    for (const image of images) {
+        const reference = canonicalImageReference(image.reference);
+        const { target, metadata: blobs, allowed, configDigest } = await hostImage(image.source);
+        const streams = new Set();
+        let transferred = 0;
+        const server = createServer((req, res) => {
+            const pathname = new URL(req.url ?? "/", "http://local").pathname;
+            const match = pathname.match(/^\/v2\/.+\/(manifests|blobs)\/([^/]+)$/);
+            if (!["GET", "HEAD"].includes(req.method ?? "") || !match) {
+                res.writeHead(404).end();
+                return;
+            }
+            const id = match[1] === "manifests" && !match[2].startsWith("sha256:") ? target.digest : match[2];
+            const desc = allowed.get(id);
+            if (!desc) {
+                res.writeHead(404).end();
+                return;
+            }
+            res.writeHead(200, { "Content-Type": desc.mediaType, "Content-Length": desc.size, "Docker-Content-Digest": id });
+            if (req.method === "HEAD") {
+                res.end();
+                return;
+            }
+            transferred += desc.size;
+            const bytes = blobs.get(id);
+            if (bytes) {
+                res.end(bytes);
+                return;
+            }
+            const proc = spawn("ctr", [...HOST_CTR, "content", "get", id], { stdio: ["ignore", "pipe", "ignore"] });
+            streams.add(proc);
+            proc.stdout.pipe(res);
+            proc.on("error", err => res.destroy(err));
+            proc.on("exit", code => { streams.delete(proc); if (code !== 0)
+                res.destroy(new Error(`content read failed: ${id}`)); });
+            res.on("close", () => { if (proc.exitCode === null)
+                proc.kill(); });
+        });
+        const directory = `/run/spectest-image-load-${createHash("sha256").update(reference).digest("hex").slice(0, 16)}`;
+        try {
+            await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, gateway, resolve); });
+            const port = server.address().port;
+            // A private hosts directory scopes this override to this single ctr call;
+            // it cannot change the cluster's configured registries or pull policy.
+            const hosts = `server = "http://${gateway}:${port}"\n`;
+            await checked(["sh", "-c", 'mkdir -p "$1/_default"; printf "%s" "$2" > "$1/_default/hosts.toml"', "sh", directory, hosts]);
+            await checked([...ctr, "images", "pull", ...local, "--hosts-dir", directory,
+                "--snapshotter", snapshotter, "--label", "io.cri-containerd.image=managed", reference], 600_000);
+            const repo = reference.slice(0, reference.lastIndexOf(":"));
+            await checked([...ctr, "images", "tag", "--force", reference, `${repo}@${target.digest}`]);
+            // CRI learns about ctr image updates asynchronously. Confirm the current
+            // build's config, not merely that an older tag already exists.
+            const deadline = Date.now() + 30_000;
+            for (;;) {
+                const result = await exec(service, ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock", "inspecti", reference], { timeoutMs: 5_000 });
+                if (result.exitCode === 0 && JSON.parse(result.stdout).status?.id === configDigest)
+                    break;
+                if (Date.now() >= deadline)
+                    throw new Error(`k3s(${service}): current build is not visible to kubelet: ${reference}`);
+                await new Promise(resolve => setTimeout(resolve, 100));
+            }
+            console.log(`[k3s-images] ${service}: ${reference} @ ${target.digest}, transferred ${transferred} bytes`);
+        }
+        finally {
+            for (const proc of streams)
+                proc.kill();
+            server.closeAllConnections();
+            await new Promise(resolve => server.close(() => resolve()));
+            await exec(service, ["rm", "-rf", directory], { timeoutMs: 5_000 }).catch(() => undefined);
+        }
+    }
+}

package/dist/harness/k3s-prepull.d.ts

@@ -0,0 +1,7 @@
+import type { SpectestContext } from "../index.js";
+/** CRI PullImage waits for unpacking and uses k3s's registry configuration.
+ * Always resolve tags on a fresh setup. Immutable digests can be used offline.
+ * Drain both workers on failure so no pulls outlive the setup hook.
+ */
+export declare function prePullK3sImages(name: string, images: readonly string[], exec: SpectestContext["exec"]): Promise<void>;
+export declare function waitForK3sRuntime(name: string, exec: SpectestContext["exec"]): Promise<void>;

package/dist/harness/k3s-prepull.js

@@ -0,0 +1,39 @@
+/** CRI PullImage waits for unpacking and uses k3s's registry configuration.
+ * Always resolve tags on a fresh setup. Immutable digests can be used offline.
+ * Drain both workers on failure so no pulls outlive the setup hook.
+ */
+export async function prePullK3sImages(name, images, exec) {
+    if (!images.length)
+        return;
+    await waitForK3sRuntime(name, exec);
+    const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
+    const queue = [...images];
+    const results = await Promise.allSettled(Array.from({ length: Math.min(2, queue.length) }, async () => {
+        for (let image = queue.shift(); image !== undefined; image = queue.shift()) {
+            if (/@sha256:[a-f0-9]{64}$/.test(image)) {
+                const existing = await exec(name, [...cri, "inspecti", image], { timeoutMs: 30_000 });
+                if (existing.exitCode === 0)
+                    continue;
+            }
+            const result = await exec(name, [...cri, "pull", image], { timeoutMs: 600_000 });
+            if (result.exitCode !== 0)
+                throw new Error(`k3s(${name}): pre-pull failed for ${image}: ${result.stderr || result.stdout}`);
+        }
+    }));
+    const failed = results.find(result => result.status === "rejected");
+    if (failed?.status === "rejected")
+        throw failed.reason;
+}
+export async function waitForK3sRuntime(name, exec) {
+    const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
+    // /readyz belongs to the API server and can precede the node runtime.
+    const deadline = Date.now() + 60_000;
+    for (;;) {
+        const result = await exec(name, [...cri, "info"], { timeoutMs: 5_000 });
+        if (result.exitCode === 0)
+            break;
+        if (Date.now() >= deadline)
+            throw new Error(`k3s(${name}): containerd did not become ready for pre-pulls: ${result.stderr || result.stdout}`);
+        await new Promise(resolve => setTimeout(resolve, 250));
+    }
+}

package/dist/harness/sticky-consumer.d.ts

@@ -1,6 +1,7 @@
+import { type KubernetesCacheMaintenance } from "./sticky-kubernetes.js";
 export declare const STICKY_CAPABILITY = "sticky-disks-v1";
 export declare const STICKY_COMPATIBILITY = "docker29-ctr21-buildkit032-overlayfs-256g-v2";
-export declare const STICKY_RUNTIME = "/run/spectest-sticky";
+export { STICKY_RUNTIME } from "./sticky-paths.js";
 export interface StickyConsumerConfig {
     path: string;
     capacityBytes: number;
@@ -19,10 +20,9 @@ export declare function initializeStickyConsumer(config: StickyConsumerConfig):
 export declare function collectStickyCache(config: StickyConsumerConfig, run?: typeof exec, usage?: () => Promise<{
     freeBytes: number;
     totalBytes: number;
-}>): Promise<void>;
+}>, kubernetes?: KubernetesCacheMaintenance): Promise<void>;
 /** Only on an isolated maintenance clone; never on the live ready environment. */
 export declare function prepareStickyConsumer(artifact?: boolean): Promise<{
     prepared: true;
     freeBytes: number;
 }>;
-export {};

package/dist/harness/sticky-consumer.js

@@ -1,9 +1,11 @@
-/** Docker/BuildKit policy for the generic sticky disk. No host publication API. */
+/** Docker, BuildKit and nested Kubernetes policy for the generic sticky disk. */
 import { execFile } from "node:child_process";
 import { mkdir, readFile, statfs, writeFile } from "node:fs/promises";
+import { STICKY_RUNTIME } from "./sticky-paths.js";
+import { openKubernetesCacheMaintenance } from "./sticky-kubernetes.js";
 export const STICKY_CAPABILITY = "sticky-disks-v1";
 export const STICKY_COMPATIBILITY = "docker29-ctr21-buildkit032-overlayfs-256g-v2";
-export const STICKY_RUNTIME = "/run/spectest-sticky";
+export { STICKY_RUNTIME } from "./sticky-paths.js";
 function exec(program, args, timeout = 60_000) {
     return new Promise((resolve, reject) => {
         execFile(program, args, { timeout, maxBuffer: 8 * 1024 * 1024 }, (error, stdout, stderr) => {
@@ -110,7 +112,7 @@ disabled_plugins = ["io.containerd.grpc.v1.cri"]
 export async function collectStickyCache(config, run = exec, usage = async () => {
     const fs = await statfs(config.path);
     return { freeBytes: fs.bavail * fs.bsize, totalBytes: fs.blocks * fs.bsize };
-}) {
+}, kubernetes) {
     validateStickyConfig(config);
     const buildctl = ["--addr", "tcp://127.0.0.1:1234", "prune"];
     await run("buildctl", [...buildctl, "--keep-duration", "168h"], 120_000);
@@ -128,6 +130,11 @@ export async function collectStickyCache(config, run = exec, usage = async () =>
     await pruneBuildkit();
     if (withinTarget(fs))
         return;
+    while (await kubernetes?.pruneOldest()) {
+        fs = await usage();
+        if (withinTarget(fs))
+            return;
+    }
     // Pressure can be entirely in recent Docker images. Drop oldest images
     // first, even within the age window, until the whole filesystem fits.
     const ids = [...new Set((await run("docker", ["image", "ls", "-aq", "--no-trunc"])).split(/\s+/).filter(Boolean))];
@@ -135,6 +142,8 @@ export async function collectStickyCache(config, run = exec, usage = async () =>
         ? JSON.parse(await run("docker", ["image", "inspect", ...ids])) : [];
     images.sort((a, b) => a.Created.localeCompare(b.Created) || a.Id.localeCompare(b.Id));
     for (const image of images) {
+        if (kubernetes?.protectedImages.has(image.Id))
+            continue;
         await run("docker", ["image", "rm", "-f", image.Id], 120_000);
         await pruneBuildkit();
         if (withinTarget(fs))
@@ -157,7 +166,13 @@ export async function prepareStickyConsumer(artifact = false) {
         const volumes = (await exec("docker", ["volume", "ls", "-q"])).split(/\s+/).filter(Boolean);
         if (volumes.length)
             await exec("docker", ["volume", "rm", ...volumes]);
-        await collectStickyCache(config);
+        const kubernetes = await openKubernetesCacheMaintenance(exec);
+        try {
+            await collectStickyCache(config, exec, undefined, kubernetes);
+        }
+        finally {
+            await kubernetes.close();
+        }
     }
     const units = ["spectest-sticky-buildkit", "spectest-sticky-docker", "spectest-sticky-containerd"];
     for (const unit of units) {

package/dist/harness/sticky-kubernetes.d.ts

@@ -0,0 +1,15 @@
+export type CacheCommand = (program: string, args: string[], timeout?: number) => Promise<string>;
+export declare function stickyKubernetesDirectory(service: string, version: string): string;
+export declare function registerStickyKubernetesStore(service: string, version: string, run?: CacheCommand): Promise<void>;
+/** Standalone containerd never starts CRI/kubelet or any inherited workload.
+ * Native API deletion preserves content and committed image snapshots while
+ * releasing sandbox, container, lease and writable snapshot references. */
+export declare const KUBERNETES_CACHE_MAINTENANCE_SH = "set -eu\nexec 2>&1\ncat > /run/cache.toml <<'CONFIG'\nversion = 2\ndisabled_plugins = [\"io.containerd.grpc.v1.cri\", \"io.containerd.cri.v1.runtime\", \"io.containerd.cri.v1.images\"]\nCONFIG\ncontainerd --config /run/cache.toml --root /var/lib/rancher/k3s/agent/containerd --state /run/cache-state --address /run/spectest-cache.sock > /run/cache.log 2>&1 &\npid=$!\nstop() { kill \"$pid\"; wait \"$pid\"; }\ntrap 'stop; exit 0' TERM INT\ntrap 'kill \"$pid\" 2>/dev/null || true' EXIT\nc() { ctr --address /run/spectest-cache.sock --namespace k8s.io \"$@\"; }\ni=0\nuntil c version >/dev/null 2>&1; do\n  kill -0 \"$pid\" || { cat /run/cache.log; exit 1; }\n  i=$((i+1)); [ \"$i\" -lt 300 ] || { cat /run/cache.log; exit 1; }\n  sleep 0.1\ndone\nids=$(c containers ls -q)\nif [ -n \"$ids\" ]; then c containers rm $ids; fi\n# 1.7 has no sandbox store; 2.x does.\ncase \"$(containerd --version)\" in\n  *v1.*) ;;\n  *) sandboxes=$(c sandboxes ls)\n     ids=$(printf '%s\\n' \"$sandboxes\" | awk 'NR > 1 {print $1}')\n     if [ -n \"$ids\" ]; then c sandboxes rm --force $ids; fi ;;\nesac\nids=$(c leases ls -q)\nif [ -n \"$ids\" ]; then c leases rm $ids; fi\n# Remove children before parents, and tolerate asynchronous GC winning a race.\nprevious=''\nwhile :; do\n  snapshots=$(c snapshots --snapshotter overlayfs ls)\n  ids=$(printf '%s\\n' \"$snapshots\" | awk 'NR > 1 && $NF != \"Committed\" {print $1}')\n  [ -n \"$ids\" ] || break\n  [ \"$ids\" != \"$previous\" ] || { echo 'cannot clean Kubernetes writable snapshots' >&2; exit 1; }\n  previous=\"$ids\"\n  for id in $ids; do c snapshots --snapshotter overlayfs rm \"$id\" >/dev/null 2>&1 || true; done\ndone\ntouch /run/cache-ready\nwhile :; do sleep 1 & wait $!; done\n";
+export interface KubernetesCacheMaintenance {
+    /** Outer images needed by live maintenance containers must not be evicted. */
+    protectedImages: Set<string>;
+    /** Evicts one complete image store, oldest service/version first. */
+    pruneOldest(): Promise<boolean>;
+    close(): Promise<void>;
+}
+export declare function openKubernetesCacheMaintenance(run: CacheCommand, root?: string): Promise<KubernetesCacheMaintenance>;

package/dist/harness/sticky-kubernetes.js

@@ -0,0 +1,163 @@
+/** Kubernetes owns a separate containerd store; Docker/BuildKit GC cannot
+ * collect it. Maintenance runs only on the isolated publication clone. */
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, readFile, readdir, rename, writeFile } from "node:fs/promises";
+import { STICKY_ROOT } from "./sticky-paths.js";
+const STORE_TARGET = "/var/lib/rancher/k3s/agent/containerd";
+const SOCKET = "/run/spectest-cache.sock";
+const ROOT = `${STICKY_ROOT}/kubernetes`;
+const IMAGE_ID = /^sha256:[a-f0-9]{64}$/;
+const SERVICE_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/;
+export function stickyKubernetesDirectory(service, version) {
+    // Version changes must not open old metadata with a different containerd.
+    const layout = createHash("sha256").update(`overlayfs-v1:${version}`).digest("hex").slice(0, 16);
+    return `${ROOT}/${service}/${layout}`;
+}
+const runCommand = async (program, args, timeout = 30_000) => (await promisify(execFile)(program, args, { timeout })).stdout.trim();
+export async function registerStickyKubernetesStore(service, version, run = runCommand) {
+    if (!SERVICE_NAME.test(service))
+        throw new Error(`invalid Kubernetes cache service name: ${service}`);
+    // Respect custom Dockerfile overrides of the component's image.
+    const image = (await run("docker", ["inspect", "--format", "{{.Image}}", service])).trim();
+    if (!IMAGE_ID.test(image))
+        throw new Error(`cannot identify Kubernetes runtime image for ${service}`);
+    const directory = stickyKubernetesDirectory(service, version);
+    await mkdir(directory, { recursive: true });
+    const temporary = `${directory}/store.${randomUUID()}.tmp`;
+    await writeFile(temporary, JSON.stringify({ schema: 1, image, updatedAt: Date.now() }));
+    await rename(temporary, `${directory}/store.json`);
+}
+/** Standalone containerd never starts CRI/kubelet or any inherited workload.
+ * Native API deletion preserves content and committed image snapshots while
+ * releasing sandbox, container, lease and writable snapshot references. */
+export const KUBERNETES_CACHE_MAINTENANCE_SH = `set -eu
+exec 2>&1
+cat > /run/cache.toml <<'CONFIG'
+version = 2
+disabled_plugins = ["io.containerd.grpc.v1.cri", "io.containerd.cri.v1.runtime", "io.containerd.cri.v1.images"]
+CONFIG
+containerd --config /run/cache.toml --root ${STORE_TARGET} --state /run/cache-state --address ${SOCKET} > /run/cache.log 2>&1 &
+pid=$!
+stop() { kill "$pid"; wait "$pid"; }
+trap 'stop; exit 0' TERM INT
+trap 'kill "$pid" 2>/dev/null || true' EXIT
+c() { ctr --address ${SOCKET} --namespace k8s.io "$@"; }
+i=0
+until c version >/dev/null 2>&1; do
+  kill -0 "$pid" || { cat /run/cache.log; exit 1; }
+  i=$((i+1)); [ "$i" -lt 300 ] || { cat /run/cache.log; exit 1; }
+  sleep 0.1
+done
+ids=$(c containers ls -q)
+if [ -n "$ids" ]; then c containers rm $ids; fi
+# 1.7 has no sandbox store; 2.x does.
+case "$(containerd --version)" in
+  *v1.*) ;;
+  *) sandboxes=$(c sandboxes ls)
+     ids=$(printf '%s\\n' "$sandboxes" | awk 'NR > 1 {print $1}')
+     if [ -n "$ids" ]; then c sandboxes rm --force $ids; fi ;;
+esac
+ids=$(c leases ls -q)
+if [ -n "$ids" ]; then c leases rm $ids; fi
+# Remove children before parents, and tolerate asynchronous GC winning a race.
+previous=''
+while :; do
+  snapshots=$(c snapshots --snapshotter overlayfs ls)
+  ids=$(printf '%s\\n' "$snapshots" | awk 'NR > 1 && $NF != "Committed" {print $1}')
+  [ -n "$ids" ] || break
+  [ "$ids" != "$previous" ] || { echo 'cannot clean Kubernetes writable snapshots' >&2; exit 1; }
+  previous="$ids"
+  for id in $ids; do c snapshots --snapshotter overlayfs rm "$id" >/dev/null 2>&1 || true; done
+done
+touch /run/cache-ready
+while :; do sleep 1 & wait $!; done
+`;
+export async function openKubernetesCacheMaintenance(run, root = ROOT) {
+    const stores = [];
+    const protectedImages = new Set();
+    const closeStore = async (store) => {
+        if (!store.active)
+            return;
+        await run("docker", ["stop", "--time", "30", store.container], 45_000);
+        const status = JSON.parse(await run("docker", ["inspect", "--format", "{{json .State}}", store.container]));
+        if (status.ExitCode !== 0 || status.OOMKilled)
+            throw new Error(`unclean Kubernetes cache shutdown: ${store.container}`);
+        await run("docker", ["rm", store.container]);
+        store.active = false;
+        if (!stores.some(other => other.active && other.descriptor.image === store.descriptor.image))
+            protectedImages.delete(store.descriptor.image);
+        // These contain runtime state, not image content or unpacked layers.
+        await run("rm", ["-rf", ...["io.containerd.runtime.v2.task", "io.containerd.grpc.v1.cri", "io.containerd.cri.v1.runtime", "tmpmounts"]
+                .map(child => `${store.directory}/containerd/${child}`)]);
+    };
+    const close = async () => {
+        const failures = [];
+        for (const store of stores) {
+            try {
+                await closeStore(store);
+            }
+            catch (error) {
+                failures.push(error);
+            }
+        }
+        if (failures.length)
+            throw new AggregateError(failures, "Kubernetes cache shutdown failed");
+    };
+    const entries = await readdir(root, { withFileTypes: true }).catch((error) => {
+        if (error.code === "ENOENT")
+            return [];
+        throw error;
+    });
+    try {
+        for (const service of entries) {
+            if (!service.isDirectory() || !SERVICE_NAME.test(service.name))
+                continue;
+            for (const layout of await readdir(`${root}/${service.name}`, { withFileTypes: true })) {
+                if (!layout.isDirectory() || !/^[a-f0-9]{16}$/.test(layout.name))
+                    continue;
+                const directory = `${root}/${service.name}/${layout.name}`;
+                const descriptor = JSON.parse(await readFile(`${directory}/store.json`, "utf8"));
+                if (descriptor.schema !== 1 || !IMAGE_ID.test(descriptor.image) || !Number.isSafeInteger(descriptor.updatedAt)) {
+                    throw new Error(`invalid Kubernetes cache descriptor: ${directory}`);
+                }
+                const container = `spectest-k8s-cache-${randomUUID()}`;
+                await run("docker", ["run", "-d", "--name", container, "--network", "none", "--privileged", "--stop-signal", "SIGTERM", "--tmpfs", "/run", "--tmpfs", "/var/run",
+                    "--mount", `type=bind,source=${directory}/containerd,target=${STORE_TARGET}`,
+                    "--entrypoint", "/bin/sh", descriptor.image, "-c", KUBERNETES_CACHE_MAINTENANCE_SH]);
+                stores.push({ directory, descriptor, container, active: true });
+                protectedImages.add(descriptor.image);
+                try {
+                    await run("docker", ["exec", container, "sh", "-c", "i=0; until [ -f /run/cache-ready ]; do i=$((i+1)); [ $i -lt 900 ] || exit 1; sleep 0.1; done"], 100_000);
+                }
+                catch (error) {
+                    const logs = await run("docker", ["logs", "--tail", "40", container]).catch(() => "");
+                    throw new Error(`Kubernetes cache preparation failed for ${directory}: ${logs}`, { cause: error });
+                }
+            }
+        }
+        stores.sort((a, b) => a.descriptor.updatedAt - b.descriptor.updatedAt || a.directory.localeCompare(b.directory));
+        let next = 0;
+        const pruneOldest = async () => {
+            const store = stores[next++];
+            if (!store)
+                return false;
+            await run("docker", ["exec", store.container, "sh", "-ec",
+                `ids=$(ctr -a ${SOCKET} -n k8s.io images ls -q); if [ -n "$ids" ]; then ctr -a ${SOCKET} -n k8s.io images rm --sync $ids; fi`], 120_000);
+            await closeStore(store);
+            // No writer remains. Discard the now-empty cache and its descriptor so
+            // obsolete k3s binaries can also be collected by the outer Docker GC.
+            await run("rm", ["-rf", store.directory]);
+            return true;
+        };
+        // Abandoned services/versions should not occupy the disk indefinitely.
+        while (stores[next] && stores[next].descriptor.updatedAt < Date.now() - 7 * 24 * 60 * 60 * 1000)
+            await pruneOldest();
+        return { protectedImages, pruneOldest, close };
+    }
+    catch (error) {
+        await close().catch(() => { });
+        throw error;
+    }
+}

package/dist/harness/sticky-paths.d.ts

@@ -0,0 +1,5 @@
+export declare const STICKY_ROOT = "/var/lib/spectest-sticky";
+export declare const STICKY_RUNTIME = "/run/spectest-sticky";
+/** Written only after the supervisor mounts this project's private disk. */
+export declare function stickyDiskPathSync(manifest?: string): string | null;
+export declare function isOnStickyDisk(path: string, root?: string | null): boolean;

package/dist/harness/sticky-paths.js

@@ -0,0 +1,18 @@
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+export const STICKY_ROOT = "/var/lib/spectest-sticky";
+export const STICKY_RUNTIME = "/run/spectest-sticky";
+/** Written only after the supervisor mounts this project's private disk. */
+export function stickyDiskPathSync(manifest = `${STICKY_RUNTIME}/consumer.json`) {
+    try {
+        const config = JSON.parse(readFileSync(manifest, "utf8"));
+        return config.path === STICKY_ROOT ? STICKY_ROOT : null;
+    }
+    catch {
+        return null;
+    }
+}
+export function isOnStickyDisk(path, root = stickyDiskPathSync()) {
+    const absolute = resolve(path);
+    return root !== null && (absolute === root || absolute.startsWith(`${root}/`));
+}

package/dist/index.d.ts

@@ -74,6 +74,8 @@ export interface EnvironmentConfig<S extends ServicesMap = ServicesMap> {
 export interface ServiceConfig {
     /** Image definition: registry pull or inline build. */
     image: ServiceImage;
+    /** @internal Images built by the harness and loaded into this component. */
+    imageBuilds?: readonly ComponentImageBuild[];
     /**
      * Shell command to run (sh -c). Defaults to the image's CMD. NB: this
      * **replaces the image's entrypoint** with `/bin/sh -c` — an
@@ -741,11 +743,14 @@ export interface ServiceTls {
      * daemon forwards proxied requests here over `spectest-net`. */
     port: number;
 }
-export type ServiceImage = {
+/** @internal Normalized wire representation; legacy definitions remain accepted. */
+export type NormalizedServiceImage = {
     type: "registry";
     reference: string;
 } | {
     type: "dockerfile";
+    /** Optional full local image reference. */
+    reference?: string;
     /**
      * Path of a Dockerfile in your repo, relative to the project root
      * (where `spectest/` lives) — `"Dockerfile"`, `"apps/api/Dockerfile"`.
@@ -787,6 +792,8 @@ export type ServiceImage = {
     buildArgs?: Readonly<Record<string, string>>;
 } | {
     type: "dockerfile";
+    /** Optional full local image reference. */
+    reference?: string;
     /**
      * Dockerfile contents, built as if the file sat outside the build
      * context (it is never written into your repo). The context is the
@@ -815,6 +822,29 @@ export type ServiceImage = {
      */
     buildArgs?: Readonly<Record<string, string>>;
 };
+/** Dockerfile build options shared by services and Kubernetes images. */
+export type BuildImage = (Omit<Extract<NormalizedServiceImage, {
+    type: "dockerfile";
+    path: string;
+}>, "type"> | Omit<Extract<NormalizedServiceImage, {
+    type: "dockerfile";
+    content: string;
+}>, "type">) & {
+    /** Full image reference, e.g. "k8s.internal:5000/runtime:spectest".
+     * Optional for a service; required for a Kubernetes build. This names a
+     * local image and does not publish it to a registry. */
+    reference?: string;
+};
+/** A registry reference or Dockerfile build. Tagged objects from older SDKs
+ * remain accepted for compatibility; new definitions do not need `type`. */
+export type ServiceImage = string | BuildImage | NormalizedServiceImage;
+/** @internal Keep the wire format compatible with existing control planes. */
+export declare function normalizeServiceImage(image: ServiceImage): NormalizedServiceImage;
+/** @internal Serializes build-only work with the owning component. */
+export interface ComponentImageBuild {
+    reference: string;
+    image: BuildImage;
+}
 /**
  * Check a service's `image` at config time, so a mistake fails the
  * environment load with a message that names it, instead of a build
@@ -831,7 +861,7 @@ export type ServiceImage = {
  * checks go through the project-file resolver, so the same rules as
  * `ctx.readProjectFile` apply.
  */
-export declare function validateServiceImage(serviceName: string, image: ServiceImage): ServiceImage;
+export declare function validateServiceImage(serviceName: string, input: ServiceImage): NormalizedServiceImage;
 export interface VolumeMount {
     /**
      * Named shared volume. Two services mounting the same `name` share one