@specific.dev/spectest 0.90.3 → 0.91.1

package/src/components/k3s-trust.ts

@@ -0,0 +1,135 @@
+/** Pod trust belongs to the cluster. Admission adds read-only ConfigMap
+ * mounts and environment defaults, without changing any image bytes. */
+import { createHash, randomUUID } from "node:crypto";
+import { createServer, type Server } from "node:https";
+
+export const POD_CA_PATH = "/spectest/ca-bundle.crt";
+const TRUST_ENV: Readonly<Record<string, string>> = {
+  NODE_EXTRA_CA_CERTS: POD_CA_PATH, SSL_CERT_FILE: POD_CA_PATH,
+  REQUESTS_CA_BUNDLE: POD_CA_PATH, AWS_CA_BUNDLE: POD_CA_PATH,
+  PGSSLROOTCERT: POD_CA_PATH, DENO_TLS_CA_STORE: "mozilla,system",
+};
+const MOUNT_PATHS = [POD_CA_PATH, "/etc/ssl/certs/ca-certificates.crt", "/etc/pki/tls/certs/ca-bundle.crt"];
+interface Container {
+  env?: { name: string; value?: string; valueFrom?: unknown }[];
+  volumeMounts?: { name: string; mountPath: string; subPath?: string; readOnly?: boolean }[];
+}
+interface Pod {
+  metadata?: { annotations?: Record<string, string> };
+  spec?: {
+    os?: { name: string };
+    volumes?: { name: string; configMap?: { name: string }; [key: string]: unknown }[];
+    containers?: Container[]; initContainers?: Container[]; ephemeralContainers?: Container[];
+  };
+}
+interface Patch { op: "add"; path: string; value: unknown }
+export const TRUST_OPT_OUT = "spectest.dev/inject-ca";
+
+export function podTrustPatch(pod: Pod, configMap: string, ephemeral = false): Patch[] {
+  if (!pod.spec || pod.spec.os?.name === "windows" || pod.metadata?.annotations?.[TRUST_OPT_OUT] === "false") return [];
+  const patches: Patch[] = [];
+  const volumes = pod.spec.volumes ?? [];
+  // Reuse our immutable ConfigMap, but never overwrite a user's volume.
+  let volume = volumes.find(v => v.configMap?.name === configMap)?.name;
+  if (!volume && ephemeral) return []; // Pod volumes cannot change on this subresource.
+  if (!volume) {
+    const names = new Set(volumes.map(v => v.name));
+    volume = "spectest-ca";
+    for (let n = 1; names.has(volume); n++) volume = `spectest-ca-${n}`;
+    const value = { name: volume, configMap: { name: configMap, defaultMode: 0o444, items: [{ key: "ca-bundle.crt", path: "ca-bundle.crt" }] } };
+    patches.push(pod.spec.volumes
+      ? { op: "add", path: "/spec/volumes/-", value }
+      : { op: "add", path: "/spec/volumes", value: [value] });
+  }
+  const groups = ephemeral ? ["ephemeralContainers"] as const : ["containers", "initContainers"] as const;
+  for (const group of groups) {
+    for (const [index, container] of (pod.spec[group] ?? []).entries()) {
+      const prefix = `/spec/${group}/${index}`;
+      const env = container.env ?? [];
+      const names = new Set(env.map(e => e.name));
+      const extraEnv = Object.entries(TRUST_ENV).filter(([name]) => !names.has(name)).map(([name, value]) => ({ name, value }));
+      const mounts = container.volumeMounts ?? [];
+      // A user mount at a path (or a parent directory) owns that path.
+      const extraMounts = MOUNT_PATHS.filter(path => !mounts.some(m => path === m.mountPath || path.startsWith(m.mountPath.replace(/\/$/, "") + "/")))
+        .map(mountPath => ({ name: volume!, mountPath, subPath: "ca-bundle.crt", readOnly: true }));
+      if (extraEnv.length) patches.push({ op: "add", path: `${prefix}/env`, value: [...env, ...extraEnv] });
+      if (extraMounts.length) patches.push({ op: "add", path: `${prefix}/volumeMounts`, value: [...mounts, ...extraMounts] });
+    }
+  }
+  return patches;
+}
+
+interface AdmissionRequest {
+  uid: string; namespace?: string; dryRun?: boolean; subResource?: string;
+  resource?: { resource: string }; object?: Pod;
+}
+export interface TrustDependencies {
+  /** Ensure this immutable ConfigMap exists in the requesting namespace. */
+  ensureBundle(namespace: string, name: string, bundle: string): Promise<void>;
+  apply(configuration: unknown): Promise<void>;
+}
+const servers = new Map<string, Server>();
+
+export async function installPodTrust(options: {
+  name: string; gateway: string; bundle: string; cert: string; key: string; ca: string;
+}, deps: TrustDependencies): Promise<void> {
+  const configMap = `spectest-ca-${createHash("sha256").update(options.bundle).digest("hex").slice(0, 20)}`;
+  const endpoint = `/admit/${randomUUID()}`;
+  const server = createServer({ cert: options.cert, key: options.key }, async (req, res) => {
+    if (req.method !== "POST" || new URL(req.url ?? "/", "https://local").pathname !== endpoint) { res.writeHead(404).end(); return; }
+    const chunks: Buffer[] = [];
+    let size = 0;
+    try {
+      for await (const chunk of req) {
+        size += chunk.length;
+        if (size > 2 * 1024 * 1024) { res.writeHead(413).end(); return; }
+        chunks.push(Buffer.from(chunk));
+      }
+      const review = JSON.parse(Buffer.concat(chunks).toString());
+      const request = review.request as AdmissionRequest | undefined;
+      if (!request?.uid || request.resource?.resource !== "pods" || !request.object || !request.namespace) {
+        res.writeHead(400).end("invalid Pod admission request"); return;
+      }
+      const patch = podTrustPatch(request.object, configMap, request.subResource === "ephemeralcontainers");
+      // Dry runs return the same patch but perform no API writes.
+      if (patch.length && !request.dryRun) await deps.ensureBundle(request.namespace, configMap, options.bundle);
+      res.setHeader("Content-Type", "application/json");
+      res.end(JSON.stringify({ apiVersion: "admission.k8s.io/v1", kind: "AdmissionReview", response: {
+        uid: request.uid, allowed: true,
+        ...(patch.length ? { patchType: "JSONPatch", patch: Buffer.from(JSON.stringify(patch)).toString("base64") } : {}),
+      } }));
+    } catch (err) {
+      console.error(`[k3s-trust] ${options.name}: ${err instanceof Error ? err.message : err}`);
+      res.writeHead(500).end("could not mount the environment CA");
+    }
+  });
+  server.requestTimeout = 15_000;
+  try {
+    await new Promise<void>((resolve, reject) => { server.once("error", reject); server.listen(0, options.gateway, resolve); });
+    const port = (server.address() as { port: number }).port;
+    await deps.apply({
+      apiVersion: "admissionregistration.k8s.io/v1", kind: "MutatingWebhookConfiguration",
+      metadata: { name: "spectest-pod-trust" },
+      webhooks: [{
+        name: "pod-trust.spectest.dev", admissionReviewVersions: ["v1"],
+        sideEffects: "NoneOnDryRun", failurePolicy: "Fail", timeoutSeconds: 10,
+        reinvocationPolicy: "IfNeeded", matchPolicy: "Equivalent",
+        clientConfig: { url: `https://${options.gateway}:${port}${endpoint}`, caBundle: Buffer.from(options.ca).toString("base64") },
+        rules: [{ operations: ["CREATE"], apiGroups: [""], apiVersions: ["v1"], resources: ["pods"], scope: "Namespaced" },
+          { operations: ["UPDATE"], apiGroups: [""], apiVersions: ["v1"], resources: ["pods/ephemeralcontainers"], scope: "Namespaced" }],
+      }],
+    });
+  } catch (err) {
+    server.closeAllConnections(); server.close(); throw err;
+  }
+  const previous = servers.get(options.name);
+  servers.set(options.name, server);
+  previous?.closeAllConnections(); previous?.close();
+}
+
+/** Release a component listener on replacement or in standalone tests. */
+export function stopPodTrust(name: string): void {
+  const server = servers.get(name);
+  servers.delete(name);
+  server?.closeAllConnections(); server?.close();
+}

package/src/components/k3s.ts

@@ -1,3 +1,7 @@
+import { isIP } from "node:net";
+import { installPodTrust } from "./k3s-trust.js";
+import { waitForAll } from "../harness/concurrency.js";
+import { loadK3sBuildImages } from "../harness/k3s-image-loader.js";
 import { AsyncLocalStorage } from "node:async_hooks";
 import { spawn as nodeSpawn } from "node:child_process";
 import { randomUUID } from "node:crypto";
@@ -31,6 +35,7 @@ import { Observable } from "@kubernetes/client-node/dist/gen/rxjsStub.js";
 import type {
   ServiceDefinition,
   ServiceHelpersContext,
+  ServiceSetupContext,
   SpectestContext,
 } from "../index.js";
 import { dnsName, provides, SELF_SERVICE_TOKEN } from "../index.js";
@@ -41,6 +46,11 @@ import {
   nestedStoreDir,
   type ImageCachePaths,
 } from "../harness/image-cache.js";
+import { stickyDiskPathSync } from "../harness/sticky-paths.js";
+import { registerStickyKubernetesStore, stickyKubernetesDirectory } from "../harness/sticky-kubernetes.js";
+import { prePullK3sImages } from "../harness/k3s-prepull.js";
+import { componentRuntime, kubernetesImages, type KubernetesImage } from "../harness/component-images.js";
+export type { KubernetesImage } from "../harness/component-images.js";
 import { deepUnwrap, readTag, wrap } from "../inspect.js";
 import type { Wrapped } from "../inspect.js";
 import { recorderAnnotate, recorderRemove } from "../recorder.js";
@@ -50,7 +60,19 @@ export type {
   KubernetesListObject,
 } from "@kubernetes/client-node";
 
+/** Kubernetes pods receive read-only environment CA mounts during admission.
+ * Image bytes and digests stay unchanged; explicit pod trust settings win. */
 export interface K3sOptions {
+  /** Images available to kubelet before project setup. Use registry strings,
+   * or Dockerfile builds with a full `reference` matching your manifests. Builds
+   * support path/content, context, exclude, target and buildArgs, and run in
+   * parallel with service builds through the standard BuildKit pipeline.
+   * Setup waits for every pull and fails if a requested image cannot be pulled.
+   * Sticky-enabled projects retain image content and unpacked layers between
+   * fresh builds automatically. Digest references can reuse the cache offline;
+   * tags are resolved again on each fresh setup. Pod imagePullPolicy still applies.
+   */
+  images?: readonly KubernetesImage[];
   /**
    * Image tag for the official `rancher/k3s` image. Default
    * `"v1.33.5-k3s1"`. Any tag works; a k3s older than 1.33 embeds a
@@ -178,8 +200,13 @@ const CRI_MANAGED_LABEL = "io.cri-containerd.image=managed";
 const BOOT_MARKER = ".spectest-boot-incomplete";
 
 /**
- * The cluster's image store, and how it shares the environment's image
- * cache (`CONTAINER_STORE.md` §12).
+ * Sticky-enabled projects bind a per-service/version directory on the
+ * sticky disk at CLUSTER_STORE_TARGET and use overlayfs. Publication's
+ * sticky-kubernetes adapter removes runtime state and retains images and
+ * unpacked layers under the shared filesystem budget. Test forks do not
+ * write back. The preparation wrapper below also handles delta rebuilds.
+ *
+ * Legacy image-cache integration (`CONTAINER_STORE.md` §12):
  *
  * The cluster's embedded containerd keeps its root on the **root disk**
  * of the environment's image cache, in a directory of its own
@@ -346,8 +373,8 @@ function storePrepareSh(cache: ImageCachePaths | null): string {
     `timeout 60 ctr -a "$A" -n k8s.io containers rm $ids >/dev/null 2>&1 || true; fi; ` +
     // containerd 2.x keeps sandboxes in a store of their own; on the 1.7
     // k3s ships they are ordinary containers and this is a no-op.
-    `sb=$(timeout 30 ctr -a "$A" -n k8s.io sandboxes ls -q 2>/dev/null); ` +
-    `[ -n "$sb" ] && timeout 60 ctr -a "$A" -n k8s.io sandboxes rm $sb >/dev/null 2>&1; return 0; }; ` +
+    `sb=$(timeout 30 ctr -a "$A" -n k8s.io sandboxes ls 2>/dev/null | awk 'NR > 1 {print $1}'); ` +
+    `[ -n "$sb" ] && timeout 60 ctr -a "$A" -n k8s.io sandboxes rm --force $sb >/dev/null 2>&1; return 0; }; ` +
     adopt +
     `if [ -e "$M" ]; then ` +
     `echo 'spectest: the previous cluster on this image store never became ready; starting from an empty one' >&2; ` +
@@ -724,7 +751,7 @@ async function issueIngressCert(
   const id = `spectest-k3s-ingress-${randomUUID().slice(0, 8)}`;
   const keyPath = `/tmp/${id}.key`;
   const crtPath = `/tmp/${id}.crt`;
-  const sans = hostnames.map((h) => `DNS:${h}`).join(",");
+  const sans = hostnames.map((h) => `${isIP(h) ? "IP" : "DNS"}:${h}`).join(",");
   const r = await runProcess(
     "openssl",
     [
@@ -1839,48 +1866,52 @@ async function setupK3sCluster(
   helpers: K3sHelpers,
   opts: { registry: boolean; ingressDomains: string[]; traefik: boolean },
 ): Promise<void> {
-  const tlsEnabled = opts.traefik && opts.ingressDomains.length > 0 && caPresent();
-  if (tlsEnabled) {
-    const { cert, key } = await issueIngressCert(
-      opts.ingressDomains.map((d) => `*.${d}`),
-    );
-    // Apply the cert Secret + dynamic-config ConfigMap before the
-    // Deployment that mounts them. `stringData` lets us hand over plain
-    // PEM; the API server base64-encodes it.
-    await helpers.client.core.createNamespacedSecret({
-      namespace: "kube-system",
-      body: {
-        metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
-        type: "kubernetes.io/tls",
-        stringData: { "tls.crt": cert, "tls.key": key },
-      },
-    });
-    await helpers.client.core.createNamespacedConfigMap({
-      namespace: "kube-system",
-      body: {
-        metadata: {
-          name: TRAEFIK_DYNAMIC_CONFIGMAP,
-          namespace: "kube-system",
-        },
-        data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
-      },
-    });
-  }
-  if (opts.traefik) await helpers.apply(buildTraefikManifest(tlsEnabled));
-  if (opts.registry) await helpers.apply(REGISTRY_MANIFEST);
-  // Both rollouts proceed independently inside the cluster — wait on them
-  // concurrently (they used to serialize, wasting up to a rollout's tail).
-  // 250ms polling: these two sit on the cold start's critical path, where
-  // the default 500ms interval wastes real time.
+  // The registry can start while Traefik's certificates are being prepared.
   const wait = (deployment: string): Promise<void> =>
     helpers.waitForRollout(deployment, {
       namespace: "kube-system",
       timeoutMs: 120_000,
       intervalMs: 250,
     });
-  const waits = opts.traefik ? [wait("traefik")] : [];
-  if (opts.registry) waits.push(wait("spectest-registry"));
-  await Promise.all(waits);
+  await waitForAll([
+    (async () => {
+      if (!opts.traefik) return;
+      const tlsEnabled = opts.ingressDomains.length > 0 && caPresent();
+      if (tlsEnabled) {
+        const { cert, key } = await issueIngressCert(
+          opts.ingressDomains.map((d) => `*.${d}`),
+        );
+        // Apply the cert Secret + dynamic-config ConfigMap before the
+        // Deployment that mounts them. `stringData` lets us hand over plain
+        // PEM; the API server base64-encodes it.
+        await helpers.client.core.createNamespacedSecret({
+          namespace: "kube-system",
+          body: {
+            metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
+            type: "kubernetes.io/tls",
+            stringData: { "tls.crt": cert, "tls.key": key },
+          },
+        });
+        await helpers.client.core.createNamespacedConfigMap({
+          namespace: "kube-system",
+          body: {
+            metadata: {
+              name: TRAEFIK_DYNAMIC_CONFIGMAP,
+              namespace: "kube-system",
+            },
+            data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
+          },
+        });
+      }
+      await helpers.apply(buildTraefikManifest(tlsEnabled));
+      await wait("traefik");
+    })(),
+    (async () => {
+      if (!opts.registry) return;
+      await helpers.apply(REGISTRY_MANIFEST);
+      await wait("spectest-registry");
+    })(),
+  ]);
 }
 
 /**
@@ -2007,6 +2038,7 @@ const DEFAULT_K3S_VERSION = "v1.33.5-k3s1";
 
 export function k3s(opts: K3sOptions = {}) {
   const version = opts.version ?? DEFAULT_K3S_VERSION;
+  const images = kubernetesImages(opts.images ?? []);
   const extra = opts.extraArgs ?? [];
   assertUsableExtraArgs(extra);
   const registryEnabled = opts.registry !== false;
@@ -2023,7 +2055,8 @@ export function k3s(opts: K3sOptions = {}) {
   // The environment's image cache, when this VM carries one: the
   // cluster's store goes on its root disk and its layers come off the
   // shared layers disk (see the store section above).
-  const cache = imageCachePathsSync();
+  const sticky = stickyDiskPathSync();
+  const cache = sticky ? null : imageCachePathsSync();
   const serverArgs = [
     "k3s",
     "server",
@@ -2122,6 +2155,7 @@ export function k3s(opts: K3sOptions = {}) {
   const readyCmd = `kubectl get --raw=/readyz >/dev/null 2>&1 && rm -f ${CLUSTER_STORE_TARGET}/${BOOT_MARKER}`;
   const def = {
     image: { type: "registry", reference: `rancher/k3s:${version}` },
+    imageBuilds: images.builds,
     command: cmd,
     privileged: true,
     tmpfs: ["/run", "/var/run"],
@@ -2135,15 +2169,17 @@ export function k3s(opts: K3sOptions = {}) {
     // the same netns and are reachable the same way, without appearing
     // in this list (it's documentation, not a firewall).
     ports: registryEnabled ? [80, 443, 6443, K3S_REGISTRY_PORT] : [80, 443, 6443],
-    // The cluster's image store (see the store section above): its
+    // Sticky disks retain an independent overlayfs store per service/version.
+    // The legacy image cache (see the store section above) keeps its
     // containerd root on the image cache's root disk, the shared layers
     // disk read-only at its guest path, golden's mkfs.erofs and the adopt
     // helper — or, with no cache, a per-environment volume. Two clusters
     // in one environment get separate roots either way (the service
     // token names the directory), because two containerd daemons cannot
     // share a store: bolt takes an exclusive lock.
-    volumes: cache
-      ? [
+    volumes: sticky
+      ? [{ source: `${stickyKubernetesDirectory(SELF_SERVICE_TOKEN, version)}/containerd`, target: CLUSTER_STORE_TARGET }]
+      : cache ? [
           { source: nestedStoreDir(cache, SELF_SERVICE_TOKEN), target: CLUSTER_STORE_TARGET },
           { source: cache.layers, target: cache.layers, readOnly: true },
           { source: MKFS_EROFS_PATH, target: MKFS_EROFS_PATH, readOnly: true },
@@ -2164,12 +2200,56 @@ export function k3s(opts: K3sOptions = {}) {
       command: readyCmd,
       timeoutSecs: opts.readyTimeoutSecs ?? 120,
     },
-    setup: async ({ name, helpers }: { name: string; helpers: K3sHelpers }) => {
-      await setupK3sCluster(name, helpers, {
-        registry: registryEnabled,
-        ingressDomains,
-        traefik: traefikEnabled,
-      });
+    setup: async (ctx: ServiceSetupContext<K3sHelpers>) => {
+      const { name, helpers, exec } = ctx;
+      const runtime = componentRuntime(ctx);
+      if (images.builds.length && !runtime) throw new Error(`k3s(${name}): prepared image inputs are missing`);
+      await waitForAll([
+        runtime ? loadK3sBuildImages(name, runtime.images, runtime.gateway, exec) : Promise.resolve(),
+        prePullK3sImages(name, images.pulls, exec),
+        (async () => {
+          if (runtime?.caBundle && caPresent()) {
+            const { cert, key } = await issueIngressCert([runtime.gateway]);
+            await installPodTrust({ name, gateway: runtime.gateway, cert, key,
+              ca: await readFile(CA_PATH, "utf8"), bundle: await readFile(runtime.caBundle, "utf8"),
+            }, {
+              apply: async configuration => { await helpers.apply(JSON.stringify(configuration)); },
+              ensureBundle: async (namespace, configMap, bundle) => {
+                try {
+                  await helpers.client.core.createNamespacedConfigMap({ namespace, body: {
+                    metadata: { name: configMap, namespace }, immutable: true, data: { "ca-bundle.crt": bundle },
+                  } });
+                } catch (err) {
+                  // A simultaneous Pod may have created the same namespace's bundle.
+                  const existing = (await helpers.client.core.readNamespacedConfigMap({ namespace, name: configMap })).unwrap();
+                  if (existing.data?.["ca-bundle.crt"] !== bundle || existing.immutable !== true) throw err;
+                }
+              },
+            });
+            // The API server observes webhook registration asynchronously. A dry-run
+            // Pod proves TLS reachability and admission before any project workload.
+            const deadline = Date.now() + 30_000;
+            for (;;) {
+              try {
+                const pod = (await helpers.client.core.createNamespacedPod({ namespace: "default", dryRun: "All", body: {
+                  metadata: { generateName: "spectest-trust-probe-" }, spec: { containers: [{ name: "probe", image: "registry.k8s.io/pause:3.10" }] },
+                } })).unwrap();
+                if (pod.spec?.containers[0].env?.some(e => e.name === "SSL_CERT_FILE" && e.value === "/spectest/ca-bundle.crt")) break;
+                if (Date.now() >= deadline) throw new Error(`k3s(${name}): Pod CA mounts did not become ready`);
+              } catch (err) {
+                if (Date.now() >= deadline) throw err;
+              }
+              await new Promise(resolve => setTimeout(resolve, 100));
+            }
+          }
+          await setupK3sCluster(name, helpers, {
+            registry: registryEnabled,
+            ingressDomains,
+            traefik: traefikEnabled,
+          });
+        })(),
+      ]);
+      if (sticky) await registerStickyKubernetesStore(name, version);
     },
     helpers: async ({ name, exec, poll }: ServiceHelpersContext): Promise<K3sHelpers> => {
       // Read the cluster's kubeconfig and address the API server by its

package/src/daemon.ts

@@ -1,3 +1,5 @@
+import { componentBuildTasks, COMPONENT_RUNTIME } from "./harness/component-images.js";
+import { waitForAll } from "./harness/concurrency.js";
 import { androidScope, androidRecordingChunk, releaseAndroidRecording, isAndroidApp, type AndroidRecording } from "./android.js";
 // Long-running HTTP daemon. Runs as a systemd unit on the VM host (not in
 // a container). Owns:
@@ -35,6 +37,7 @@ import {
   dnsName as makeDnsDecl,
   isWildcard,
   validateServiceImage,
+  normalizeServiceImage,
   proxy as makeProxyDecl,
 } from "./index.js";
 import type { DnsTarget, LoweredIngress } from "./index.js";
@@ -154,6 +157,7 @@ import {
 } from "./harness/ingress-table.js";
 import { startTlsTerminator, type TlsTerminator } from "./harness/tls-terminator.js";
 import { runContainerArgs } from "./harness/container-run.js";
+import { isOnStickyDisk } from "./harness/sticky-paths.js";
 import {
   BUILDKIT_CACHE_DIR,
   BUILDKIT_CACHE_UNION,
@@ -239,7 +243,7 @@ import type {
   ServiceConfig,
   ServiceDefinition,
   ServiceHandles,
-  ServiceImage,
+  NormalizedServiceImage,
   SpectestContext,
   Terminal,
   TerminalOpts,
@@ -256,10 +260,21 @@ import type { InternalTerminal, TerminalFrameSink } from "./terminal.js";
  * name (container name, image tag, ready-probe host). Widened to
  * accept any helpers shape so a typed `setup({ helpers })` call from
  * a component (e.g. K3sHelpers) lines up at the call site. */
-type NamedService = ServiceDefinition<Record<string, unknown>> & { name: string };
+type NamedService = Omit<ServiceDefinition<Record<string, unknown>>, "image"> & { name: string; image: NormalizedServiceImage };
 
 function namedServices(cfg: EnvironmentConfig): NamedService[] {
-  return Object.entries(cfg.services).map(([name, def]) => ({ name, ...def }));
+  return Object.entries(cfg.services).map(([name, def]) => ({ name, ...def, image: normalizeServiceImage(def.image) }));
+}
+
+/** Build tasks participate in image preparation, never container startup. */
+function imageTasks(services: NamedService[]): NamedService[] {
+  const tasks = componentBuildTasks(services).map(task => ({ ...task, image: validateServiceImage(task.name, task.image) }));
+  const names = new Set(services.map(s => s.name));
+  for (const task of tasks) {
+    if (names.has(task.name)) throw new Error(`image task name conflicts with service: ${task.name}`);
+    names.add(task.name);
+  }
+  return [...services, ...tasks];
 }
 
 const DEFAULT_TEST_TIMEOUT_MS = 60_000;
@@ -977,7 +992,7 @@ async function ensureVolumes(svc: NamedService): Promise<string[]> {
       // A directory on a cache disk is the same kind of thing: a nested
       // runtime's containerd root (`k3s()`), kept as a cache by the
       // lineage exactly as the container store one level up is.
-      const durable = host.startsWith("/var/cache/spectest/") || isOnImageCache(host, imageCachePathsSync(IMAGE_CACHE_MANIFEST));
+      const durable = host.startsWith("/var/cache/spectest/") || isOnStickyDisk(host) || isOnImageCache(host, imageCachePathsSync(IMAGE_CACHE_MANIFEST));
       if (vol.source?.startsWith("/") && !durable) {
         await recordAbsoluteVolumeDir(host);
       }
@@ -1365,6 +1380,18 @@ async function resolveDockerfileBuild(svc: NamedService): Promise<DockerfileBuil
 async function prepareServiceImage(
   svc: NamedService,
   opts?: { dedup?: boolean },
+): Promise<{ tag: string; buildSteps?: BuildStep[] }> {
+  const result = await prepareServiceImageInner(svc, opts);
+  if (svc.image.type === "dockerfile" && svc.image.reference) {
+    const tagged = await docker(["tag", result.tag, svc.image.reference], 30_000);
+    if (tagged.code !== 0) throw new Error(`cannot name built image ${svc.image.reference}: ${tagged.stderr}`);
+  }
+  return result;
+}
+
+async function prepareServiceImageInner(
+  svc: NamedService,
+  opts?: { dedup?: boolean },
 ): Promise<{ tag: string; buildSteps?: BuildStep[] }> {
   const tag = imageTag(svc.name);
   if (svc.image.type === "registry") {
@@ -1414,7 +1441,7 @@ async function prepareServiceImage(
         progressService(svc.name, { status: "failed" });
         throw err;
       }
-      // first.tag is already CA-layered; tagging it covers this service too.
+      // Reuse the same application image; runtime trust is added separately.
       const tagr = await docker(["tag", first.tag, tag], 30_000);
       if (tagr.code !== 0) {
         throw new Error(`docker tag ${first.tag} ${tag} failed: ${tagr.stderr.trim()}`);
@@ -3175,7 +3202,8 @@ function specToNamedService(spec: RuntimeServiceSpec): NamedService {
   // run here: the image is checked (a `path` Dockerfile and a `context`
   // must exist), and coverage adapters get their configure step.
   const svc = { ...(rest as ServiceConfig), image: validateServiceImage(name, rest.image) };
-  return { name, ...applyCoverageAdapters(name, svc) };
+  const configured = applyCoverageAdapters(name, svc);
+  return { name, ...configured, image: normalizeServiceImage(configured.image) };
 }
 
 /** Implementation behind `ctx.startService` / a fake's `ctx.startService`.
@@ -3507,12 +3535,14 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
   const bootStart = Date.now();
   const cfg: EnvironmentConfig = requireLoaded().project.environment;
   const services = namedServices(cfg);
+  const tasks = imageTasks(services);
+  const componentTasks = componentBuildTasks(services);
   const timings = new Map<string, ServiceTiming>();
   if (prepared && (prepared.version !== 1 || !prepared.images ||
-      Object.keys(prepared.images).sort().join("\n") !== services.map(s => s.name).sort().join("\n"))) {
+      Object.keys(prepared.images).sort().join("\n") !== tasks.map(s => s.name).sort().join("\n"))) {
     throw new Error("prepared image manifest does not match the environment's services");
   }
-  progressInit(services);
+  progressInit(tasks);
   // Build dedup is only valid within one workspace generation — a fresh
   // bootstrap may follow a workspace re-upload with the same dockerfile
   // text but different build-context content.
@@ -3535,7 +3565,7 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
       if ((await readIgnoreFile(path.join(WORKSPACE, ".dockerignore"))) !== null) return;
       await fs.writeFile(
         path.join(WORKSPACE, ".dockerignore"),
-        unionDockerignore(services),
+        unionDockerignore(tasks),
       );
     })(),
   ]);
@@ -3572,11 +3602,14 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
     if (found.code !== 0 || found.stdout.trim() !== entry.image) {
       throw new Error(`prepared image missing for ${svc.name}: ${entry.image}`);
     }
-    return { tag: entry.image, buildSteps: entry.buildSteps };
+    const tag = imageTag(svc.name);
+    const tagged = await docker(["tag", entry.image, tag], 30_000);
+    if (tagged.code !== 0) throw new Error(`cannot restore prepared image name for ${svc.name}: ${tagged.stderr}`);
+    return { tag, buildSteps: entry.buildSteps };
   };
   const prep = new Map<string, Promise<void>>();
   const prepOffsets = new Map<string, number>();
-  for (const svc of services) {
+  for (const svc of tasks) {
     const p = (async () => {
       const t0 = Date.now();
       prepOffsets.set(svc.name, Date.now() - bootStart);
@@ -3602,7 +3635,7 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
     // No application services, setup hooks, ingress, or runtime state in the builder.
     await Promise.all(prep.values());
     const images: PreparedImages["images"] = Object.create(null);
-    for (const svc of services) {
+    for (const svc of tasks) {
       const out = await docker(["image", "inspect", "--format", "{{.Id}}", tags.get(svc.name)!], 30_000);
       const image = out.stdout.trim();
       if (out.code !== 0 || !/^sha256:[a-f0-9]{64}$/.test(image)) {
@@ -3654,19 +3687,30 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
     progressService(svc.name, { status: "probing", detail: "ready check" });
     await waitForReady(svc);
     const tSetup = Date.now();
-    if (svc.setup) {
-      progressService(svc.name, { status: "probing", detail: "running setup" });
-      const helpers = await ensureHelpers(svc.name, svc);
-      try {
-        await svc.setup({
-          name: svc.name,
-          helpers,
-          ...(await spectestContext({ service: svc.name, includeSelf: true })),
-        });
-      } catch (err) {
-        throw await withContainerPostMortem(svc.name, err);
+    const builds = componentTasks.filter(task => task.owner === svc.name);
+    // Setup can overlap pulls/infrastructure with builds, and consume each
+    // image as it becomes ready. Still drain every build if setup fails.
+    await waitForAll([...builds.map(task => prep.get(task.name)!), (async () => {
+      if (svc.setup) {
+        progressService(svc.name, { status: "probing", detail: "running setup" });
+        const helpers = await ensureHelpers(svc.name, svc);
+        try {
+          await svc.setup({
+            name: svc.name,
+            helpers,
+            ...(await spectestContext({ service: svc.name, includeSelf: true })),
+            ...{ [COMPONENT_RUNTIME]: {
+              images: builds.map(task => ({ source: imageTag(task.name), reference: task.reference, ready: prep.get(task.name)! })),
+              gateway: await bridgeGatewayIp(),
+              caBundle: await ensureCaBundle(),
+            } },
+          });
+        } catch (err) {
+          throw await withContainerPostMortem(svc.name, err);
+        }
       }
-    }
+    })()]);
+    for (const task of builds) progressService(task.name, { status: "ready", detail: `loaded ${task.reference}` });
     progressService(svc.name, { status: "ready", detail: undefined });
     const ti = timings.get(svc.name);
     if (ti) {
@@ -4223,9 +4267,10 @@ async function computeProjectReach(all: NamedService[]): Promise<ReachRules> {
 async function computeProjectReachInner(all: NamedService[]): Promise<ReachRules> {
   const files = await walkProjectFiles(WORKSPACE, [".git"]);
   const sources: ReachSource[] = [];
-  for (const svc of all) {
+  const owners = new Map(componentBuildTasks(all).map(t => [t.name, t.owner]));
+  for (const svc of imageTasks(all)) {
     const src: ReachSource = {
-      service: svc.name,
+      service: owners.get(svc.name) ?? svc.name,
       volumeSources: (svc.volumes ?? []).flatMap((v) => (v.source && !v.reachExempt ? [v.source] : [])),
       ...(svc.inputs && svc.inputs.length > 0 ? { inputs: [...svc.inputs] } : {}),
     };
@@ -6720,7 +6765,7 @@ export function harnessMethods(state: RouteState): MethodTable {
     bootstrap: async (params) =>
       exclusive(state, "inFlightBootstrap", "bootstrap already in progress", async () => {
         const prepared = params.prepared === undefined ? undefined : preparedImages(
-          params.prepared, params.identity, namedServices(requireLoaded().project.environment).map(s => s.name),
+          params.prepared, params.identity, imageTasks(namedServices(requireLoaded().project.environment)).map(s => s.name),
         );
         return { ok: true, timings: await bootstrap(prepared) };
       }),