@specific.dev/spectest 0.71.1 → 0.73.0

package/src/daemon.ts

@@ -135,6 +135,7 @@ import {
 } from "./harness/ingress-table.js";
 import { startTlsTerminator, type TlsTerminator } from "./harness/tls-terminator.js";
 import { runContainerArgs } from "./harness/container-run.js";
+import { IMAGE_CACHE_MANIFEST, imageCachePathsSync, isOnImageCache, type ImageCachePaths } from "./harness/image-cache.js";
 import {
   assertAbsolute,
   certificateHostnames,
@@ -714,7 +715,6 @@ async function hasBuildx(): Promise<boolean> {
 // in-VM buildkitd keeps its exported cache there too, so a fresh VM finds
 // every layer it built before. Detected once; if the daemon will not
 // start, dockerd's own BuildKit builds instead.
-const IMAGE_CACHE_MANIFEST = "/run/spectest-image-cache.json";
 const LOCAL_BUILDER_NAME = "spectest-local";
 const LOCAL_BUILDKIT_ADDR = "tcp://127.0.0.1:1234";
 /** The bring-up script the cache base bakes (`base.rs::BUILDKITD_UP_SH`). */
@@ -723,16 +723,8 @@ const BUILDKITD_UP_PATH = "/usr/local/bin/spectest-buildkitd-up";
 /** Where the control plane mounted this VM's image cache: containerd's
  *  root (read-write, this VM's own) and the layers disk (read-only, shared
  *  by every VM of a generation). `null` when the VM carries no cache. */
-async function imageCachePaths(): Promise<{ root: string; layers: string } | null> {
-  try {
-    const raw = await fs.readFile(IMAGE_CACHE_MANIFEST, "utf8");
-    const parsed = JSON.parse(raw) as { disks?: { role?: string; path?: string }[] };
-    const root = (parsed.disks ?? []).find((d) => d.role === "root" && d.path)?.path;
-    const layers = (parsed.disks ?? []).find((d) => d.role === "layers" && d.path)?.path;
-    return root && layers ? { root, layers } : null;
-  } catch {
-    return null;
-  }
+async function imageCachePaths(): Promise<ImageCachePaths | null> {
+  return imageCachePathsSync(IMAGE_CACHE_MANIFEST);
 }
 
 let _localBuilder: boolean | undefined;
@@ -876,7 +868,10 @@ async function ensureVolumes(svc: NamedService): Promise<string[]> {
       // serving older SDKs. Leaving it out of the manifest is what
       // protects a project running this SDK against a server whose
       // teardown guard predates it.
-      const durable = host.startsWith("/var/cache/spectest/");
+      // A directory on a cache disk is the same kind of thing: a nested
+      // runtime's containerd root (`k3s()`), kept as a cache by the
+      // lineage exactly as the container store one level up is.
+      const durable = host.startsWith("/var/cache/spectest/") || isOnImageCache(host, imageCachePathsSync(IMAGE_CACHE_MANIFEST));
       if (vol.source?.startsWith("/") && !durable) {
         await recordAbsoluteVolumeDir(host);
       }
@@ -1244,7 +1239,6 @@ async function prepareServiceImage(
     if (tagr.code !== 0) {
       throw new Error(`docker tag ${ref} ${tag} failed: ${tagr.stderr.trim()}`);
     }
-    await ensureCaTrustedImage(svc.name, tag);
     return { tag };
   }
   // Dockerfile build. Within one bootstrap, identical definitions (shared
@@ -1288,61 +1282,6 @@ async function prepareServiceImage(
   return buildServiceImage(svc.name, image, tag);
 }
 
-/** Printed by the folded CA step when it could not write the trust
- *  store at all — the one outcome that still needs the derivative build. */
-const CA_FOLD_UNWRITABLE = "[spectest-ca] trust store not writable";
-
-/**
- * The CA-trust steps as a suffix appended to a dockerfile service's OWN
- * Dockerfile, so one build produces the finished image instead of a build
- * plus a derivative rebuild ({@link ensureCaTrustedImage}) per service.
- * Returns null when there is no CA to layer, or when the PEM can't be
- * quoted — the caller then falls back to the derivative build.
- *
- * The PEM is written INLINE rather than `COPY`d: the build context is
- * `/workspace` under a per-service ignore file that the project itself
- * contributes to (a `**` line with re-includes is the common idiom), and
- * a context path we don't control is a context path that can be excluded.
- * printf needs nothing but a shell.
- *
- * Two rules make this safe to bolt onto user code. It must never fail the
- * build — every branch ends in an echo, so the RUN exits 0 whatever the
- * image lacks — and it must never change the image, beyond the trust
- * store: notably no `USER root`, since we cannot know statically what
- * user to hand back. An image that declares a non-root user therefore
- * fails to write and is finished by the derivative build, which inspects
- * the built image and can escalate properly.
- */
-async function caTrustSuffix(): Promise<string | null> {
-  if (!existsSync(CA_PATH)) return null;
-  const pem = (await fs.readFile(CA_PATH, "utf8")).trim();
-  // A quote in the PEM would break out of the shell quoting below. PEM is
-  // base64 and dashes, so this is a guard, not a case we expect.
-  if (!pem || pem.includes("'")) return null;
-  const args = pem
-    .split("\n")
-    .map((l) => `'${l.trimEnd()}'`)
-    .join(" ");
-  const dst = "/usr/local/share/ca-certificates/spectest-ca.crt";
-  return `
-# spectest: trust the environment's root CA. Appended by the harness —
-# not part of the project's Dockerfile.
-RUN P='[spectest-ca]'; \\
-    mkdir -p /usr/local/share/ca-certificates 2>/dev/null; \\
-    if printf '%s\\n' ${args} > ${dst} 2>/dev/null; then \\
-      if command -v update-ca-certificates >/dev/null 2>&1 && update-ca-certificates >/dev/null 2>&1; then \\
-        echo "$P trusted via update-ca-certificates"; \\
-      elif command -v update-ca-trust >/dev/null 2>&1 && cp ${dst} /etc/pki/ca-trust/source/anchors/spectest-ca.crt && update-ca-trust extract >/dev/null 2>&1; then \\
-        echo "$P trusted via update-ca-trust"; \\
-      else \\
-        echo "$P no system CA trust tool in image; env-var trust only"; \\
-      fi; \\
-    else \\
-      echo "$P trust store not writable by this image's user"; \\
-    fi
-`;
-}
-
 /** The built form of a dockerfile image. `resolveServiceImage` read a
  *  `path` into `content` at config time, so a service that still carries
  *  `path` here skipped that step — a programming error, not user input. */
@@ -1355,73 +1294,38 @@ function dockerfileContent(svc: NamedService): DockerfileBuild {
 }
 
 /**
- * Build a dockerfile service's image.
+ * Build a dockerfile service's image — the project's Dockerfile as
+ * written, nothing appended.
  *
- * The CA-trust layer is folded into THIS build when it can be (see
- * {@link caTrustSuffix}), so a service costs one image build and one
- * export rather than two. The derivative build stays as the fallback for
- * everything the folded form can't serve: an image with no shell to run
- * the step (distroless, scratch — the appended RUN can't execute, so the
- * build fails and we rebuild the project's Dockerfile untouched), and an
- * image that declares a non-root user (the step runs as that user and
- * can't write the trust store).
+ * Trust in the environment's CA is not the image's business any more:
+ * `runContainer` mounts the combined bundle over the system trust store
+ * (`container-run.ts`, `SYSTEM_TRUST_BUNDLE_PATHS`). Until 2026-09-06 a
+ * CA step was folded into this build and a derivative `docker build`
+ * finished any image the fold could not write to (non-root `USER`, no
+ * shell), which was one or two extra builds per service, and one per
+ * pulled image, on every cold start.
  */
 async function buildServiceImage(
   name: string,
   image: DockerfileBuild,
   tag: string,
 ): Promise<{ tag: string; buildSteps?: BuildStep[] }> {
-  const suffix = await caTrustSuffix();
-  let attempt = await runServiceBuild(name, image, tag, suffix);
-  if (!attempt.ok && suffix) {
-    // Our step must not be able to break a project's build.
-    // eslint-disable-next-line no-console
-    console.warn(
-      `[ca-trust] folded CA step could not run in ${name}'s image; rebuilding without it`,
-    );
-    attempt = await runServiceBuild(name, image, tag, null);
-  }
+  const attempt = await runServiceBuild(name, image, tag);
   if (!attempt.ok) {
     progressService(name, { status: "failed" });
     throw new Error(`docker build for ${name} failed:\n${attempt.log}`);
   }
-  // The derivative build is still needed for the one thing the folded step
-  // cannot do: write the trust store of an image that does not run as
-  // root. That is decided on the IMAGE, not on the build log — a CACHED
-  // layer prints nothing, so a log-only check would quietly stop
-  // re-applying the moment BuildKit had the layer. The log covers the
-  // rarer case of a root image whose /etc is read-only.
-  //
-  // An image with no trust tool at all needs nothing further: the
-  // derivative build would reach the same dead end, and `runContainer`'s
-  // env vars are the fallback either way.
-  const needsDerivative =
-    !attempt.folded ||
-    attempt.log.includes(CA_FOLD_UNWRITABLE) ||
-    !(await imageRunsAsRoot(tag));
-  if (needsDerivative) await ensureCaTrustedImage(name, tag);
   return { tag, buildSteps: attempt.buildSteps };
 }
 
-/** Whether `tag`'s declared `USER` is root (or unset, which means root). */
-async function imageRunsAsRoot(tag: string): Promise<boolean> {
-  const user = (
-    await docker(["image", "inspect", "--format", "{{.Config.User}}", tag], 60_000)
-  ).stdout.trim();
-  return user === "" || user === "root" || user === "0";
-}
-
 async function runServiceBuild(
   name: string,
   image: DockerfileBuild,
   tag: string,
-  caSuffix: string | null,
-): Promise<{ ok: boolean; folded: boolean; log: string; buildSteps?: BuildStep[] }> {
+): Promise<{ ok: boolean; log: string; buildSteps?: BuildStep[] }> {
   let buildSteps: BuildStep[] | undefined;
   {
-    const content = caSuffix
-      ? `${image.content.replace(/\n*$/, "\n")}${caSuffix}`
-      : image.content;
+    const content = image.content;
     const dfDir = path.join(WORKSPACE, ".spectest", "services", name);
     await fs.mkdir(dfDir, { recursive: true });
     const dfPath = path.join(dfDir, "Dockerfile");
@@ -1519,9 +1423,7 @@ async function runServiceBuild(
     });
     const log = `${build.stderr.trim()}\n${build.stdout.trim()}`;
     if (build.code !== 0) {
-      // The caller decides whether this is fatal: a failure with the CA
-      // step appended is retried without it before anyone hears about it.
-      return { ok: false, folded: caSuffix !== null, log };
+      return { ok: false, log };
     }
     if (useBuildKit) {
       // Keep only the slowest dozen steps ≥1s — enough to profile, small
@@ -1530,55 +1432,7 @@ async function runServiceBuild(
         .filter((s) => s.secs >= 1)
         .slice(0, 12);
     }
-    return { ok: true, folded: caSuffix !== null, log, buildSteps };
-  }
-}
-
-/**
- * Build a derivative image on top of `tag` that copies the spectest
- * root CA into the system trust store. Tagged back as `tag`, so the
- * rest of the orchestrator (runContainer, image cache) is oblivious.
- * Failures are warned-and-ignored: the env-var injection in
- * runContainer is the universal fallback, so apps that use it (most
- * Node/Python/Ruby/AWS clients) still trust the CA even when the
- * image's trust store can't be updated.
- */
-async function ensureCaTrustedImage(serviceName: string, tag: string): Promise<void> {
-  if (!existsSync(CA_PATH)) {
-    // Daemon running outside a base-snapshot VM (dev/test). Nothing to
-    // layer; env vars also harmless (they point at a missing path, but
-    // most consumers ignore missing files).
-    return;
-  }
-  const ctxDir = path.join(WORKSPACE, ".spectest", "ca-trust", serviceName);
-  await fs.mkdir(ctxDir, { recursive: true });
-  await fs.copyFile(CA_PATH, path.join(ctxDir, "spectest-ca.crt"));
-  // An image that declares a non-root `USER` runs this step as that user,
-  // and writing the trust store then fails ("cannot create
-  // /etc/ssl/certs/ca-certificates.crt.new: Permission denied") — the CA
-  // silently never lands. Take root for the one command, then hand the
-  // image back its own user so containers still run as it.
-  const declaredUser = (
-    await docker(["image", "inspect", "--format", "{{.Config.User}}", tag], 60_000)
-  ).stdout.trim();
-  const needsRoot = declaredUser !== "" && declaredUser !== "root" && declaredUser !== "0";
-  const dockerfile = `FROM ${tag}
-${needsRoot ? "USER root\n" : ""}COPY spectest-ca.crt /usr/local/share/ca-certificates/spectest-ca.crt
-RUN if command -v update-ca-certificates >/dev/null 2>&1; then \\
-      update-ca-certificates; \\
-    elif command -v update-ca-trust >/dev/null 2>&1; then \\
-      cp /usr/local/share/ca-certificates/spectest-ca.crt /etc/pki/ca-trust/source/anchors/spectest-ca.crt && update-ca-trust extract; \\
-    else \\
-      echo "[spectest] no system CA trust tool in image; env-var trust only"; \\
-    fi
-${needsRoot ? `USER ${declaredUser}\n` : ""}`;
-  await fs.writeFile(path.join(ctxDir, "Dockerfile"), dockerfile);
-  const build = await docker(["build", "-t", tag, ctxDir], 300_000);
-  if (build.code !== 0) {
-    // eslint-disable-next-line no-console
-    console.warn(
-      `[ca-trust] could not layer spectest CA into ${serviceName} (${tag}); env-var fallback only:\n${build.stderr.trim() || build.stdout.trim()}`,
-    );
+    return { ok: true, log, buildSteps };
   }
 }
 
@@ -1590,10 +1444,14 @@ let caBundlePromise: Promise<string | null> | null = null;
  * return its path, or null when there's no CA to trust (daemon running
  * outside a base-snapshot VM).
  *
- * This is what the replace-semantics trust env vars must point at. Falling
- * back to the bare CA when the guest bundle is unreadable keeps the old
- * behaviour — fakes verify, public HTTPS doesn't — which is strictly better
- * than dropping the CA and breaking the fakes everything else depends on.
+ * This is what the replace-semantics trust env vars must point at, and
+ * what `runContainer` mounts over the image's own system trust store
+ * (`container-run.ts`). Falling back to the bare CA when the guest bundle
+ * is unreadable keeps the old behaviour for the variables — fakes verify,
+ * public HTTPS doesn't — which is strictly better than dropping the CA and
+ * breaking the fakes everything else depends on; the system-store mount
+ * is skipped in that case, since the bare CA there would take the public
+ * roots away from clients that read no variable.
  */
 async function ensureCaBundle(): Promise<string | null> {
   caBundlePromise ??= (async () => {

package/src/harness/container-run.test.ts

@@ -1,6 +1,11 @@
 import { describe, expect, test } from "bun:test";
 
-import { TCP_RETRIES2, runContainerArgs, type ContainerRunInput } from "./container-run";
+import {
+  SYSTEM_TRUST_BUNDLE_PATHS,
+  TCP_RETRIES2,
+  runContainerArgs,
+  type ContainerRunInput,
+} from "./container-run";
 
 const base = (over: Partial<ContainerRunInput> = {}): ContainerRunInput => ({
   svc: { name: "web" },
@@ -122,6 +127,31 @@ describe("certificate trust", () => {
     const mounts = args.filter((a) => a === "--volume=/etc/spectest/ca.crt:/etc/spectest/ca.crt:ro");
     expect(mounts).toHaveLength(1);
   });
+
+  /** Clients that read no variable — Go, curl, OpenSSL defaults, Python's
+   *  `ssl` — open the distro's bundle file. The combined bundle is mounted
+   *  over each known location, read-only, in place of the derivative
+   *  image build this replaced. */
+  test("mounts the bundle over the system trust store paths", () => {
+    const args = runContainerArgs(base({ caBundle: "/etc/spectest/ca-bundle.crt" }));
+    expect(SYSTEM_TRUST_BUNDLE_PATHS).toContain("/etc/ssl/certs/ca-certificates.crt");
+    expect(SYSTEM_TRUST_BUNDLE_PATHS).toContain("/etc/pki/tls/certs/ca-bundle.crt");
+    for (const p of SYSTEM_TRUST_BUNDLE_PATHS) {
+      expect(args).toContain(`--volume=/etc/spectest/ca-bundle.crt:${p}:ro`);
+    }
+  });
+
+  /** The bare CA over the system store would drop every public root, the
+   *  same trap as the replacing variables — so no bundle means no mount. */
+  test("never mounts the bare CA over the system trust store", () => {
+    // No bundle at all, and the fallback where the bundle IS the bare CA.
+    for (const caBundle of [null, "/etc/spectest/ca.crt"]) {
+      const args = runContainerArgs(base({ caBundle }));
+      for (const p of SYSTEM_TRUST_BUNDLE_PATHS) {
+        expect(args.some((a) => a.endsWith(`:${p}:ro`))).toBe(false);
+      }
+    }
+  });
 });
 
 describe("service configuration", () => {

package/src/harness/container-run.ts

@@ -35,6 +35,20 @@
  * `DENO_TLS_CA_STORE=mozilla,system` is what adds the platform store,
  * and the platform store is `SSL_CERT_FILE` — the combined bundle.
  *
+ * **The system trust store is the fourth, and it is a mount, not a
+ * layer.** Go, curl, OpenSSL's defaults and Python's `ssl` with no
+ * variable set read the distro's bundle file directly, so the combined
+ * bundle is bind-mounted read-only over the paths those bundles live at
+ * ({@link SYSTEM_TRUST_BUNDLE_PATHS}). Until 2026-09-06 this was a
+ * derivative image build per service (`COPY` the CA, run
+ * `update-ca-certificates`) — a `docker build` for every image, built or
+ * pulled, on every cold start, and most of what a cached bring-up still
+ * paid. A mount costs nothing, needs no cache, and behaves the same on a
+ * warm, delta or cold start. What it does not cover: a Java keystore
+ * (`cacerts`), and an entrypoint that runs `update-ca-certificates`
+ * itself — its final `mv` over the bundle fails with EBUSY on a
+ * mountpoint.
+ *
  * **`command` and `args` are different overrides.** `command` replaces the
  * entrypoint and runs through `sh -c`; `args` keeps the entrypoint and
  * overrides CMD, which is what init-wrapped images like postgres need to
@@ -95,6 +109,22 @@ export interface ContainerRunInput {
  */
 export const TCP_RETRIES2 = 6;
 
+/**
+ * Where a distro keeps the bundle that clients reading the system store
+ * open. Debian, Ubuntu and Alpine resolve every default path to the first
+ * through symlinks (`/etc/ssl/cert.pem`, `/usr/lib/ssl/cert.pem`); the
+ * RHEL family resolves to the second (itself a symlink into
+ * `/etc/pki/ca-trust/extracted/pem/`, which the runtime follows before it
+ * mounts). Go checks both names itself. An image with neither path gets
+ * the file created by the mount, which is what a scratch image with a
+ * Go binary wants. Read-only, so a container cannot change what its
+ * siblings trust.
+ */
+export const SYSTEM_TRUST_BUNDLE_PATHS = [
+  "/etc/ssl/certs/ca-certificates.crt",
+  "/etc/pki/tls/certs/ca-bundle.crt",
+] as const;
+
 /** Build the full `docker run` argv. Pure: no I/O, no module state. */
 export function runContainerArgs(input: ContainerRunInput): string[] {
   const { svc, tag, network } = input;
@@ -151,6 +181,13 @@ export function runContainerArgs(input: ContainerRunInput): string[] {
     args.push("-e", `SSL_CERT_FILE=${bundle}`);
     args.push("-e", `REQUESTS_CA_BUNDLE=${bundle}`);
     args.push("-e", `AWS_CA_BUNDLE=${bundle}`);
+    // The system store itself, for clients that read no variable at all.
+    // Only a real bundle: the caller falls back to the bare CA when the
+    // guest's roots are unreadable, and the bare CA over the system store
+    // would leave Go and curl trusting spectest and nothing else.
+    if (bundle !== input.caPath) {
+      for (const p of SYSTEM_TRUST_BUNDLE_PATHS) args.push(`--volume=${bundle}:${p}:ro`);
+    }
   }
 
   // The service's own env comes after ours, so a project can override the

package/src/harness/image-cache.ts

@@ -0,0 +1,79 @@
+/**
+ * The image cache as the guest sees it (`CONTAINER_STORE.md`).
+ *
+ * The control plane writes one manifest per VM at start naming the two
+ * cache disks it attached: the **root** (containerd's own root,
+ * read-write, this VM's clone) and the **layers** disk (read-only, one
+ * EROFS file per layer, shared by every VM of a generation). Both paths
+ * are fixed by the control plane; the manifest is how a harness learns
+ * whether this VM carries a cache at all (the fake backend does not,
+ * and neither does a server older than the cache).
+ *
+ * Read synchronously as well as asynchronously: a component's service
+ * definition is built inside `defineEnvironment`, which is synchronous,
+ * and `k3s()` decides its mounts there.
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+
+/** Written by `env.rs` before the harness starts. */
+export const IMAGE_CACHE_MANIFEST = "/run/spectest-image-cache.json";
+
+/** Where the cache's paths are, when this VM carries one. */
+export interface ImageCachePaths {
+  /** containerd's root: read-write, this VM's own clone. */
+  root: string;
+  /** The layers disk: read-only, shared by every VM of a generation. */
+  layers: string;
+}
+
+/**
+ * Directory under the root disk holding a nested runtime's containerd
+ * root, one per service: `<root>/spectest-nested/<service>`. The merge
+ * (`image_cache/merge.rs::NESTED_DIR`) reads every store it finds there
+ * exactly as it reads the disk's own.
+ */
+export const NESTED_STORES_DIR = "spectest-nested";
+
+/** The guest's static `mkfs.erofs`, which a nested runtime's EROFS
+ *  differ needs and no runtime image ships. */
+export const MKFS_EROFS_PATH = "/usr/local/bin/mkfs.erofs";
+
+/** The guest's adopt helper (`base.rs::STORE_ADOPT_SH`), POSIX sh so a
+ *  nested runtime's busybox can run the same file. */
+export const STORE_ADOPT_PATH = "/usr/local/bin/spectest-store-adopt";
+
+function parse(raw: string): ImageCachePaths | null {
+  const parsed = JSON.parse(raw) as { disks?: { role?: string; path?: string }[] };
+  const root = (parsed.disks ?? []).find((d) => d.role === "root" && d.path)?.path;
+  const layers = (parsed.disks ?? []).find((d) => d.role === "layers" && d.path)?.path;
+  return root && layers ? { root, layers } : null;
+}
+
+/** The cache's paths, or `null` when this VM carries none.
+ *  `SPECTEST_IMAGE_CACHE_MANIFEST` points a test at another file. */
+export function imageCachePathsSync(
+  manifest: string = process.env.SPECTEST_IMAGE_CACHE_MANIFEST || IMAGE_CACHE_MANIFEST,
+): ImageCachePaths | null {
+  try {
+    if (!existsSync(manifest)) return null;
+    return parse(readFileSync(manifest, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+/** The host directory a nested runtime keeps its containerd root in. */
+export function nestedStoreDir(paths: ImageCachePaths, service: string): string {
+  return `${paths.root}/${NESTED_STORES_DIR}/${service}`;
+}
+
+/**
+ * Is a volume's host path on a cache disk? Such a directory is a cache
+ * the lineage keeps — like the container store one level up — and the
+ * delta-restore teardown must not wipe it.
+ */
+export function isOnImageCache(hostPath: string, paths: ImageCachePaths | null): boolean {
+  if (!paths) return false;
+  return hostPath === paths.root || hostPath.startsWith(`${paths.root}/`) || hostPath === paths.layers || hostPath.startsWith(`${paths.layers}/`);
+}

package/src/harness/volume-paths.ts

@@ -63,8 +63,9 @@ export function sanitizeSegment(p: string): string {
  * `files` does: a component cannot know the map key the user will give it,
  * and an **absolute** source gets no automatic per-service directory. A
  * component that needs one — a nested runtime keeping its store under
- * {@link NESTED_STORE_ROOT}, where two of them sharing one directory would
- * be two daemons on one metadata store — writes the token into the path.
+ * the image cache's `spectest-nested/<service>` (`image-cache.ts::nestedStoreDir`),
+ * where two of them sharing one directory would be two daemons on one
+ * metadata store — writes the token into the path.
  *
  * `workspace` is a parameter rather than a module constant so the rule is
  * testable without touching the filesystem.