@specific.dev/spectest 0.131.13 → 0.131.14

package/dist/daemon.js

@@ -5,7 +5,7 @@ import { phoneScope, phoneRecordingChunk, releasePhoneRecording } from "./phone.
 import { runWithBackgroundFailures } from "./harness/background-failures.js";
 import { generateHostCert as mintHostCert } from "./harness/certificate.js";
 import { containerdImageOutput } from "./harness/built-image.js";
-import { componentBuildTasks, COMPONENT_RUNTIME } from "./harness/component-images.js";
+import { componentBuildTasks, COMPONENT_RUNTIME, isComponentImageTask, isDigestReference } from "./harness/component-images.js";
 import { waitForAll } from "./harness/concurrency.js";
 import { updateServiceProgress, serviceProgressSnapshot } from "./harness/service-progress.js";
 import { androidScope, androidRecordingChunk, releaseAndroidRecording, isAndroidApp } from "./android.js";
@@ -492,6 +492,18 @@ function ensureLocalBuildkitd() {
     })();
     return _localBuilder;
 }
+/** `true` when a digest-pinned reference is already complete in the store. */
+async function imagePresentByDigest(ref) {
+    if (!isDigestReference(ref))
+        return false;
+    // `inspect` succeeds only when every layer of the image for this
+    // platform is in the store; a partial or missing image falls through to
+    // the ordinary pull, which fetches what is absent. dockerd's containerd
+    // store also unpacks on demand at container creation, so a record whose
+    // snapshots were reclaimed is repaired there rather than failing.
+    const present = await docker(["image", "inspect", "--format", "{{.Id}}", ref], 30_000);
+    return present.code === 0;
+}
 /** Host directory backing one volume mount. Rules (notably which volumes
  *  survive the delta-restore teardown) live in `harness/volume-paths.ts`. */
 function resolveHostPath(service, vol) {
@@ -898,6 +910,19 @@ async function prepareServiceImageInner(svc, opts) {
         // tags (`foo:latest`) at whatever the previous generation pulled, for
         // as long as the delta chain lives — a silent semantic divergence
         // from the cold build a delta restore must be equivalent to.
+        //
+        // The one reference that cannot float is a digest. `name@sha256:…`
+        // names exactly one image for as long as it exists, so a copy already
+        // in the store is the image the registry would return, and the
+        // round-trip buys nothing: skip it. Component images pinned by digest
+        // and user references pinned by digest take this path; a tag never does.
+        if (await imagePresentByDigest(ref)) {
+            progressService(svc.name, { status: "pulling", detail: `present by digest` });
+            const tagged = await docker(["tag", ref, tag], 30_000);
+            if (tagged.code !== 0)
+                throw new Error(`docker tag ${ref} ${tag} failed: ${tagged.stderr.trim()}`);
+            return { tag };
+        }
         progressService(svc.name, { status: "pulling", detail: `pulling ${ref}` });
         let layers = 0;
         const pull = await shxStream("docker", ["pull", ref], 900_000, undefined, (line) => {
@@ -1042,11 +1067,30 @@ async function runServiceBuild(name, image, tag) {
             // containerd exporter replaces the tag directly, bypassing Docker's
             // dangling-image retention. Old versions must remain usable until our
             // image GC decides they are no longer needed (including by nested stores).
+            //
+            // `compression=uncompressed` (`containerdImageOutput`): the image never leaves this store, so
+            // gzip on export only to gunzip on unpack is pure cost — on Listen,
+            // 64 s compressing a 2.7 GB layer and 56 s decompressing it again,
+            // single-threaded, in the same guest that already held the layer as
+            // a build snapshot (CONTAINER_STORE.md, "Image delivery inside the
+            // guest"). With the default `force-compression=false` this applies
+            // to layers this build creates; a pulled base image's gzip blobs are
+            // reused as they are, so no existing layer is re-encoded or
+            // re-unpacked. The uncompressed blob is a second on-disk copy of the
+            // layer; the sticky headroom logic already budgets for blobs.
+            //
+            // `unpack`: dockerd needs the snapshot chain to start a container,
+            // so a service image is unpacked here rather than at `docker run`.
+            // A component image task (a `k3s()` build) is consumed only through
+            // the content store — `k3s-image-loader.ts` serves its blobs and the
+            // cluster's containerd unpacks them into its own store — so unpacking
+            // it in dockerd's store would be a third copy nobody reads.
+            const unpack = !isComponentImageTask(name);
             buildArgs = [
                 "buildx", "build",
                 "--builder", LOCAL_BUILDER_NAME,
                 "--progress=plain",
-                "--output", containerdImageOutput(tag),
+                "--output", containerdImageOutput(tag, { unpack }),
                 ...argFlags,
                 "-f", dfPath, contextDir,
             ];

package/dist/harness/built-image.d.ts

@@ -1,6 +1,15 @@
 /** Containerd exports replace mutable tags without Docker's dangling-image
- * retention. Keep each result addressable until the SDK's image GC evicts it. */
-export declare function containerdImageOutput(reference: string): string;
+ * retention. Keep each result addressable until the SDK's image GC evicts it.
+ *
+ * Layers this build creates are stored uncompressed: the image never leaves
+ * the VM's store, so compressing on export only to decompress on unpack was
+ * pure cost (64 s + 56 s on Listen's 2.7 GB layer). `force-compression` stays
+ * off, so a pulled base image's blobs are reused as they are. `unpack` is
+ * false for images only a nested store reads (CONTAINER_STORE.md, "Image
+ * delivery inside the guest"). */
+export declare function containerdImageOutput(reference: string, options?: {
+    unpack?: boolean;
+}): string;
 export interface Descriptor {
     digest: string;
     size: number;

package/dist/harness/built-image.js

@@ -4,9 +4,17 @@ import { createHash } from "node:crypto";
 import { promisify } from "node:util";
 import { canonicalImageReference } from "./component-images.js";
 /** Containerd exports replace mutable tags without Docker's dangling-image
- * retention. Keep each result addressable until the SDK's image GC evicts it. */
-export function containerdImageOutput(reference) {
-    return `type=image,name=${canonicalImageReference(reference)},name-canonical=true,unpack=true`;
+ * retention. Keep each result addressable until the SDK's image GC evicts it.
+ *
+ * Layers this build creates are stored uncompressed: the image never leaves
+ * the VM's store, so compressing on export only to decompress on unpack was
+ * pure cost (64 s + 56 s on Listen's 2.7 GB layer). `force-compression` stays
+ * off, so a pulled base image's blobs are reused as they are. `unpack` is
+ * false for images only a nested store reads (CONTAINER_STORE.md, "Image
+ * delivery inside the guest"). */
+export function containerdImageOutput(reference, options = {}) {
+    const unpack = options.unpack ?? true;
+    return `type=image,name=${canonicalImageReference(reference)},name-canonical=true,unpack=${unpack},compression=uncompressed`;
 }
 const digest = (bytes) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
 const descriptor = (bytes, mediaType) => ({ digest: digest(bytes), size: bytes.length, mediaType });

package/dist/harness/component-images.d.ts

@@ -5,10 +5,17 @@ export type KubernetesImage = string | (BuildImage & {
 });
 /** containerd/CRI use fully qualified names, including the default tag. */
 export declare function canonicalImageReference(reference: string): string;
+/** A `name@sha256:…` reference names one immutable image; a tag can move. */
+export declare function isDigestReference(reference: string): boolean;
 export declare function kubernetesImages(images: readonly KubernetesImage[]): {
     pulls: string[];
     builds: ComponentImageBuild[];
 };
+/** Prefix of every component image task name; `isComponentImageTask` is the test. */
+export declare const COMPONENT_IMAGE_TASK_PREFIX = "spectest-image-";
+/** A component image task builds for a component's own runtime (today: a
+ * `k3s()` cluster's containerd), never for a container dockerd starts. */
+export declare function isComponentImageTask(name: string): boolean;
 /** Stable internal task names, separate from the service graph. */
 export declare function componentBuildTasks<T extends ServiceConfig & {
     name: string;

package/dist/harness/component-images.js

@@ -24,6 +24,10 @@ export function canonicalImageReference(reference) {
         result += ":latest";
     return result;
 }
+/** A `name@sha256:…` reference names one immutable image; a tag can move. */
+export function isDigestReference(reference) {
+    return /@sha256:[a-f0-9]{64}$/.test(reference);
+}
 export function kubernetesImages(images) {
     if (!Array.isArray(images))
         throw new Error("k3s(): images must be an array of registry references or Dockerfile builds");
@@ -54,10 +58,17 @@ export function kubernetesImages(images) {
     }
     return { pulls, builds };
 }
+/** Prefix of every component image task name; `isComponentImageTask` is the test. */
+export const COMPONENT_IMAGE_TASK_PREFIX = "spectest-image-";
+/** A component image task builds for a component's own runtime (today: a
+ * `k3s()` cluster's containerd), never for a container dockerd starts. */
+export function isComponentImageTask(name) {
+    return name.startsWith(COMPONENT_IMAGE_TASK_PREFIX);
+}
 /** Stable internal task names, separate from the service graph. */
 export function componentBuildTasks(services) {
     return services.flatMap(svc => (svc.imageBuilds ?? []).map(build => ({
-        name: `spectest-image-${createHash("sha256").update(`${svc.name}\0${build.reference}`).digest("hex")}`,
+        name: `${COMPONENT_IMAGE_TASK_PREFIX}${createHash("sha256").update(`${svc.name}\0${build.reference}`).digest("hex")}`,
         owner: svc.name,
         reference: build.reference,
         image: build.image,

package/dist/harness/sticky-consumer.js

@@ -101,6 +101,12 @@ disabled_plugins = ["io.containerd.grpc.v1.cri"]
     dockerConfig["data-root"] = `${config.path}/docker`;
     dockerConfig.features = { "containerd-snapshotter": true };
     dockerConfig["storage-driver"] = "overlayfs";
+    // Pull up to ten layers at once instead of dockerd's default three. The
+    // containerd image store honours this since Docker 29.7.0; extraction is
+    // still one layer at a time, so this only moves the download side off
+    // the critical path of many-layer images. Set here as well as in the
+    // golden's daemon.json so an SDK release carries it without a rebuild.
+    dockerConfig["max-concurrent-downloads"] = 10;
     await writeFile(`${STICKY_RUNTIME}/docker.json`, JSON.stringify(dockerConfig));
     await writeFile(`${STICKY_RUNTIME}/buildkit.toml`, stickyBuildkitConfig(config, (await diskUsage(config.path)).reservedBytes));
     const start = async (unit, program, args) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@specific.dev/spectest",
-  "version": "0.131.13",
+  "version": "0.131.14",
   "description": "Spectest SDK for defining test environments in TypeScript.",
   "type": "module",
   "main": "./dist/index.js",

package/src/daemon.ts

@@ -5,7 +5,7 @@ import { phoneScope, phoneRecordingChunk, releasePhoneRecording, type PhoneRecor
 import { runWithBackgroundFailures } from "./harness/background-failures.js";
 import { generateHostCert as mintHostCert } from "./harness/certificate.js";
 import { containerdImageOutput } from "./harness/built-image.js";
-import { componentBuildTasks, COMPONENT_RUNTIME } from "./harness/component-images.js";
+import { componentBuildTasks, COMPONENT_RUNTIME, isComponentImageTask, isDigestReference } from "./harness/component-images.js";
 import { waitForAll } from "./harness/concurrency.js";
 import { updateServiceProgress, serviceProgressSnapshot, type ServiceProgress } from "./harness/service-progress.js";
 import { androidScope, androidRecordingChunk, releaseAndroidRecording, isAndroidApp, type AndroidRecording } from "./android.js";
@@ -749,6 +749,18 @@ function ensureLocalBuildkitd(): Promise<boolean> {
   return _localBuilder;
 }
 
+/** `true` when a digest-pinned reference is already complete in the store. */
+async function imagePresentByDigest(ref: string): Promise<boolean> {
+  if (!isDigestReference(ref)) return false;
+  // `inspect` succeeds only when every layer of the image for this
+  // platform is in the store; a partial or missing image falls through to
+  // the ordinary pull, which fetches what is absent. dockerd's containerd
+  // store also unpacks on demand at container creation, so a record whose
+  // snapshots were reclaimed is repaired there rather than failing.
+  const present = await docker(["image", "inspect", "--format", "{{.Id}}", ref], 30_000);
+  return present.code === 0;
+}
+
 /** Host directory backing one volume mount. Rules (notably which volumes
  *  survive the delta-restore teardown) live in `harness/volume-paths.ts`. */
 function resolveHostPath(service: string, vol: VolumeMount): string {
@@ -1214,6 +1226,18 @@ async function prepareServiceImageInner(
     // tags (`foo:latest`) at whatever the previous generation pulled, for
     // as long as the delta chain lives — a silent semantic divergence
     // from the cold build a delta restore must be equivalent to.
+    //
+    // The one reference that cannot float is a digest. `name@sha256:…`
+    // names exactly one image for as long as it exists, so a copy already
+    // in the store is the image the registry would return, and the
+    // round-trip buys nothing: skip it. Component images pinned by digest
+    // and user references pinned by digest take this path; a tag never does.
+    if (await imagePresentByDigest(ref)) {
+      progressService(svc.name, { status: "pulling", detail: `present by digest` });
+      const tagged = await docker(["tag", ref, tag], 30_000);
+      if (tagged.code !== 0) throw new Error(`docker tag ${ref} ${tag} failed: ${tagged.stderr.trim()}`);
+      return { tag };
+    }
     progressService(svc.name, { status: "pulling", detail: `pulling ${ref}` });
     let layers = 0;
     const pull = await shxStream("docker", ["pull", ref], 900_000, undefined, (line) => {
@@ -1368,11 +1392,30 @@ async function runServiceBuild(
       // containerd exporter replaces the tag directly, bypassing Docker's
       // dangling-image retention. Old versions must remain usable until our
       // image GC decides they are no longer needed (including by nested stores).
+      //
+      // `compression=uncompressed` (`containerdImageOutput`): the image never leaves this store, so
+      // gzip on export only to gunzip on unpack is pure cost — on Listen,
+      // 64 s compressing a 2.7 GB layer and 56 s decompressing it again,
+      // single-threaded, in the same guest that already held the layer as
+      // a build snapshot (CONTAINER_STORE.md, "Image delivery inside the
+      // guest"). With the default `force-compression=false` this applies
+      // to layers this build creates; a pulled base image's gzip blobs are
+      // reused as they are, so no existing layer is re-encoded or
+      // re-unpacked. The uncompressed blob is a second on-disk copy of the
+      // layer; the sticky headroom logic already budgets for blobs.
+      //
+      // `unpack`: dockerd needs the snapshot chain to start a container,
+      // so a service image is unpacked here rather than at `docker run`.
+      // A component image task (a `k3s()` build) is consumed only through
+      // the content store — `k3s-image-loader.ts` serves its blobs and the
+      // cluster's containerd unpacks them into its own store — so unpacking
+      // it in dockerd's store would be a third copy nobody reads.
+      const unpack = !isComponentImageTask(name);
       buildArgs = [
         "buildx", "build",
         "--builder", LOCAL_BUILDER_NAME,
         "--progress=plain",
-        "--output", containerdImageOutput(tag),
+        "--output", containerdImageOutput(tag, { unpack }),
         ...argFlags,
         "-f", dfPath, contextDir,
       ];

package/src/harness/built-image.ts

@@ -5,9 +5,17 @@ import { promisify } from "node:util";
 import { canonicalImageReference } from "./component-images.js";
 
 /** Containerd exports replace mutable tags without Docker's dangling-image
- * retention. Keep each result addressable until the SDK's image GC evicts it. */
-export function containerdImageOutput(reference: string): string {
-  return `type=image,name=${canonicalImageReference(reference)},name-canonical=true,unpack=true`;
+ * retention. Keep each result addressable until the SDK's image GC evicts it.
+ *
+ * Layers this build creates are stored uncompressed: the image never leaves
+ * the VM's store, so compressing on export only to decompress on unpack was
+ * pure cost (64 s + 56 s on Listen's 2.7 GB layer). `force-compression` stays
+ * off, so a pulled base image's blobs are reused as they are. `unpack` is
+ * false for images only a nested store reads (CONTAINER_STORE.md, "Image
+ * delivery inside the guest"). */
+export function containerdImageOutput(reference: string, options: { unpack?: boolean } = {}): string {
+  const unpack = options.unpack ?? true;
+  return `type=image,name=${canonicalImageReference(reference)},name-canonical=true,unpack=${unpack},compression=uncompressed`;
 }
 
 export interface Descriptor { digest: string; size: number; mediaType: string; platform?: { os: string; architecture: string } }

package/src/harness/component-images.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "bun:test";
-import { canonicalImageReference, componentBuildTasks, kubernetesImages } from "./component-images";
+import { canonicalImageReference, componentBuildTasks, isComponentImageTask, isDigestReference, kubernetesImages } from "./component-images";
 import { validateServiceImage, type BuildImage } from "../index";
 
 test("services and Kubernetes share all Dockerfile build options without a type discriminator", () => {
@@ -42,3 +42,21 @@ test("build task identity survives list reordering but is scoped to the owning c
   expect(() => validateServiceImage("app", { content: "FROM scratch", buildArgs: { "bad-key": "x" } })).toThrow("ARG");
   expect(() => validateServiceImage("app", { content: "FROM scratch", exclude: [1] as never })).toThrow("exclude");
 });
+
+test("component image tasks are recognised by name and services are not", () => {
+  const [task] = componentBuildTasks([{ name: "cluster", image: { type: "registry", reference: "rancher/k3s:v1.33.1-k3s1" },
+    imageBuilds: [{ reference: "registry.internal/app:dev", image: { type: "dockerfile", content: "FROM scratch" } }] } as any]);
+  expect(isComponentImageTask(task!.name)).toBe(true);
+  expect(isComponentImageTask("cluster")).toBe(false);
+  expect(isComponentImageTask("spectest-image")).toBe(false);
+});
+
+test("only a full sha256 digest reference is immutable", () => {
+  const digest = "a".repeat(64);
+  expect(isDigestReference(`postgres@sha256:${digest}`)).toBe(true);
+  expect(isDigestReference(`ghcr.io/org/app:1.2@sha256:${digest}`)).toBe(true);
+  expect(isDigestReference("postgres:16.4")).toBe(false);
+  expect(isDigestReference("postgres:latest")).toBe(false);
+  expect(isDigestReference(`postgres@sha256:${digest.slice(1)}`)).toBe(false);
+  expect(isDigestReference(`postgres@sha512:${digest}`)).toBe(false);
+});

package/src/harness/component-images.ts

@@ -26,6 +26,11 @@ export function canonicalImageReference(reference: string): string {
   return result;
 }
 
+/** A `name@sha256:…` reference names one immutable image; a tag can move. */
+export function isDigestReference(reference: string): boolean {
+  return /@sha256:[a-f0-9]{64}$/.test(reference);
+}
+
 export function kubernetesImages(images: readonly KubernetesImage[]): { pulls: string[]; builds: ComponentImageBuild[] } {
   if (!Array.isArray(images)) throw new Error("k3s(): images must be an array of registry references or Dockerfile builds");
   const pulls: string[] = [];
@@ -53,10 +58,19 @@ export function kubernetesImages(images: readonly KubernetesImage[]): { pulls: s
   return { pulls, builds };
 }
 
+/** Prefix of every component image task name; `isComponentImageTask` is the test. */
+export const COMPONENT_IMAGE_TASK_PREFIX = "spectest-image-";
+
+/** A component image task builds for a component's own runtime (today: a
+ * `k3s()` cluster's containerd), never for a container dockerd starts. */
+export function isComponentImageTask(name: string): boolean {
+  return name.startsWith(COMPONENT_IMAGE_TASK_PREFIX);
+}
+
 /** Stable internal task names, separate from the service graph. */
 export function componentBuildTasks<T extends ServiceConfig & { name: string }>(services: readonly T[]) {
   return services.flatMap(svc => (svc.imageBuilds ?? []).map(build => ({
-    name: `spectest-image-${createHash("sha256").update(`${svc.name}\0${build.reference}`).digest("hex")}`,
+    name: `${COMPONENT_IMAGE_TASK_PREFIX}${createHash("sha256").update(`${svc.name}\0${build.reference}`).digest("hex")}`,
     owner: svc.name,
     reference: build.reference,
     image: build.image,

package/src/harness/sticky-consumer.ts

@@ -104,6 +104,12 @@ disabled_plugins = ["io.containerd.grpc.v1.cri"]
   dockerConfig["data-root"] = `${config.path}/docker`;
   dockerConfig.features = { "containerd-snapshotter": true };
   dockerConfig["storage-driver"] = "overlayfs";
+  // Pull up to ten layers at once instead of dockerd's default three. The
+  // containerd image store honours this since Docker 29.7.0; extraction is
+  // still one layer at a time, so this only moves the download side off
+  // the critical path of many-layer images. Set here as well as in the
+  // golden's daemon.json so an SDK release carries it without a rebuild.
+  dockerConfig["max-concurrent-downloads"] = 10;
   await writeFile(`${STICKY_RUNTIME}/docker.json`, JSON.stringify(dockerConfig));
   await writeFile(`${STICKY_RUNTIME}/buildkit.toml`, stickyBuildkitConfig(config, (await diskUsage(config.path)).reservedBytes));
   const start = async (unit: string, program: string, args: string[]) => {