@specific.dev/spectest 0.131.4 → 0.131.6

package/dist/daemon.js

@@ -47,6 +47,7 @@ import { isDesktopApp } from "./desktop.js";
 // each easy to restate subtly differently.
 import { buildArgFlags, buildContentKey as computeBuildContentKey, buildTargetFlags, imageTag, isGeneratedDockerignore, pickProjectIgnore, serviceDockerignore as composeServiceDockerignore, unionDockerignore, } from "./harness/build-context.js";
 import { startServices } from "./harness/service-graph.js";
+import { ensureStickyBuildHeadroom } from "./harness/sticky-consumer.js";
 import { fakeFunctions, fakeFunctionParameters, callFakeFunction } from "./harness/fake-functions.js";
 import { dispatchFake } from "./harness/fake-websocket.js";
 import { casesMetadata as catalogueCases, groupsMetadata as catalogueGroups, } from "./harness/catalogue.js";
@@ -2667,6 +2668,7 @@ async function bootstrapInner(prepared, identity) {
     if (identity !== undefined) {
         // No application services, setup hooks, ingress, or runtime state in the builder.
         await Promise.all(prep.values());
+        await ensureStickyBuildHeadroom();
         const images = Object.create(null);
         for (const svc of tasks) {
             const out = await docker(["image", "inspect", "--format", "{{.Id}}", tags.get(svc.name)], 30_000);
@@ -2759,6 +2761,7 @@ async function bootstrapInner(prepared, identity) {
             ti.setupMs = svc.setup ? Date.now() - tSetup : 0;
         }
     });
+    await ensureStickyBuildHeadroom();
     // Containers now have IPs — re-seed so service-targeted wildcards (e.g.
     // k3s ingressDomains → the cluster container) resolve. Captured into the
     // warm template, so warm starts inherit the resolved entries.

package/dist/harness/sticky-consumer.d.ts

@@ -19,8 +19,18 @@ export declare function validateStickyConfig(config: StickyConsumerConfig): void
  * only minFreeSpace, BuildKit 0.32.2 sets keepBytes == totalSize when healthy,
  * and pruneOnce's strict '<' check can still evict a record on every GC pass.
  * Actual filesystem capacity is smaller than the device due to metadata. */
-export declare function stickyBuildkitConfig(config: StickyConsumerConfig): string;
+export declare function stickyBuildkitConfig(config: StickyConsumerConfig, reservedBytes?: number): string;
 export declare function initializeStickyConsumer(config: StickyConsumerConfig): Promise<void>;
+/** Builds can finish before automatic GC restores its free-space reserve.
+ * Reclaim only unused BuildKit records before project setup adds runtime data.
+ * Docker images, containers, volumes and nested Kubernetes stores stay live.
+ */
+export declare function ensureBuildHeadroom(config: StickyConsumerConfig, run?: typeof exec, usage?: () => Promise<{
+    freeBytes: number;
+    totalBytes: number;
+    reservedBytes: number;
+}>): Promise<void>;
+export declare function ensureStickyBuildHeadroom(): Promise<void>;
 /** Filesystem pressure takes precedence over cache age/BuildKit record sizes.
  * Only called after runtime containers/volumes have been removed on the
  * isolated publication clone. Never removes files behind the daemons' backs.
@@ -28,6 +38,7 @@ export declare function initializeStickyConsumer(config: StickyConsumerConfig):
 export declare function collectStickyCache(config: StickyConsumerConfig, retention: StickyImageRetention, run?: typeof exec, usage?: () => Promise<{
     freeBytes: number;
     totalBytes: number;
+    reservedBytes?: number;
 }>, kubernetes?: KubernetesCacheMaintenance): Promise<void>;
 /** Only on an isolated maintenance clone; never on the live ready environment. */
 export declare function prepareStickyConsumer(artifact?: boolean): Promise<{

package/dist/harness/sticky-consumer.js

@@ -45,7 +45,7 @@ export function validateStickyConfig(config) {
  * only minFreeSpace, BuildKit 0.32.2 sets keepBytes == totalSize when healthy,
  * and pruneOnce's strict '<' check can still evict a record on every GC pass.
  * Actual filesystem capacity is smaller than the device due to metadata. */
-export function stickyBuildkitConfig(config) {
+export function stickyBuildkitConfig(config, reservedBytes = 0) {
     validateStickyConfig(config);
     return `root = "${config.path}/buildkit"
 # This standalone BuildKit does not inherit dockerd's registry mirrors.
@@ -65,7 +65,7 @@ export function stickyBuildkitConfig(config) {
 [[worker.containerd.gcpolicy]]
   all = true
   maxUsedSpace = ${config.capacityBytes}
-  minFreeSpace = ${config.headroomBytes}
+  minFreeSpace = ${config.headroomBytes + reservedBytes}
 `;
 }
 export async function initializeStickyConsumer(config) {
@@ -102,7 +102,7 @@ disabled_plugins = ["io.containerd.grpc.v1.cri"]
     dockerConfig.features = { "containerd-snapshotter": true };
     dockerConfig["storage-driver"] = "overlayfs";
     await writeFile(`${STICKY_RUNTIME}/docker.json`, JSON.stringify(dockerConfig));
-    await writeFile(`${STICKY_RUNTIME}/buildkit.toml`, stickyBuildkitConfig(config));
+    await writeFile(`${STICKY_RUNTIME}/buildkit.toml`, stickyBuildkitConfig(config, (await diskUsage(config.path)).reservedBytes));
     const start = async (unit, program, args) => {
         await exec("systemd-run", ["--quiet", "--unit", unit, "-p", "TimeoutStopSec=30",
             // Pinned BuildKit 0.32.2 returns 1 after its normal first-SIGTERM
@@ -121,14 +121,47 @@ disabled_plugins = ["io.containerd.grpc.v1.cri"]
     await exec("docker", ["buildx", "create", "--name", "spectest-local", "--driver", "remote", "tcp://127.0.0.1:1234"]);
     await exec("docker", ["buildx", "inspect", "--bootstrap", "spectest-local"]);
 }
+async function diskUsage(path) {
+    const fs = await statfs(path);
+    return { freeBytes: fs.bavail * fs.bsize, totalBytes: fs.blocks * fs.bsize,
+        reservedBytes: Math.max(0, fs.bfree - fs.bavail) * fs.bsize };
+}
+// BuildKit 0.32.2 compares MinFreeSpace against statfs.bfree, whereas
+// applications and kubelet need bavail. Include ext4's reserved blocks so
+// pruning restores the required usable space instead of stopping early.
+function pruneTarget(config, usage) {
+    return String(Math.ceil((config.headroomBytes + (usage.reservedBytes ?? 0)) / 1e6));
+}
+/** Builds can finish before automatic GC restores its free-space reserve.
+ * Reclaim only unused BuildKit records before project setup adds runtime data.
+ * Docker images, containers, volumes and nested Kubernetes stores stay live.
+ */
+export async function ensureBuildHeadroom(config, run = exec, usage = () => diskUsage(config.path)) {
+    validateStickyConfig(config);
+    let fs = await usage();
+    for (let pass = 0; fs.freeBytes < config.headroomBytes && pass < 3; pass++) {
+        await run("buildctl", ["--addr", "tcp://127.0.0.1:1234", "prune", "--all", "--free-storage",
+            pruneTarget(config, fs)], 120_000);
+        fs = await usage();
+    }
+    if (fs.freeBytes < config.headroomBytes) {
+        throw new Error(`environment build left insufficient disk headroom: ${Math.floor(fs.freeBytes / 2 ** 20)} MiB available, ${Math.ceil(config.headroomBytes / 2 ** 20)} MiB required`);
+    }
+}
+export async function ensureStickyBuildHeadroom() {
+    const saved = await readFile(`${STICKY_RUNTIME}/consumer.json`, "utf8").catch((error) => {
+        if (error.code === "ENOENT")
+            return undefined;
+        throw error;
+    });
+    if (saved !== undefined)
+        await ensureBuildHeadroom(JSON.parse(saved));
+}
 /** Filesystem pressure takes precedence over cache age/BuildKit record sizes.
  * Only called after runtime containers/volumes have been removed on the
  * isolated publication clone. Never removes files behind the daemons' backs.
  */
-export async function collectStickyCache(config, retention, run = exec, usage = async () => {
-    const fs = await statfs(config.path);
-    return { freeBytes: fs.bavail * fs.bsize, totalBytes: fs.blocks * fs.bsize };
-}, kubernetes) {
+export async function collectStickyCache(config, retention, run = exec, usage = () => diskUsage(config.path), kubernetes) {
     validateStickyConfig(config);
     const buildctl = ["--addr", "tcp://127.0.0.1:1234", "prune"];
     // Docker 29's containerd until filter uses local image-record creation;
@@ -156,7 +189,7 @@ export async function collectStickyCache(config, retention, run = exec, usage =
     // a per-consumer cap evicts useful cache even when the disk is mostly empty.
     const pruneBuildkit = async () => {
         await run("buildctl", [...buildctl, "--all", "--free-storage",
-            String(Math.ceil(config.headroomBytes / 1e6))], 120_000);
+            pruneTarget(config, fs)], 120_000);
         fs = await usage();
     };
     await pruneBuildkit();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@specific.dev/spectest",
-  "version": "0.131.4",
+  "version": "0.131.6",
   "description": "Spectest SDK for defining test environments in TypeScript.",
   "type": "module",
   "main": "./dist/index.js",

package/src/daemon.ts

@@ -99,6 +99,7 @@ import {
   unionDockerignore,
 } from "./harness/build-context.js";
 import { startServices } from "./harness/service-graph.js";
+import { ensureStickyBuildHeadroom } from "./harness/sticky-consumer.js";
 import { fakeFunctions, fakeFunctionParameters, callFakeFunction } from "./harness/fake-functions.js";
 import { dispatchFake } from "./harness/fake-websocket.js";
 import {
@@ -3167,6 +3168,7 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
   if (identity !== undefined) {
     // No application services, setup hooks, ingress, or runtime state in the builder.
     await Promise.all(prep.values());
+    await ensureStickyBuildHeadroom();
     const images: PreparedImages["images"] = Object.create(null);
     for (const svc of tasks) {
       const out = await docker(["image", "inspect", "--format", "{{.Id}}", tags.get(svc.name)!], 30_000);
@@ -3258,6 +3260,8 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
     }
   });
 
+  await ensureStickyBuildHeadroom();
+
   // Containers now have IPs — re-seed so service-targeted wildcards (e.g.
   // k3s ingressDomains → the cluster container) resolve. Captured into the
   // warm template, so warm starts inherit the resolved entries.

package/src/harness/sticky-consumer.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "bun:test";
-import { STICKY_COMPATIBILITY, validateStickyConfig, collectStickyCache, stickyBuildkitConfig } from "./sticky-consumer";
+import { STICKY_COMPATIBILITY, validateStickyConfig, collectStickyCache, stickyBuildkitConfig, ensureBuildHeadroom } from "./sticky-consumer";
 
 import type { StickyImageRetention } from "./sticky-image-retention";
 
@@ -12,6 +12,41 @@ const config = { path: "/var/lib/spectest-sticky", capacityBytes: 256 * GiB,
   headroomBytes: 32 * GiB,
   compatibility: STICKY_COMPATIBILITY };
 
+test("completed builds preserve healthy caches and restore only missing runtime headroom", async () => {
+  let free = 32 * GiB;
+  const calls: string[][] = [];
+  const run = async (command: string, args: string[]) => {
+    calls.push([command, ...args]);
+    free += 10 * GiB;
+    return "";
+  };
+  await ensureBuildHeadroom(config, run, async () => ({ freeBytes: free, totalBytes: 251 * GiB, reservedBytes: 0 }));
+  expect(calls).toEqual([]);
+  // Tangent's build left 13 GiB before native scratch crossed the
+  // kubelet eviction threshold. Containerd reclamation can take passes.
+  free = 13 * GiB;
+  await ensureBuildHeadroom(config, run, async () => ({ freeBytes: free, totalBytes: 251 * GiB, reservedBytes: 0 }));
+  expect(free).toBe(33 * GiB);
+  expect(calls).toEqual(Array.from({ length: 2 }, () => ["buildctl", "--addr", "tcp://127.0.0.1:1234",
+    "prune", "--all", "--free-storage", String(Math.ceil(32 * GiB / 1e6))]));
+});
+
+test("unreclaimable build storage fails before project setup without deleting runtime state", async () => {
+  const commands: string[] = [];
+  await expect(ensureBuildHeadroom(config, async (command) => {
+    commands.push(command);
+    return "";
+  }, async () => ({ freeBytes: 13 * GiB, totalBytes: 251 * GiB, reservedBytes: 0 }))).rejects.toThrow("13312 MiB available, 32768 MiB required");
+  expect(commands).toEqual(["buildctl", "buildctl", "buildctl"]);
+});
+
+test("build headroom checks propagate failed GC and disk observations", async () => {
+  await expect(ensureBuildHeadroom(config, async () => { throw new Error("GC unavailable"); },
+    async () => ({ freeBytes: 13 * GiB, totalBytes: 251 * GiB, reservedBytes: 0 }))).rejects.toThrow("GC unavailable");
+  await expect(ensureBuildHeadroom(config, async () => { throw new Error("must not run"); },
+    async () => { throw new Error("filesystem unavailable"); })).rejects.toThrow("filesystem unavailable");
+});
+
 test("sticky consumer validates disk capacity and required free space", () => {
   expect(() => validateStickyConfig(config)).not.toThrow();
   // Retention has no per-consumer size caps.
@@ -160,3 +195,30 @@ test("GC does not remove an image while it supplies the Kubernetes maintenance d
   });
   expect(removed).toEqual(["old"]);
 });
+
+
+test("BuildKit thresholds include ext4 reserved blocks for build and publication GC", async () => {
+  // The failed Tangent build had 23 GiB available but more than 32 GiB
+  // physically free, so an unadjusted BuildKit request reclaimed nothing.
+  const reservedBytes = 13 * GiB;
+  const worker = (Bun.TOML.parse(stickyBuildkitConfig(config, reservedBytes)) as any).worker.containerd;
+  expect(worker.gcpolicy[0].minFreeSpace).toBe(45 * GiB);
+  for (const build of [true, false]) {
+    let freeBytes = 23 * GiB;
+    const requests: number[] = [];
+    const run = async (_command: string, args: string[]) => {
+      const index = args.indexOf("--free-storage");
+      if (index >= 0) {
+        requests.push(Number(args[index + 1]) * 1e6);
+        freeBytes = 33 * GiB;
+      }
+      return "";
+    };
+    const usage = async () => ({ freeBytes, totalBytes: 251 * GiB, reservedBytes });
+    if (build) await ensureBuildHeadroom(config, run, usage);
+    else await collectStickyCache(config, retention(), run, usage);
+    expect(requests).toHaveLength(1);
+    expect(requests[0]).toBeGreaterThanOrEqual(45 * GiB);
+    expect(requests[0]).toBeLessThan(45 * GiB + 1e6);
+  }
+});

package/src/harness/sticky-consumer.ts

@@ -50,7 +50,7 @@ export function validateStickyConfig(config: StickyConsumerConfig): void {
  * only minFreeSpace, BuildKit 0.32.2 sets keepBytes == totalSize when healthy,
  * and pruneOnce's strict '<' check can still evict a record on every GC pass.
  * Actual filesystem capacity is smaller than the device due to metadata. */
-export function stickyBuildkitConfig(config: StickyConsumerConfig): string {
+export function stickyBuildkitConfig(config: StickyConsumerConfig, reservedBytes = 0): string {
   validateStickyConfig(config);
   return `root = "${config.path}/buildkit"
 # This standalone BuildKit does not inherit dockerd's registry mirrors.
@@ -70,7 +70,7 @@ export function stickyBuildkitConfig(config: StickyConsumerConfig): string {
 [[worker.containerd.gcpolicy]]
   all = true
   maxUsedSpace = ${config.capacityBytes}
-  minFreeSpace = ${config.headroomBytes}
+  minFreeSpace = ${config.headroomBytes + reservedBytes}
 `;
 }
 
@@ -105,7 +105,7 @@ disabled_plugins = ["io.containerd.grpc.v1.cri"]
   dockerConfig.features = { "containerd-snapshotter": true };
   dockerConfig["storage-driver"] = "overlayfs";
   await writeFile(`${STICKY_RUNTIME}/docker.json`, JSON.stringify(dockerConfig));
-  await writeFile(`${STICKY_RUNTIME}/buildkit.toml`, stickyBuildkitConfig(config));
+  await writeFile(`${STICKY_RUNTIME}/buildkit.toml`, stickyBuildkitConfig(config, (await diskUsage(config.path)).reservedBytes));
   const start = async (unit: string, program: string, args: string[]) => {
     await exec("systemd-run", ["--quiet", "--unit", unit, "-p", "TimeoutStopSec=30",
       // Pinned BuildKit 0.32.2 returns 1 after its normal first-SIGTERM
@@ -125,6 +125,48 @@ disabled_plugins = ["io.containerd.grpc.v1.cri"]
   await exec("docker", ["buildx", "inspect", "--bootstrap", "spectest-local"]);
 }
 
+async function diskUsage(path: string) {
+  const fs = await statfs(path);
+  return { freeBytes: fs.bavail * fs.bsize, totalBytes: fs.blocks * fs.bsize,
+    reservedBytes: Math.max(0, fs.bfree - fs.bavail) * fs.bsize };
+}
+
+// BuildKit 0.32.2 compares MinFreeSpace against statfs.bfree, whereas
+// applications and kubelet need bavail. Include ext4's reserved blocks so
+// pruning restores the required usable space instead of stopping early.
+function pruneTarget(config: StickyConsumerConfig, usage: { reservedBytes?: number }): string {
+  return String(Math.ceil((config.headroomBytes + (usage.reservedBytes ?? 0)) / 1e6));
+}
+
+/** Builds can finish before automatic GC restores its free-space reserve.
+ * Reclaim only unused BuildKit records before project setup adds runtime data.
+ * Docker images, containers, volumes and nested Kubernetes stores stay live.
+ */
+export async function ensureBuildHeadroom(
+  config: StickyConsumerConfig,
+  run: typeof exec = exec,
+  usage = () => diskUsage(config.path),
+): Promise<void> {
+  validateStickyConfig(config);
+  let fs = await usage();
+  for (let pass = 0; fs.freeBytes < config.headroomBytes && pass < 3; pass++) {
+    await run("buildctl", ["--addr", "tcp://127.0.0.1:1234", "prune", "--all", "--free-storage",
+      pruneTarget(config, fs)], 120_000);
+    fs = await usage();
+  }
+  if (fs.freeBytes < config.headroomBytes) {
+    throw new Error(`environment build left insufficient disk headroom: ${Math.floor(fs.freeBytes / 2 ** 20)} MiB available, ${Math.ceil(config.headroomBytes / 2 ** 20)} MiB required`);
+  }
+}
+
+export async function ensureStickyBuildHeadroom(): Promise<void> {
+  const saved = await readFile(`${STICKY_RUNTIME}/consumer.json`, "utf8").catch((error: NodeJS.ErrnoException) => {
+    if (error.code === "ENOENT") return undefined;
+    throw error;
+  });
+  if (saved !== undefined) await ensureBuildHeadroom(JSON.parse(saved));
+}
+
 /** Filesystem pressure takes precedence over cache age/BuildKit record sizes.
  * Only called after runtime containers/volumes have been removed on the
  * isolated publication clone. Never removes files behind the daemons' backs.
@@ -133,10 +175,7 @@ export async function collectStickyCache(
   config: StickyConsumerConfig,
   retention: StickyImageRetention,
   run: typeof exec = exec,
-  usage = async (): Promise<{ freeBytes: number; totalBytes: number }> => {
-    const fs = await statfs(config.path);
-    return { freeBytes: fs.bavail * fs.bsize, totalBytes: fs.blocks * fs.bsize };
-  },
+  usage: () => Promise<{ freeBytes: number; totalBytes: number; reservedBytes?: number }> = () => diskUsage(config.path),
   kubernetes?: KubernetesCacheMaintenance,
 ): Promise<void> {
   validateStickyConfig(config);
@@ -164,7 +203,7 @@ export async function collectStickyCache(
   // a per-consumer cap evicts useful cache even when the disk is mostly empty.
   const pruneBuildkit = async () => {
     await run("buildctl", [...buildctl, "--all", "--free-storage",
-      String(Math.ceil(config.headroomBytes / 1e6))], 120_000);
+      pruneTarget(config, fs)], 120_000);
     fs = await usage();
   };
   await pruneBuildkit();