@specific.dev/spectest 0.90.3 → 0.91.1

package/src/harness/sticky-kubernetes.ts

@@ -0,0 +1,165 @@
+/** Kubernetes owns a separate containerd store; Docker/BuildKit GC cannot
+ * collect it. Maintenance runs only on the isolated publication clone. */
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, readFile, readdir, rename, writeFile } from "node:fs/promises";
+import { STICKY_ROOT } from "./sticky-paths.js";
+
+export type CacheCommand = (program: string, args: string[], timeout?: number) => Promise<string>;
+const STORE_TARGET = "/var/lib/rancher/k3s/agent/containerd";
+const SOCKET = "/run/spectest-cache.sock";
+const ROOT = `${STICKY_ROOT}/kubernetes`;
+const IMAGE_ID = /^sha256:[a-f0-9]{64}$/;
+const SERVICE_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/;
+interface StoreDescriptor { schema: 1; image: string; updatedAt: number }
+
+export function stickyKubernetesDirectory(service: string, version: string): string {
+  // Version changes must not open old metadata with a different containerd.
+  const layout = createHash("sha256").update(`overlayfs-v1:${version}`).digest("hex").slice(0, 16);
+  return `${ROOT}/${service}/${layout}`;
+}
+
+const runCommand: CacheCommand = async (program, args, timeout = 30_000) =>
+  (await promisify(execFile)(program, args, { timeout })).stdout.trim();
+
+export async function registerStickyKubernetesStore(
+  service: string, version: string, run: CacheCommand = runCommand,
+): Promise<void> {
+  if (!SERVICE_NAME.test(service)) throw new Error(`invalid Kubernetes cache service name: ${service}`);
+  // Respect custom Dockerfile overrides of the component's image.
+  const image = (await run("docker", ["inspect", "--format", "{{.Image}}", service])).trim();
+  if (!IMAGE_ID.test(image)) throw new Error(`cannot identify Kubernetes runtime image for ${service}`);
+  const directory = stickyKubernetesDirectory(service, version);
+  await mkdir(directory, { recursive: true });
+  const temporary = `${directory}/store.${randomUUID()}.tmp`;
+  await writeFile(temporary, JSON.stringify({ schema: 1, image, updatedAt: Date.now() } satisfies StoreDescriptor));
+  await rename(temporary, `${directory}/store.json`);
+}
+
+/** Standalone containerd never starts CRI/kubelet or any inherited workload.
+ * Native API deletion preserves content and committed image snapshots while
+ * releasing sandbox, container, lease and writable snapshot references. */
+export const KUBERNETES_CACHE_MAINTENANCE_SH = `set -eu
+exec 2>&1
+cat > /run/cache.toml <<'CONFIG'
+version = 2
+disabled_plugins = ["io.containerd.grpc.v1.cri", "io.containerd.cri.v1.runtime", "io.containerd.cri.v1.images"]
+CONFIG
+containerd --config /run/cache.toml --root ${STORE_TARGET} --state /run/cache-state --address ${SOCKET} > /run/cache.log 2>&1 &
+pid=$!
+stop() { kill "$pid"; wait "$pid"; }
+trap 'stop; exit 0' TERM INT
+trap 'kill "$pid" 2>/dev/null || true' EXIT
+c() { ctr --address ${SOCKET} --namespace k8s.io "$@"; }
+i=0
+until c version >/dev/null 2>&1; do
+  kill -0 "$pid" || { cat /run/cache.log; exit 1; }
+  i=$((i+1)); [ "$i" -lt 300 ] || { cat /run/cache.log; exit 1; }
+  sleep 0.1
+done
+ids=$(c containers ls -q)
+if [ -n "$ids" ]; then c containers rm $ids; fi
+# 1.7 has no sandbox store; 2.x does.
+case "$(containerd --version)" in
+  *v1.*) ;;
+  *) sandboxes=$(c sandboxes ls)
+     ids=$(printf '%s\\n' "$sandboxes" | awk 'NR > 1 {print $1}')
+     if [ -n "$ids" ]; then c sandboxes rm --force $ids; fi ;;
+esac
+ids=$(c leases ls -q)
+if [ -n "$ids" ]; then c leases rm $ids; fi
+# Remove children before parents, and tolerate asynchronous GC winning a race.
+previous=''
+while :; do
+  snapshots=$(c snapshots --snapshotter overlayfs ls)
+  ids=$(printf '%s\\n' "$snapshots" | awk 'NR > 1 && $NF != "Committed" {print $1}')
+  [ -n "$ids" ] || break
+  [ "$ids" != "$previous" ] || { echo 'cannot clean Kubernetes writable snapshots' >&2; exit 1; }
+  previous="$ids"
+  for id in $ids; do c snapshots --snapshotter overlayfs rm "$id" >/dev/null 2>&1 || true; done
+done
+touch /run/cache-ready
+while :; do sleep 1 & wait $!; done
+`;
+
+interface Store { directory: string; descriptor: StoreDescriptor; container: string; active: boolean }
+export interface KubernetesCacheMaintenance {
+  /** Outer images needed by live maintenance containers must not be evicted. */
+  protectedImages: Set<string>;
+  /** Evicts one complete image store, oldest service/version first. */
+  pruneOldest(): Promise<boolean>;
+  close(): Promise<void>;
+}
+
+export async function openKubernetesCacheMaintenance(run: CacheCommand, root = ROOT): Promise<KubernetesCacheMaintenance> {
+  const stores: Store[] = [];
+  const protectedImages = new Set<string>();
+  const closeStore = async (store: Store) => {
+    if (!store.active) return;
+    await run("docker", ["stop", "--time", "30", store.container], 45_000);
+    const status = JSON.parse(await run("docker", ["inspect", "--format", "{{json .State}}", store.container]));
+    if (status.ExitCode !== 0 || status.OOMKilled) throw new Error(`unclean Kubernetes cache shutdown: ${store.container}`);
+    await run("docker", ["rm", store.container]);
+    store.active = false;
+    if (!stores.some(other => other.active && other.descriptor.image === store.descriptor.image)) protectedImages.delete(store.descriptor.image);
+    // These contain runtime state, not image content or unpacked layers.
+    await run("rm", ["-rf", ...["io.containerd.runtime.v2.task", "io.containerd.grpc.v1.cri", "io.containerd.cri.v1.runtime", "tmpmounts"]
+      .map(child => `${store.directory}/containerd/${child}`)]);
+  };
+  const close = async () => {
+    const failures: unknown[] = [];
+    for (const store of stores) {
+      try { await closeStore(store); } catch (error) { failures.push(error); }
+    }
+    if (failures.length) throw new AggregateError(failures, "Kubernetes cache shutdown failed");
+  };
+  const entries = await readdir(root, { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => {
+    if (error.code === "ENOENT") return [];
+    throw error;
+  });
+  try {
+    for (const service of entries) {
+      if (!service.isDirectory() || !SERVICE_NAME.test(service.name)) continue;
+      for (const layout of await readdir(`${root}/${service.name}`, { withFileTypes: true })) {
+        if (!layout.isDirectory() || !/^[a-f0-9]{16}$/.test(layout.name)) continue;
+        const directory = `${root}/${service.name}/${layout.name}`;
+        const descriptor: StoreDescriptor = JSON.parse(await readFile(`${directory}/store.json`, "utf8"));
+        if (descriptor.schema !== 1 || !IMAGE_ID.test(descriptor.image) || !Number.isSafeInteger(descriptor.updatedAt)) {
+          throw new Error(`invalid Kubernetes cache descriptor: ${directory}`);
+        }
+        const container = `spectest-k8s-cache-${randomUUID()}`;
+        await run("docker", ["run", "-d", "--name", container, "--network", "none", "--privileged", "--stop-signal", "SIGTERM", "--tmpfs", "/run", "--tmpfs", "/var/run",
+          "--mount", `type=bind,source=${directory}/containerd,target=${STORE_TARGET}`,
+          "--entrypoint", "/bin/sh", descriptor.image, "-c", KUBERNETES_CACHE_MAINTENANCE_SH]);
+        stores.push({ directory, descriptor, container, active: true });
+        protectedImages.add(descriptor.image);
+        try {
+          await run("docker", ["exec", container, "sh", "-c", "i=0; until [ -f /run/cache-ready ]; do i=$((i+1)); [ $i -lt 900 ] || exit 1; sleep 0.1; done"], 100_000);
+        } catch (error) {
+          const logs = await run("docker", ["logs", "--tail", "40", container]).catch(() => "");
+          throw new Error(`Kubernetes cache preparation failed for ${directory}: ${logs}`, { cause: error });
+        }
+      }
+    }
+    stores.sort((a, b) => a.descriptor.updatedAt - b.descriptor.updatedAt || a.directory.localeCompare(b.directory));
+    let next = 0;
+    const pruneOldest = async () => {
+      const store = stores[next++];
+      if (!store) return false;
+      await run("docker", ["exec", store.container, "sh", "-ec",
+        `ids=$(ctr -a ${SOCKET} -n k8s.io images ls -q); if [ -n "$ids" ]; then ctr -a ${SOCKET} -n k8s.io images rm --sync $ids; fi`], 120_000);
+      await closeStore(store);
+      // No writer remains. Discard the now-empty cache and its descriptor so
+      // obsolete k3s binaries can also be collected by the outer Docker GC.
+      await run("rm", ["-rf", store.directory]);
+      return true;
+    };
+    // Abandoned services/versions should not occupy the disk indefinitely.
+    while (stores[next] && stores[next]!.descriptor.updatedAt < Date.now() - 7 * 24 * 60 * 60 * 1000) await pruneOldest();
+    return { protectedImages, pruneOldest, close };
+  } catch (error) {
+    await close().catch(() => {});
+    throw error;
+  }
+}

package/src/harness/sticky-paths.ts

@@ -0,0 +1,18 @@
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+
+export const STICKY_ROOT = "/var/lib/spectest-sticky";
+export const STICKY_RUNTIME = "/run/spectest-sticky";
+
+/** Written only after the supervisor mounts this project's private disk. */
+export function stickyDiskPathSync(manifest = `${STICKY_RUNTIME}/consumer.json`): string | null {
+  try {
+    const config = JSON.parse(readFileSync(manifest, "utf8"));
+    return config.path === STICKY_ROOT ? STICKY_ROOT : null;
+  } catch { return null; }
+}
+
+export function isOnStickyDisk(path: string, root = stickyDiskPathSync()): boolean {
+  const absolute = resolve(path);
+  return root !== null && (absolute === root || absolute.startsWith(`${root}/`));
+}

package/src/index.ts

@@ -253,6 +253,8 @@ export interface EnvironmentConfig<S extends ServicesMap = ServicesMap> {
 export interface ServiceConfig {
   /** Image definition: registry pull or inline build. */
   image: ServiceImage;
+  /** @internal Images built by the harness and loaded into this component. */
+  imageBuilds?: readonly ComponentImageBuild[];
   /**
    * Shell command to run (sh -c). Defaults to the image's CMD. NB: this
    * **replaces the image's entrypoint** with `/bin/sh -c` — an
@@ -1158,10 +1160,13 @@ export interface ServiceTls {
   port: number;
 }
 
-export type ServiceImage =
+/** @internal Normalized wire representation; legacy definitions remain accepted. */
+export type NormalizedServiceImage =
   | { type: "registry"; reference: string }
   | {
       type: "dockerfile";
+      /** Optional full local image reference. */
+      reference?: string;
       /**
        * Path of a Dockerfile in your repo, relative to the project root
        * (where `spectest/` lives) — `"Dockerfile"`, `"apps/api/Dockerfile"`.
@@ -1204,6 +1209,8 @@ export type ServiceImage =
     }
   | {
       type: "dockerfile";
+      /** Optional full local image reference. */
+      reference?: string;
       /**
        * Dockerfile contents, built as if the file sat outside the build
        * context (it is never written into your repo). The context is the
@@ -1233,6 +1240,38 @@ export type ServiceImage =
       buildArgs?: Readonly<Record<string, string>>;
     };
 
+/** Dockerfile build options shared by services and Kubernetes images. */
+export type BuildImage = (
+  | Omit<Extract<NormalizedServiceImage, { type: "dockerfile"; path: string }>, "type">
+  | Omit<Extract<NormalizedServiceImage, { type: "dockerfile"; content: string }>, "type">
+) & {
+  /** Full image reference, e.g. "k8s.internal:5000/runtime:spectest".
+   * Optional for a service; required for a Kubernetes build. This names a
+   * local image and does not publish it to a registry. */
+  reference?: string;
+};
+
+/** A registry reference or Dockerfile build. Tagged objects from older SDKs
+ * remain accepted for compatibility; new definitions do not need `type`. */
+export type ServiceImage = string | BuildImage | NormalizedServiceImage;
+
+/** @internal Keep the wire format compatible with existing control planes. */
+export function normalizeServiceImage(image: ServiceImage): NormalizedServiceImage {
+  if (typeof image === "string") return { type: "registry", reference: image };
+  if (!image || typeof image !== "object" || Array.isArray(image)) throw new Error("image must be a registry reference or Dockerfile build");
+  if ("type" in image) {
+    if (image.type !== "registry" && image.type !== "dockerfile") throw new Error("unsupported image type");
+    return image;
+  }
+  return { ...image, type: "dockerfile" };
+}
+
+/** @internal Serializes build-only work with the owning component. */
+export interface ComponentImageBuild {
+  reference: string;
+  image: BuildImage;
+}
+
 /** A Dockerfile `ARG` name: what `docker build --build-arg` accepts as a key. */
 const BUILD_ARG_NAME_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
 
@@ -1274,8 +1313,13 @@ const BUILD_TARGET_RE = /^[^\s]+$/;
  * checks go through the project-file resolver, so the same rules as
  * `ctx.readProjectFile` apply.
  */
-export function validateServiceImage(serviceName: string, image: ServiceImage): ServiceImage {
+export function validateServiceImage(serviceName: string, input: ServiceImage): NormalizedServiceImage {
+  const image = normalizeServiceImage(input);
+  const reference = typeof input === "string" ? input : "reference" in input ? input.reference : undefined;
+  if (reference !== undefined && (typeof reference !== "string" || !reference || reference.startsWith("-") || /[\s\x00]/.test(reference))) throw new Error(`service "${serviceName}" image reference must be a non-empty image name without whitespace`);
   if (image.type !== "dockerfile") return image;
+  if (reference?.includes("@")) throw new Error(`service "${serviceName}" build reference must be an image name/tag, not a digest`);
+  if (image.exclude !== undefined && (!Array.isArray(image.exclude) || image.exclude.some(e => typeof e !== "string"))) throw new Error(`service "${serviceName}" image exclude must be an array of glob patterns`);
   validateBuildArgs(serviceName, image.buildArgs);
   const hasPath = typeof image.path === "string";
   const hasContent = typeof image.content === "string";
@@ -1286,7 +1330,7 @@ export function validateServiceImage(serviceName: string, image: ServiceImage):
   }
   if (!hasPath && !hasContent) {
     throw new Error(
-      `service "${serviceName}" image { type: "dockerfile" } needs \`path\` (a Dockerfile in your repo, relative to the project root) or \`content\``,
+      `service "${serviceName}" image build needs \`path\` (a Dockerfile in your repo, relative to the project root) or \`content\``,
     );
   }
   if (hasPath) {
@@ -2491,6 +2535,7 @@ export function defineEnvironment<
   const { fakes, services: rawServices, ...rest } = input;
   const expanded = expandServiceGroups(rawServices);
   for (const [key, svc] of Object.entries(expanded)) {
+    for (const build of svc.imageBuilds ?? []) validateServiceImage(`${key} image ${build.reference}`, build.image);
     expanded[key] = applyCoverageAdapters(key, {
       ...svc,
       image: validateServiceImage(key, svc.image),