@mark1russell7/kind 0.0.0-stage → 0.1.0

package/build/index.js

@@ -0,0 +1,2325 @@
+import { $ as SERVER, A as dropOf, B as restartsAt, C as RELATIVE, D as backendsAt, E as quantileOf, F as outcomesOf, G as sliceEmptyOf, H as signalOf, I as podsAt, J as CLIENT, K as slicesOf, L as podsOf, M as failuresOf, N as intakeOf, O as byNode, P as kernelCounts, Q as REDIS, R as recordsOf, S as unitOf, T as middle, U as signalsOf, V as rulesOf, W as sliceDropOf, X as LOG_LABEL, Y as HEALTH_PORT, Z as PORT, _ as checkRun, a as calibrationOf, b as propagationOf, c as predictionText, d as FORMAT_V1, et as bandOf, f as judge, g as LAST_SLACK, h as KIND_NOISE, i as readRecord, j as emptyOf, k as deletionOf, l as timeSpread, m as upgradeCell, n as kindCellRecordSchema, nt as secondsOf, o as cellsTable, p as upgrade, q as transitionOf, r as kindCellRecordV1Schema, rt as showBand, s as medianText, t as STAGES, tt as seconds, u as FORMAT, v as factsOf, w as medianInterval, x as referenceOf, y as judgeKind, z as requestsOf } from "./judge-CW1IX2zP.js";
+import { valueIn } from "@mark1russell7/quantity";
+import { execFile, execFileSync } from "node:child_process";
+import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { promisify } from "node:util";
+import { createHash } from "node:crypto";
+import { fileURLToPath } from "node:url";
+import { failSpan, lint, service } from "@mark1russell7/service";
+import { createRequire } from "node:module";
+import { tmpdir } from "node:os";
+import { harnessImage } from "@mark1russell7/harness";
+import { redisClient } from "@mark1russell7/service/redis";
+//#region src/backends.ts
+const hexIp = (hex) => [
+	0,
+	2,
+	4,
+	6
+].map((at) => parseInt(hex.slice(at, at + 2), 16)).join(".");
+/**
+* This function gives the backends of a Service in a dump of the rules of kube-proxy, as
+* `ip:port`, sorted. A backend of IPVS with the weight zero takes no new connection, thus it is
+* not a backend.
+* - iptables: the `KUBE-SVC` chain of the Service jumps to a `KUBE-SEP` chain for each backend, and that chain does the DNAT.
+* - nftables: the `service-` chain of the Service does the DNAT to each backend, or it names an `endpoint-` chain for each one.
+* - IPVS: `/proc/net/ip_vs` has the line of the virtual service, then a line for each real server.
+*/
+const backendsOf = (mode, text, rule) => {
+	const out = /* @__PURE__ */ new Set();
+	const lines = text.split("\n");
+	if (mode === "iptables") {
+		const seps = /* @__PURE__ */ new Set();
+		for (const line of lines) {
+			if (!line.startsWith("-A KUBE-SVC-") || !line.includes(`${rule.ns}/${rule.svc}:`)) continue;
+			const jump = / -j (KUBE-SEP-\S+)/.exec(line);
+			if (jump !== null) seps.add(jump[1]);
+		}
+		for (const line of lines) {
+			const chain = /^-A (KUBE-SEP-\S+) /.exec(line);
+			if (chain === null || !seps.has(chain[1])) continue;
+			const to = / --to-destination (\S+)/.exec(line);
+			if (to !== null) out.add(to[1]);
+		}
+	} else if (mode === "nftables") {
+		let inside = false;
+		const ours = `-${rule.ns}/${rule.svc}/`;
+		for (const line of lines) {
+			const head = /^\s*chain\s+"?([^"\s{]+)"?\s*\{/.exec(line);
+			if (head !== null) inside = head[1].startsWith("service-") && head[1].includes(ours);
+			else if (/^\s*(set|map)\s/.test(line)) inside = false;
+			if (!inside) continue;
+			for (const match of line.matchAll(/(?:goto|jump)\s+"?endpoint-[0-9A-Za-z]+(-[^\s,}"]+)__([0-9.]+)\/(\d+)"?/g)) if (match[1].startsWith(ours)) out.add(`${match[2]}:${match[3]}`);
+			for (const match of line.matchAll(/(?:^|[\s{,:])(\d+\.\d+\.\d+\.\d+) \. (\d+)(?=\s*[,}]|\s*$)/g)) out.add(`${match[1]}:${match[2]}`);
+			for (const match of line.matchAll(/\bdnat (?:ip )?to (\d+\.\d+\.\d+\.\d+):(\d+)/g)) out.add(`${match[1]}:${match[2]}`);
+		}
+	} else {
+		let inside = false;
+		for (const line of lines) {
+			const service = /^(TCP|UDP|SCTP)\s+([0-9A-F]{8}):([0-9A-F]{4})\s/.exec(line);
+			if (service !== null) {
+				inside = service[1] === "TCP" && hexIp(service[2]) === rule.ip && parseInt(service[3], 16) === rule.port;
+				continue;
+			}
+			const real = /^\s+->\s+([0-9A-F]{8}):([0-9A-F]{4})\s+\S+\s+(\d+)/.exec(line);
+			if (inside && real !== null && Number(real[3]) > 0) out.add(`${hexIp(real[1])}:${parseInt(real[2], 16)}`);
+		}
+	}
+	return [...out].sort();
+};
+//#endregion
+//#region src/scenario.ts
+const KUBE_PROXY_MODES = [
+	"iptables",
+	"ipvs",
+	"nftables"
+];
+/**
+* These are the node images of kind v0.33.0, from its release notes. The image of Kubernetes 1.37
+* is v1.37.0. The research names v1.37.1 for some defaults, thus a default that changed in the
+* patch release is a mismatch to record.
+*/
+const NODE_IMAGES = {
+	"1.37": {
+		kubernetes: "v1.37.0",
+		image: "kindest/node:v1.37.0@sha256:a1ed56cfb0e7b93589bdf97c8cd566405a265939e3620fc4f5de89adff580ae5"
+	},
+	"1.36": {
+		kubernetes: "v1.36.4",
+		image: "kindest/node:v1.36.4@sha256:099e049362a1526b2db71494e1947aae99bd16290d7c895f2b7ea312e3cbfaed"
+	}
+};
+/** This function gives a scenario with the defaults of Kubernetes for kube-proxy, as explicit values. */
+const scenario = (name, mode, workers, kubernetes = "1.37") => ({
+	name,
+	kubernetes,
+	workers,
+	kubeProxy: {
+		mode,
+		minSyncPeriod: {
+			value: 1,
+			unit: "s"
+		},
+		syncPeriod: {
+			value: 30,
+			unit: "s"
+		}
+	},
+	cni: "kindnet",
+	podSubnet: "10.244.0.0/16",
+	serviceSubnet: "10.96.0.0/16"
+});
+const NAME = /^[a-z0-9]([a-z0-9-]{0,30}[a-z0-9])?$/;
+/** This function gives the problems of a scenario: a name that kind refuses, no image for the version, or a count out of range. */
+const scenarioProblems = (s) => {
+	const problems = [];
+	if (!NAME.test(s.name)) problems.push(`the name ${s.name} is not lowercase letters, digits and dashes of 1 to 32 characters`);
+	if (NODE_IMAGES[s.kubernetes] === void 0) problems.push(`no node image is pinned for Kubernetes ${s.kubernetes}`);
+	if (!Number.isInteger(s.workers) || s.workers < 0 || s.workers > 8) problems.push(`${s.workers} workers is not a whole number from 0 to 8`);
+	if (cidrOverlap(s.podSubnet, s.serviceSubnet)) problems.push(`the pod subnet ${s.podSubnet} overlaps the service subnet ${s.serviceSubnet}`);
+	return problems;
+};
+/** This function gives the names of the node containers, as kind names them: the control plane, then each worker. */
+const nodeNames = (s) => [`${s.name}-control-plane`, ...Array.from({ length: s.workers }, (_, at) => `${s.name}-worker${at === 0 ? "" : at + 1}`)];
+/** This function gives a duration as kube-proxy reads it, in whole milliseconds: `1s`, `500ms`. */
+const goDuration = (amount) => {
+	const ms = valueIn(amount, "ms");
+	if (!Number.isInteger(ms) || ms < 0) throw new RangeError(`${ms} ms is not a whole number of milliseconds`);
+	return ms % 1e3 === 0 ? `${ms / 1e3}s` : `${ms}ms`;
+};
+/** This function gives the KubeProxyConfiguration patch of a scenario, as kubeadm reads it. */
+const kubeProxyPatch = (s) => [
+	"apiVersion: kubeproxy.config.k8s.io/v1alpha1",
+	"kind: KubeProxyConfiguration",
+	`mode: ${s.kubeProxy.mode}`,
+	`${s.kubeProxy.mode}:`,
+	`  minSyncPeriod: ${goDuration(s.kubeProxy.minSyncPeriod)}`,
+	`  syncPeriod: ${goDuration(s.kubeProxy.syncPeriod)}`,
+	""
+].join("\n");
+/** This function generates the configuration of kind from a scenario. It refuses a scenario with a problem. */
+const kindConfig = (s) => {
+	const problems = scenarioProblems(s);
+	if (problems.length > 0) throw new Error(`the scenario ${s.name}: ${problems.join("; ")}`);
+	const image = NODE_IMAGES[s.kubernetes].image;
+	return {
+		kind: "Cluster",
+		apiVersion: "kind.x-k8s.io/v1alpha4",
+		networking: {
+			ipFamily: "ipv4",
+			kubeProxyMode: s.kubeProxy.mode,
+			disableDefaultCNI: false,
+			podSubnet: s.podSubnet,
+			serviceSubnet: s.serviceSubnet
+		},
+		nodes: nodeNames(s).map((node, at) => ({
+			role: at === 0 ? "control-plane" : "worker",
+			image,
+			labels: { "lab.systems/node": node.slice(s.name.length + 1) }
+		})),
+		kubeadmConfigPatches: [kubeProxyPatch(s)]
+	};
+};
+/** This function gives the first and the last address of an IPv4 CIDR, as numbers. */
+const cidrRange = (cidr) => {
+	const match = /^(\d+)\.(\d+)\.(\d+)\.(\d+)\/(\d+)$/.exec(cidr);
+	if (match === null) throw new Error(`not an IPv4 CIDR: ${cidr}`);
+	const octets = match.slice(1, 5).map(Number);
+	const bits = Number(match[5]);
+	if (octets.some((octet) => octet > 255) || bits > 32) throw new Error(`not an IPv4 CIDR: ${cidr}`);
+	const base = octets.reduce((sum, octet) => sum * 256 + octet, 0);
+	const size = 2 ** (32 - bits);
+	const first = Math.floor(base / size) * size;
+	return [first, first + size - 1];
+};
+/** This function tells if two IPv4 CIDRs share an address. */
+const cidrOverlap = (a, b) => {
+	const [a0, a1] = cidrRange(a);
+	const [b0, b1] = cidrRange(b);
+	return a0 <= b1 && b0 <= a1;
+};
+//#endregion
+//#region src/tools.ts
+/** This release of kind is v0.33.0, from 2026-08-26. Its default node image is Kubernetes 1.37.0. The sums are from its `.sha256sum` assets. */
+const KIND = {
+	name: "kind",
+	version: "v0.33.0",
+	assets: {
+		"win32-x64": {
+			url: "https://github.com/kubernetes-sigs/kind/releases/download/v0.33.0/kind-windows-amd64",
+			sha256: "4b22adaa135368c5a465d56bbd8e520cbea87272a06ca00b6078e7b81515c9fc"
+		},
+		"linux-x64": {
+			url: "https://github.com/kubernetes-sigs/kind/releases/download/v0.33.0/kind-linux-amd64",
+			sha256: "aee6151561422756b764a4ae28e7f44cda5af5a9eead3cc9985112b1de8d8e0d"
+		},
+		"linux-arm64": {
+			url: "https://github.com/kubernetes-sigs/kind/releases/download/v0.33.0/kind-linux-arm64",
+			sha256: "20022bee6cfcd5086cb7234d218e3454e6090022f2a8f55d1fa7fcf42c3867a2"
+		},
+		"darwin-arm64": {
+			url: "https://github.com/kubernetes-sigs/kind/releases/download/v0.33.0/kind-darwin-arm64",
+			sha256: "0c8c7dbe5e23594a198b786c4bc13dacc101fa6196b0cb0b23a1ca44e61f4b4f"
+		}
+	}
+};
+/** This release of kubectl is v1.37.0, the version of the node image. The sums are from the `.sha256` files of `dl.k8s.io`. */
+const KUBECTL = {
+	name: "kubectl",
+	version: "v1.37.0",
+	assets: {
+		"win32-x64": {
+			url: "https://dl.k8s.io/release/v1.37.0/bin/windows/amd64/kubectl.exe",
+			sha256: "4721b614a67bb4932a0369e61f4a323d8c6ca00943d3a2ff14837c124da06f0e"
+		},
+		"linux-x64": {
+			url: "https://dl.k8s.io/release/v1.37.0/bin/linux/amd64/kubectl",
+			sha256: "6129359f4e1f3848a5572ccb0b26cf28b8ca08cef38c95a765b2f64a2c961a2f"
+		},
+		"linux-arm64": {
+			url: "https://dl.k8s.io/release/v1.37.0/bin/linux/arm64/kubectl",
+			sha256: "922df28df248cc00a9e025f947704f1d1482de64ece54cfe57e61f19eaf1eef3"
+		},
+		"darwin-arm64": {
+			url: "https://dl.k8s.io/release/v1.37.0/bin/darwin/arm64/kubectl",
+			sha256: "583beedaebe422e71d3f1a96acef8b1fef86ea2f09a45ad01aa6c9ce287c1380"
+		}
+	}
+};
+/** This folder is the cache of the tools: `.cache/` in this package, which git ignores. `KIND_CACHE` names another folder. */
+const cacheFolder = () => process.env["KIND_CACHE"] ?? fileURLToPath(new URL("../.cache/", import.meta.url));
+/** This function gives the platform key of this process, as the assets name it. */
+const platformKey = (platform = process.platform, arch = process.arch) => `${platform}-${arch}`;
+/** This function gives the sha256 of some bytes, in hexadecimal. */
+const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
+/** This function gives the path of a tool in the cache, for a platform. */
+const toolPath = (tool, key = platformKey(), cache = cacheFolder()) => join(cache, "tools", `${tool.name}-${tool.version}-${key}`, key.startsWith("win32") ? `${tool.name}.exe` : tool.name);
+/**
+* This function gives the path of a tool, after a download when the cache does not have it. It
+* refuses bytes whose sha256 is not the pinned value, from the cache or from the download.
+*/
+const ensureTool = async (tool, key = platformKey(), cache = cacheFolder()) => {
+	const asset = tool.assets[key];
+	if (asset === void 0) throw new Error(`${tool.name} ${tool.version} has no pinned asset for ${key}`);
+	const path = toolPath(tool, key, cache);
+	if (existsSync(path)) {
+		const found = sha256(readFileSync(path));
+		if (found !== asset.sha256) throw new Error(`${path} has the sha256 ${found}, not the pinned ${asset.sha256}`);
+		return path;
+	}
+	const response = await fetch(asset.url, { redirect: "follow" });
+	if (!response.ok) throw new Error(`the download of ${asset.url} answered ${response.status}`);
+	const bytes = new Uint8Array(await response.arrayBuffer());
+	const found = sha256(bytes);
+	if (found !== asset.sha256) throw new Error(`${asset.url} has the sha256 ${found}, not the pinned ${asset.sha256}`);
+	mkdirSync(dirname(path), { recursive: true });
+	const partial = `${path}.partial`;
+	writeFileSync(partial, bytes);
+	if (!key.startsWith("win32")) chmodSync(partial, 493);
+	renameSync(partial, path);
+	return path;
+};
+//#endregion
+//#region src/cluster.ts
+const run = promisify(execFile);
+/** This function starts a program and gives its standard output. A failure gives its standard error in the message. */
+const command = async (program, args, options = {}) => {
+	try {
+		const child = run(program, [...args], {
+			env: {
+				...process.env,
+				...options.env
+			},
+			maxBuffer: 256 << 20,
+			timeout: options.timeout ?? 6e5,
+			windowsHide: true
+		});
+		if (options.input !== void 0) child.child.stdin?.end(options.input);
+		const { stdout } = await child;
+		return stdout.toString();
+	} catch (error) {
+		const failed = error;
+		throw new Error(`${program} ${args.join(" ")} failed: ${(failed.stderr ?? failed.message ?? "").toString().trim().slice(0, 2e3)}`);
+	}
+};
+/** This function gives the subnets of the Docker networks, by their names. */
+const dockerSubnets = async () => {
+	const ids = (await command("docker", [
+		"network",
+		"ls",
+		"-q"
+	])).split(/\s+/).filter((id) => id !== "");
+	if (ids.length === 0) return [];
+	return JSON.parse(await command("docker", [
+		"network",
+		"inspect",
+		...ids
+	])).flatMap((network) => (network.IPAM?.Config ?? []).map((config) => config.Subnet ?? "").filter((subnet) => /^\d+\.\d+\.\d+\.\d+\/\d+$/.test(subnet)).map((subnet) => [network.Name, subnet]));
+};
+/**
+* This function makes a kind cluster from a scenario, in a folder that holds its config, its
+* kubeconfig and the cache of kubectl. The cluster gets a Docker network of its own. The function
+* refuses a pod or service subnet that overlaps a Docker network, because that breaks the traffic
+* between the nodes.
+*/
+const createCluster = async (scenario, dir) => {
+	const kind = await ensureTool(KIND);
+	const kubectlPath = await ensureTool(KUBECTL);
+	mkdirSync(dir, { recursive: true });
+	const kubeconfig = join(dir, "kubeconfig");
+	const network = `systems-${scenario.name}`;
+	const config = kindConfig(scenario);
+	for (const [name, subnet] of await dockerSubnets()) {
+		if (name === network) continue;
+		for (const ours of [scenario.podSubnet, scenario.serviceSubnet]) if (cidrOverlap(ours, subnet)) throw new Error(`the subnet ${ours} overlaps the Docker network ${name} (${subnet})`);
+	}
+	const file = join(dir, "kind.json");
+	writeFileSync(file, `${JSON.stringify(config, null, 2)}\n`);
+	const env = { KIND_EXPERIMENTAL_DOCKER_NETWORK: network };
+	const nodes = nodeNames(scenario);
+	const kubectl = (args, input) => command(kubectlPath, [
+		"--kubeconfig",
+		kubeconfig,
+		"--cache-dir",
+		join(dir, "kubectl-cache"),
+		...args
+	], input === void 0 ? {} : { input });
+	const cluster = {
+		scenario,
+		dir,
+		kubeconfig,
+		network,
+		nodes,
+		kubectl,
+		async apply(objects) {
+			await kubectl([
+				"apply",
+				"-f",
+				"-"
+			], JSON.stringify({
+				apiVersion: "v1",
+				kind: "List",
+				items: objects
+			}));
+		},
+		exec: (node, argv) => command("docker", [
+			"exec",
+			node,
+			...argv
+		]),
+		async load(image) {
+			await command(kind, [
+				"load",
+				"docker-image",
+				"--name",
+				scenario.name,
+				image
+			], {
+				env,
+				timeout: 9e5
+			});
+		},
+		async credentials() {
+			const internal = join(dir, "kubeconfig.internal");
+			writeFileSync(internal, await command(kind, [
+				"get",
+				"kubeconfig",
+				"--internal",
+				"--name",
+				scenario.name
+			], { env }));
+			const view = JSON.parse(await command(kubectlPath, [
+				"--kubeconfig",
+				internal,
+				"config",
+				"view",
+				"--raw",
+				"-o",
+				"json"
+			]));
+			return {
+				server: view.clusters[0].cluster.server,
+				ca: view.clusters[0].cluster["certificate-authority-data"],
+				cert: view.users[0].user["client-certificate-data"],
+				key: view.users[0].user["client-key-data"]
+			};
+		},
+		async manifest() {
+			const nodesJson = JSON.parse(await kubectl([
+				"get",
+				"nodes",
+				"-o",
+				"json"
+			]));
+			const proxy = JSON.parse(await kubectl([
+				"-n",
+				"kube-system",
+				"get",
+				"configmap",
+				"kube-proxy",
+				"-o",
+				"json"
+			]));
+			const mode = /^mode:\s*(\S*)/m.exec(proxy.data["config.conf"] ?? "")?.[1] ?? "";
+			const version = JSON.parse(await kubectl([
+				"version",
+				"-o",
+				"json"
+			]));
+			return {
+				kind: (await command(kind, ["version"])).trim(),
+				nodeImage: NODE_IMAGES[scenario.kubernetes].image,
+				server: version.serverVersion?.gitVersion ?? "",
+				mode: mode.replaceAll("\"", ""),
+				docker: (await command("docker", [
+					"version",
+					"--format",
+					"{{.Server.Version}}"
+				])).trim(),
+				nodes: nodesJson.items.map((item) => ({
+					name: item.metadata.name,
+					kubelet: item.status.nodeInfo["kubeletVersion"] ?? "",
+					kernel: item.status.nodeInfo["kernelVersion"] ?? "",
+					runtime: item.status.nodeInfo["containerRuntimeVersion"] ?? "",
+					os: item.status.nodeInfo["osImage"] ?? ""
+				}))
+			};
+		},
+		async delete() {
+			await command(kind, [
+				"delete",
+				"cluster",
+				"--name",
+				scenario.name,
+				"--kubeconfig",
+				kubeconfig
+			], { env });
+			await command("docker", [
+				"network",
+				"rm",
+				network
+			]).catch(() => void 0);
+		}
+	};
+	await command(kind, [
+		"create",
+		"cluster",
+		"--name",
+		scenario.name,
+		"--config",
+		file,
+		"--kubeconfig",
+		kubeconfig,
+		"--wait",
+		"180s"
+	], {
+		env,
+		timeout: 9e5
+	});
+	return cluster;
+};
+//#endregion
+//#region src/compose.ts
+/** This name is the controller of the service, and its one route reads Redis when the service has Redis. */
+const CONTROLLER = "work";
+const ROUTE = "GET /work";
+/** This name is the check that a flag file fails: the seam of the liveness restart. */
+const FLAG_CHECK = "self.flag";
+/** This name is the watchdog of the event loop. */
+const WATCHDOG = "self.event-loop";
+/**
+* This function composes the service of a declaration: its policies, its couplings, its checks,
+* one server, one controller and each stage of the shutdown. A declaration with Redis needs the
+* client, because the client declares the dependency and its checks.
+*/
+const compose = (name, declaration, parts = {}) => {
+	const svc = service(name).readiness(declaration.readiness).liveness(declaration.liveness).startup(declaration.startup).cooperativeWindow(declaration.cooperativeWindow);
+	for (const coupling of declaration.couplings) svc.couple(coupling);
+	if (declaration.redis !== null) {
+		if (parts.redis === void 0) throw new Error(`${name} declares Redis, thus its composition needs the client`);
+		parts.redis.attach(svc, {
+			poll: declaration.poll,
+			steps: { "round-trip": declaration.redis.gates }
+		});
+	}
+	if (declaration.flag !== null) svc.check({
+		name: FLAG_CHECK,
+		dependency: null,
+		step: "local-state",
+		ownConnection: true,
+		gates: declaration.flag,
+		poll: declaration.poll
+	}, parts.flag);
+	if (declaration.watchdog !== null) svc.watchdog({
+		name: WATCHDOG,
+		tick: declaration.watchdog.tick,
+		threshold: declaration.watchdog.threshold,
+		gates: "liveness",
+		poll: declaration.poll
+	});
+	svc.server("http", {}, parts.http);
+	const controller = svc.controller(CONTROLLER);
+	if (declaration.redis !== null) controller.requires(parts.redis.name);
+	controller.route(ROUTE, { idempotent: true });
+	if (parts.health !== void 0) svc.healthServer(parts.health);
+	for (const [stage, cap] of declaration.phases) svc.phase(stage, cap);
+	return svc;
+};
+//#endregion
+//#region src/faults.ts
+const needs = (time, what) => {
+	if (time === void 0) throw new Error(`the trigger needs ${what}`);
+	return time;
+};
+/** This function gives an amount in microseconds as a whole number, for `cpu.max`. */
+const micros = (amount) => Math.round(valueIn(amount, "us"));
+/** This value marks the name of the target pod in the arguments of kubectl. The runner replaces it. */
+const POD = "{pod}";
+/** This value marks the node of the target pod in the arguments of kubectl. */
+const NODE = "{node}";
+/** This value marks the next version of the Deployment, for a rollout. */
+const VERSION = "{version}";
+/** This function gives the fault plan of a cell. */
+const planOf = (cell) => {
+	const ns = ["-n", "{namespace}"];
+	const trigger = cell.trigger;
+	const replicas = cell.pod.replicas;
+	switch (trigger.kind) {
+		case "pod-deletion": return {
+			inject: [{
+				do: "kubectl",
+				args: [
+					...ns,
+					"delete",
+					"pod",
+					POD,
+					"--wait=false"
+				]
+			}],
+			hold: null,
+			heal: [],
+			restore: []
+		};
+		case "rolling-update": return {
+			inject: [{
+				do: "kubectl",
+				args: [
+					...ns,
+					"set",
+					"env",
+					`deployment/lab`,
+					`LAB_VERSION=${VERSION}`
+				]
+			}],
+			hold: null,
+			heal: [],
+			restore: []
+		};
+		case "node-drain": return {
+			inject: [{
+				do: "kubectl",
+				args: [
+					"drain",
+					NODE,
+					"--ignore-daemonsets",
+					"--delete-emptydir-data",
+					"--timeout=180s",
+					`--pod-selector=app=lab`
+				],
+				background: true
+			}],
+			hold: null,
+			heal: [],
+			restore: [{
+				do: "kubectl",
+				args: ["uncordon", NODE]
+			}]
+		};
+		case "scale-in": return {
+			inject: [{
+				do: "kubectl",
+				args: [
+					...ns,
+					"scale",
+					`deployment/lab`,
+					`--replicas=${replicas - 1}`
+				]
+			}],
+			hold: null,
+			heal: [],
+			restore: [{
+				do: "kubectl",
+				args: [
+					...ns,
+					"scale",
+					`deployment/lab`,
+					`--replicas=${replicas}`
+				]
+			}]
+		};
+		case "crash": return {
+			inject: [{
+				do: "signal",
+				target: "server",
+				signal: "SIGKILL"
+			}],
+			hold: null,
+			heal: [],
+			restore: []
+		};
+		case "liveness-restart": return {
+			inject: [{
+				do: "flag",
+				target: "server",
+				on: true
+			}],
+			hold: null,
+			heal: [],
+			restore: []
+		};
+		case "freeze-process":
+			if (cell.template === "T07") return {
+				inject: [
+					{
+						do: "pause",
+						target: "kube-proxy",
+						time: needs(trigger.hold, "a hold")
+					},
+					{
+						do: "wait",
+						time: needs(trigger.lead, "a lead")
+					},
+					{
+						do: "kubectl",
+						args: [
+							...ns,
+							"delete",
+							"pod",
+							POD,
+							"--wait=false"
+						]
+					}
+				],
+				hold: null,
+				heal: [],
+				restore: []
+			};
+			return {
+				inject: [{
+					do: "signal",
+					target: "redis",
+					signal: "SIGSTOP"
+				}],
+				hold: needs(trigger.hold, "a hold"),
+				heal: [{
+					do: "signal",
+					target: "redis",
+					signal: "SIGCONT"
+				}],
+				restore: []
+			};
+		case "rst": return {
+			inject: [{
+				do: "inject",
+				target: "redis",
+				action: "rst"
+			}],
+			hold: needs(trigger.hold, "a hold"),
+			heal: [{
+				do: "inject",
+				target: "redis",
+				action: "clear-rst"
+			}],
+			restore: []
+		};
+		case "blackhole": return {
+			inject: [{
+				do: "inject",
+				target: "redis",
+				action: "blackhole"
+			}],
+			hold: needs(trigger.hold, "a hold"),
+			heal: [{
+				do: "inject",
+				target: "redis",
+				action: "clear"
+			}],
+			restore: []
+		};
+		case "peer-gone-without-close": return {
+			inject: [{
+				do: "inject",
+				target: "redis",
+				action: "blackhole"
+			}, {
+				do: "kubectl",
+				args: [
+					...ns,
+					"scale",
+					`deployment/${REDIS}`,
+					"--replicas=0"
+				]
+			}],
+			hold: needs(trigger.hold, "a hold"),
+			heal: [{
+				do: "kubectl",
+				args: [
+					...ns,
+					"scale",
+					`deployment/${REDIS}`,
+					"--replicas=1"
+				]
+			}],
+			restore: []
+		};
+		case "app-stall": return {
+			inject: [{
+				do: "stall",
+				target: "server",
+				time: needs(trigger.hold, "a hold")
+			}],
+			hold: null,
+			heal: [],
+			restore: []
+		};
+		case "rollout": return {
+			inject: [{
+				do: "kubectl",
+				args: [
+					...ns,
+					"scale",
+					`deployment/${REDIS}`,
+					"--replicas=0"
+				]
+			}, {
+				do: "kubectl",
+				args: [
+					...ns,
+					"set",
+					"env",
+					`deployment/lab`,
+					`LAB_VERSION=${VERSION}`
+				]
+			}],
+			hold: needs(trigger.hold, "a hold"),
+			heal: [{
+				do: "kubectl",
+				args: [
+					...ns,
+					"scale",
+					`deployment/${REDIS}`,
+					"--replicas=1"
+				]
+			}],
+			restore: []
+		};
+		case "cpu-throttle": {
+			const max = trigger.cpuMax;
+			if (max === void 0) throw new Error("the trigger needs its cpu.max");
+			return {
+				inject: [{
+					do: "cpu",
+					target: "server",
+					max: `${micros(max.quota)} ${micros(max.period)}`
+				}],
+				hold: needs(trigger.hold, "a hold"),
+				heal: [{
+					do: "cpu",
+					target: "server",
+					max: `max ${micros(max.period)}`
+				}],
+				restore: []
+			};
+		}
+	}
+};
+/** This function tells if a trigger stops a pod of the service with its grace: a deletion, an eviction, a scale-in or a rollout. */
+const deletes = (cell) => [
+	"pod-deletion",
+	"rolling-update",
+	"node-drain",
+	"scale-in"
+].includes(cell.trigger.kind) || cell.template === "T07" && cell.trigger.kind === "freeze-process";
+//#endregion
+//#region src/image.ts
+/** This folder is the root of the package, and its `image/` folder has the files of the image. */
+const PACKAGE = fileURLToPath(new URL("../", import.meta.url));
+/** This folder is the root of the repository, when the package is in the workspace of the repository. */
+const REPO = fileURLToPath(new URL("../../../", import.meta.url));
+/** These are the packages of this repository that the image holds, in the sequence of their dependencies. */
+const WORKSPACE_PACKAGES = [
+	"quantity",
+	"algebra",
+	"service"
+];
+/** These are the files of the image, by their name in the context and their source in the package. */
+const FILES = [
+	["Dockerfile", "image/Dockerfile"],
+	["package.json", "image/package.json"],
+	["npm-lock.json", "image/npm-lock.json"],
+	["app.mjs", "image/app.mjs"],
+	["client.mjs", "image/client.mjs"],
+	["agent.mjs", "image/agent.mjs"],
+	["watch.mjs", "image/watch.mjs"],
+	["compose.ts", "src/compose.ts"],
+	["backends.ts", "src/backends.ts"]
+];
+const shell = process.platform === "win32";
+/** This function gives each file of a staged context, with its path in the context, in a stable sequence. */
+const filesOf = (root, at = "") => readdirSync(join(root, at), { withFileTypes: true }).flatMap((entry) => entry.isDirectory() ? filesOf(root, join(at, entry.name)) : [join(at, entry.name)]).sort();
+/** This function gives the tag of a staged context from its content: the name, then 12 digits of a sha256. */
+const contentTag = (name, root) => {
+	const hash = createHash("sha256");
+	for (const file of filesOf(root)) {
+		hash.update(file.replaceAll("\\", "/"));
+		hash.update(readFileSync(join(root, file)));
+	}
+	return `${name}:${hash.digest("hex").slice(0, 12)}`;
+};
+/** This function tells if the package is in the workspace of its repository. From npm, it is not. */
+const inWorkspace = () => existsSync(join(REPO, "pnpm-workspace.yaml")) && WORKSPACE_PACKAGES.every((name) => existsSync(join(REPO, "packages", name, "package.json")));
+/**
+* This function gives the folder of each installed package that the image holds. Node finds a
+* package from the folder of a package that depends on it. The service depends on the algebra,
+* thus the search starts with the service.
+*/
+const installedFolders = () => {
+	const folders = /* @__PURE__ */ new Map();
+	for (const name of [...WORKSPACE_PACKAGES].reverse()) {
+		const found = [PACKAGE, ...folders.values()].flatMap((from) => {
+			try {
+				return [dirname(realpathSync(createRequire(join(from, "package.json")).resolve(`@mark1russell7/${name}/package.json`)))];
+			} catch {
+				return [];
+			}
+		})[0];
+		if (found === void 0) throw new Error(`@mark1russell7/${name} is not installed with @mark1russell7/kind`);
+		folders.set(name, found);
+	}
+	return folders;
+};
+/**
+* This function stages the build context of the lab image. In the workspace, it builds the packages
+* of the repository and packs each one as npm gets it. From npm, it packs the installed packages.
+* Then it copies the files of the image.
+*/
+const stageLabImage = () => {
+	const context = mkdtempSync(join(tmpdir(), "systems-kind-lab-"));
+	const packed = join(context, "workspace");
+	mkdirSync(packed);
+	if (inWorkspace()) {
+		const filters = WORKSPACE_PACKAGES.flatMap((name) => ["--filter", `@mark1russell7/${name}`]);
+		execFileSync("pnpm", [...filters, "build"], {
+			cwd: REPO,
+			shell,
+			stdio: "ignore"
+		});
+		for (const name of WORKSPACE_PACKAGES) execFileSync("pnpm", [
+			"pack",
+			"--pack-destination",
+			packed
+		], {
+			cwd: join(REPO, "packages", name),
+			shell,
+			stdio: "ignore"
+		});
+	} else for (const folder of installedFolders().values()) execFileSync("npm", [
+		"pack",
+		"--pack-destination",
+		packed,
+		"--ignore-scripts"
+	], {
+		cwd: folder,
+		shell,
+		stdio: "ignore"
+	});
+	for (const [at, from] of FILES) copyFileSync(join(PACKAGE, from), join(context, at));
+	return context;
+};
+let built = null;
+/** This function builds the lab image one time for each process, and gives its tag. Docker keeps its layers between builds. */
+const labImage = () => {
+	built ??= (async () => {
+		const context = stageLabImage();
+		try {
+			const tag = contentTag("systems-kind-lab", context);
+			const { GenericContainer } = await import("testcontainers");
+			await GenericContainer.fromDockerfile(context).build(tag, { deleteOnExit: false });
+			return tag;
+		} finally {
+			rmSync(context, {
+				recursive: true,
+				force: true
+			});
+		}
+	})();
+	return built;
+};
+//#endregion
+//#region src/lab.ts
+/** These kinds of kernel record stay in the raw record of a run. The lab counts the other kinds and drops them. */
+const KEPT_KERNEL = /* @__PURE__ */ new Set([
+	"rst_tx",
+	"rst_rx",
+	"rst_v4",
+	"error",
+	"retrans",
+	"begin",
+	"fail"
+]);
+/** This container class can share the process namespace of another container. */
+const shared = async () => {
+	const { GenericContainer } = await import("testcontainers");
+	return class extends GenericContainer {
+		withPidMode(mode) {
+			this.hostConfig.PidMode = mode;
+			return this;
+		}
+	};
+};
+/** This function collects the JSON lines of a stream, and gives each one to `take`. */
+const consume = (take) => (stream) => {
+	let partial = "";
+	stream.on("data", (chunk) => {
+		partial += chunk.toString();
+		const split = partial.split("\n");
+		partial = split.pop() ?? "";
+		for (const text of split) {
+			const trimmed = text.trim();
+			if (!trimmed.startsWith("{")) continue;
+			try {
+				take(JSON.parse(trimmed));
+			} catch {}
+		}
+	});
+};
+/** This function gives a stamp of the injector of the harness, in nanoseconds, as milliseconds. */
+const injectorStamp = (stdout) => {
+	const stamp = JSON.parse(stdout.trim().split("\n").at(-1));
+	return {
+		t0: Number(BigInt(stamp.t0)) / 1e6,
+		t1: Number(BigInt(stamp.t1)) / 1e6
+	};
+};
+const output = async (container, argv) => {
+	const result = await container.exec([...argv]);
+	if (result.exitCode !== 0) throw new Error(`${argv.join(" ")} exited ${result.exitCode}: ${result.output.trim().slice(0, 500)}`);
+	return result.stdout;
+};
+/**
+* This function opens the instruments of a cluster. Each node gets the agent of the lab and the
+* injector of the harness, privileged and in the PID namespace of the node. Thus the agent signals
+* a process of a pod by its PID on the node. The injector enters the network namespace of a pod
+* with `nsenter`.
+*/
+const openKindLab = async (cluster, image) => {
+	const harness = await harnessImage();
+	const Shared = await shared();
+	const { GenericContainer } = await import("testcontainers");
+	let lines = [];
+	const dropped = {};
+	const listeners = /* @__PURE__ */ new Set();
+	const push = (line) => {
+		lines.push(line);
+		for (const listener of listeners) listener(line);
+	};
+	const sidecars = /* @__PURE__ */ new Map();
+	const opened = [];
+	let watcher = null;
+	let prober = null;
+	try {
+		for (const node of cluster.nodes) {
+			const agent = await new Shared(image).withPidMode(`container:${node}`).withPrivilegedMode().withNetworkMode("none").withEnvironment({ NODE_NAME: node }).withCommand([
+				"node",
+				"agent.mjs",
+				"serve"
+			]).withLogConsumer(consume((line) => push({
+				...line,
+				node
+			}))).start();
+			opened.push(agent);
+			const injector = await new Shared(harness).withPidMode(`container:${node}`).withPrivilegedMode().withNetworkMode("none").start();
+			opened.push(injector);
+			sidecars.set(node, {
+				agent,
+				injector
+			});
+		}
+	} catch (error) {
+		for (const container of opened.reverse()) await container.stop({ timeout: 0 }).catch(() => void 0);
+		throw error;
+	}
+	const sidecarOf = (node) => {
+		const found = sidecars.get(node);
+		if (found === void 0) throw new Error(`the cluster has no node ${node}`);
+		return found;
+	};
+	const firstNode = cluster.nodes[0];
+	const lab = {
+		cluster,
+		image,
+		get lines() {
+			return lines;
+		},
+		dropped,
+		own() {
+			return [
+				...opened,
+				...watcher === null ? [] : [watcher],
+				...prober === null ? [] : [prober]
+			].map((container) => container.getName().replace(/^\//, ""));
+		},
+		take() {
+			const out = lines;
+			lines = [];
+			return out;
+		},
+		listen(listener) {
+			listeners.add(listener);
+			return () => listeners.delete(listener);
+		},
+		async watch(namespaces) {
+			if (watcher !== null) await watcher.stop({ timeout: 0 }).catch(() => void 0);
+			const kube = await cluster.credentials();
+			watcher = await new GenericContainer(image).withNetworkMode(cluster.network).withEnvironment({
+				KUBE: JSON.stringify(kube),
+				NAMESPACES: namespaces.join(",")
+			}).withCommand(["node", "watch.mjs"]).withLogConsumer(consume((line) => push(line))).start();
+		},
+		async probe(netns) {
+			await lab.stopProbe();
+			let begun = () => {};
+			const ready = new Promise((resolve, reject) => {
+				begun = (ok, why) => ok ? resolve() : reject(/* @__PURE__ */ new Error(`the prober failed: ${why}`));
+			});
+			prober = await new GenericContainer(harness).withPrivilegedMode().withNetworkMode("none").withCommand(["/opt/harness/prober", ...netns.map(String)]).withLogConsumer(consume((record) => {
+				const ev = String(record["ev"]);
+				if (ev === "begin") begun(true, "");
+				if (ev === "fail") begun(false, String(record["why"]));
+				if (!KEPT_KERNEL.has(ev) && ev !== "clock") {
+					dropped[ev] = (dropped[ev] ?? 0) + 1;
+					return;
+				}
+				const mono = Number(record["mono"] ?? 0);
+				push({
+					...record,
+					ev: `kernel.${ev}`,
+					t: mono / 1e6
+				});
+			})).start();
+			await Promise.race([ready, new Promise((_, reject) => setTimeout(() => reject(/* @__PURE__ */ new Error("no begin line from the prober in 60 s")), 6e4))]);
+		},
+		async stopProbe() {
+			if (prober !== null) await prober.stop({ timeout: 0 }).catch(() => void 0);
+			prober = null;
+		},
+		async rules(rule) {
+			const file = rule === null ? null : JSON.stringify({
+				...rule,
+				every: valueIn(rule.every, "ms")
+			});
+			for (const { agent } of sidecars.values()) if (file === null) await output(agent, [
+				"rm",
+				"-f",
+				"/tmp/rules.json"
+			]);
+			else await output(agent, [
+				"sh",
+				"-c",
+				`printf '%s' '${file}' > /tmp/rules.json`
+			]);
+		},
+		async act(node, spec) {
+			const stdout = await output(sidecarOf(node).agent, [
+				"node",
+				"agent.mjs",
+				"act",
+				JSON.stringify(spec)
+			]);
+			const result = JSON.parse(stdout.trim().split("\n").at(-1));
+			if (!result.ok) throw new Error(`the act ${JSON.stringify(spec)} on ${node} failed: ${result.why}`);
+			push({
+				ev: "act",
+				t: result.t0,
+				t1: result.t1,
+				node,
+				spec
+			});
+			return {
+				t0: result.t0,
+				t1: result.t1
+			};
+		},
+		async inject(node, pid, args) {
+			const stdout = await output(sidecarOf(node).injector, [
+				"nsenter",
+				"-t",
+				String(pid),
+				"-n",
+				"/opt/harness/inject",
+				...args
+			]);
+			const stamp = injectorStamp(stdout);
+			push({
+				ev: "act",
+				t: stamp.t0,
+				t1: stamp.t1,
+				node,
+				spec: {
+					op: "inject",
+					pid,
+					args
+				}
+			});
+			return stamp;
+		},
+		async now() {
+			return (await lab.act(firstNode, { op: "now" })).t0;
+		},
+		async pidOf(node, namespace, pod, container) {
+			const ids = (await cluster.exec(node, [
+				"crictl",
+				"ps",
+				"-q",
+				"--state",
+				"running",
+				"--label",
+				`io.kubernetes.pod.namespace=${namespace}`,
+				"--label",
+				`io.kubernetes.pod.name=${pod}`,
+				"--name",
+				`^${container}$`
+			])).split(/\s+/).filter((id) => id !== "");
+			if (ids.length !== 1) throw new Error(`${namespace}/${pod}/${container} on ${node} has ${ids.length} running containers, not 1`);
+			const pid = JSON.parse(await cluster.exec(node, [
+				"crictl",
+				"inspect",
+				ids[0]
+			])).info?.pid;
+			if (pid === void 0 || !(pid > 1)) throw new Error(`crictl gives no PID for ${namespace}/${pod}/${container}`);
+			return pid;
+		},
+		async netnsOf(node, pid) {
+			const link = (await cluster.exec(node, ["readlink", `/proc/${pid}/ns/net`])).trim();
+			const inode = /^net:\[(\d+)\]$/.exec(link)?.[1];
+			if (inode === void 0) throw new Error(`not a network namespace: ${link}`);
+			return Number(inode);
+		},
+		async cgroupOf(node, pid) {
+			const text = (await cluster.exec(node, ["cat", `/proc/${pid}/cgroup`])).trim();
+			const path = /^0::(\S+)$/m.exec(text)?.[1];
+			if (path === void 0 || !/\/cri-containerd-[0-9a-f]{64}\.scope$/.test(path)) throw new Error(`the cgroup of ${pid} on ${node} is not the scope of one container: ${text}`);
+			return `/proc/1/root/sys/fs/cgroup${path}`;
+		},
+		async close() {
+			await lab.stopProbe();
+			if (watcher !== null) await watcher.stop({ timeout: 0 }).catch(() => void 0);
+			watcher = null;
+			for (const container of opened.reverse()) await container.stop({ timeout: 0 }).catch(() => void 0);
+		}
+	};
+	return lab;
+};
+/** This function pins a Docker image by digest under a local tag that holds the digest, for `kind load` and `imagePullPolicy: Never`. */
+const localPin = async (image) => {
+	const match = /^([^@]+?)(?::([^@:/]+))?@sha256:([0-9a-f]{64})$/.exec(image);
+	if (match === null) throw new Error(`${image} is not pinned by a digest`);
+	const [, repository, tag, digest] = match;
+	const local = `systems-lab/${repository.split("/").at(-1)}:${tag ?? "pinned"}-${digest.slice(0, 12)}`;
+	if (!(await command("docker", [
+		"image",
+		"inspect",
+		"--format",
+		"{{json .RepoDigests}}",
+		`${repository}@sha256:${digest}`
+	]).catch(() => "")).includes(digest)) await command("docker", ["pull", `${repository}@sha256:${digest}`], { timeout: 9e5 });
+	await command("docker", [
+		"tag",
+		`${repository}@sha256:${digest}`,
+		local
+	]);
+	return local;
+};
+//#endregion
+//#region src/manifests.ts
+/** This image is Redis 7.4, the stock target of the harness, pinned by digest. */
+const REDIS_IMAGE = "redis:7.4@sha256:c6eabf748fc7a61dbb5a705c78bcf3d6377b1127a97d0ce965c11c44ba46896f";
+const whole = (amount, what) => {
+	const seconds = valueIn(amount, "s");
+	if (!Number.isInteger(seconds) || seconds < 0) throw new RangeError(`${what} is ${seconds} s, not a whole number of seconds`);
+	return seconds;
+};
+/** This function gives the namespace of a cell: `lab-` and 10 digits of the sha256 of its name. */
+const namespaceOf = (cell) => `lab-${createHash("sha256").update(cell.name).digest("hex").slice(0, 10)}`;
+/** This function gives a probe of the pod: an HTTP GET of the health server, with each field of the cell. */
+const probeOf = (path, probe) => ({
+	httpGet: {
+		path,
+		port: HEALTH_PORT
+	},
+	initialDelaySeconds: 0,
+	periodSeconds: whole(probe.period, `the period of ${path}`),
+	timeoutSeconds: whole(probe.timeout, `the timeout of ${path}`),
+	failureThreshold: probe.threshold,
+	successThreshold: 1
+});
+/** This function gives the environment of the app: its config as JSON, and the name of its pod. */
+const appEnv = (cell) => [{
+	name: "CONFIG",
+	value: JSON.stringify({
+		name: "lab",
+		port: PORT,
+		healthPort: HEALTH_PORT,
+		declaration: cell.declaration
+	})
+}, {
+	name: "POD_NAME",
+	valueFrom: { fieldRef: { fieldPath: "metadata.name" } }
+}];
+/** These are the objects of Redis: one replica behind a Service, with no persistence. */
+const redisObjects = (namespace, image) => [{
+	apiVersion: "apps/v1",
+	kind: "Deployment",
+	metadata: {
+		name: REDIS,
+		namespace,
+		labels: { app: REDIS }
+	},
+	spec: {
+		replicas: 1,
+		selector: { matchLabels: { app: REDIS } },
+		template: {
+			metadata: { labels: { app: REDIS } },
+			spec: {
+				terminationGracePeriodSeconds: 1,
+				containers: [{
+					name: REDIS,
+					image,
+					imagePullPolicy: "Never",
+					command: [
+						"redis-server",
+						"--save",
+						"",
+						"--appendonly",
+						"no"
+					],
+					ports: [{ containerPort: 6379 }],
+					readinessProbe: {
+						tcpSocket: { port: 6379 },
+						periodSeconds: 1,
+						failureThreshold: 1,
+						successThreshold: 1,
+						timeoutSeconds: 1
+					}
+				}]
+			}
+		}
+	}
+}, {
+	apiVersion: "v1",
+	kind: "Service",
+	metadata: {
+		name: REDIS,
+		namespace
+	},
+	spec: {
+		selector: { app: REDIS },
+		ports: [{
+			port: 6379,
+			targetPort: 6379
+		}]
+	}
+}];
+/** These are the objects of the service: a Service of type ClusterIP, and the Deployment with the settings of the cell. */
+const serverObjects = (cell, namespace, image, version = 1) => {
+	const preStop = whole(cell.pod.preStop, "the preStop sleep");
+	return [{
+		apiVersion: "v1",
+		kind: "Service",
+		metadata: {
+			name: "lab",
+			namespace
+		},
+		spec: {
+			selector: { app: "lab" },
+			ports: [{
+				name: "http",
+				port: 80,
+				targetPort: PORT
+			}]
+		}
+	}, {
+		apiVersion: "apps/v1",
+		kind: "Deployment",
+		metadata: {
+			name: "lab",
+			namespace,
+			labels: { app: "lab" }
+		},
+		spec: {
+			replicas: cell.pod.replicas,
+			selector: { matchLabels: { app: "lab" } },
+			strategy: {
+				type: "RollingUpdate",
+				rollingUpdate: {
+					maxSurge: "25%",
+					maxUnavailable: "25%"
+				}
+			},
+			minReadySeconds: 0,
+			template: {
+				metadata: { labels: {
+					app: "lab",
+					[LOG_LABEL]: "1",
+					"lab.systems/version": String(version)
+				} },
+				spec: {
+					terminationGracePeriodSeconds: whole(cell.pod.grace, "the grace period"),
+					enableServiceLinks: true,
+					topologySpreadConstraints: [{
+						maxSkew: 1,
+						topologyKey: "kubernetes.io/hostname",
+						whenUnsatisfiable: "ScheduleAnyway",
+						labelSelector: { matchLabels: { app: "lab" } }
+					}],
+					containers: [{
+						name: "app",
+						image,
+						imagePullPolicy: "Never",
+						command: ["node", "app.mjs"],
+						env: [...appEnv(cell), {
+							name: "LAB_VERSION",
+							value: String(version)
+						}],
+						ports: [{ containerPort: PORT }, { containerPort: HEALTH_PORT }],
+						...cell.pod.startup === null ? {} : { startupProbe: probeOf("/startup", cell.pod.startup) },
+						readinessProbe: probeOf("/readiness", cell.pod.readiness),
+						livenessProbe: probeOf("/liveness", cell.pod.liveness),
+						...preStop === 0 ? {} : { lifecycle: { preStop: { sleep: { seconds: preStop } } } }
+					}]
+				}
+			}
+		}
+	}];
+};
+/** These are the objects of the clients: a DaemonSet with one client on each node, the control plane too. */
+const clientObjects = (cell, namespace, image, target) => [{
+	apiVersion: "apps/v1",
+	kind: "DaemonSet",
+	metadata: {
+		name: CLIENT,
+		namespace,
+		labels: { app: CLIENT }
+	},
+	spec: {
+		selector: { matchLabels: { app: CLIENT } },
+		template: {
+			metadata: { labels: {
+				app: CLIENT,
+				[LOG_LABEL]: "1"
+			} },
+			spec: {
+				terminationGracePeriodSeconds: 0,
+				tolerations: [{
+					key: "node-role.kubernetes.io/control-plane",
+					operator: "Exists",
+					effect: "NoSchedule"
+				}],
+				containers: [{
+					name: CLIENT,
+					image,
+					imagePullPolicy: "Never",
+					command: ["node", "client.mjs"],
+					env: [
+						{
+							name: "TARGET",
+							value: target
+						},
+						{
+							name: "EVERY_MS",
+							value: String(valueIn(cell.workload.every, "ms"))
+						},
+						{
+							name: "TIMEOUT_MS",
+							value: String(valueIn(cell.workload.timeout, "ms"))
+						},
+						{
+							name: "POD_NAME",
+							valueFrom: { fieldRef: { fieldPath: "metadata.name" } }
+						},
+						{
+							name: "NODE_NAME",
+							valueFrom: { fieldRef: { fieldPath: "spec.nodeName" } }
+						}
+					]
+				}]
+			}
+		}
+	}
+}];
+//#endregion
+//#region src/predict.ts
+/**
+* This module is the arithmetic of the predictions. Each band is a range of times after a
+* reference event of the run. Each term comes from a declaration, from the pod settings, or from
+* the calibration of the propagation. No term comes from a default. The arithmetic operates in
+* seconds, and each result is a quantity.
+*/
+const s$1 = (amount) => valueIn(amount, "s");
+/** This function rounds a time to the microsecond, thus a band has no float residue. */
+const us = (value) => Math.round(value * 1e6) / 1e6;
+const band = (lo, hi) => bandOf(us(lo), us(hi));
+/** This propagation is the documented bound, before a calibration: from zero to one `minSyncPeriod` of 1 s. */
+const DOCUMENTED = {
+	lo: seconds(0),
+	hi: seconds(1),
+	from: "minSyncPeriod (1 s), not calibrated"
+};
+/**
+* This function gives the time from a flip of the probed state to the decision of the kubelet. The
+* next probe comes within one period, then `threshold − 1` more periods follow. A probe of an
+* endpoint that does not answer fails at its timeout.
+*/
+const kubeletBand = (probe, hangs) => {
+	const late = hangs ? s$1(probe.timeout) : 0;
+	return band((probe.threshold - 1) * s$1(probe.period) + late, probe.threshold * s$1(probe.period) + late);
+};
+/**
+* This function gives the time from a fault of a dependency to the trip of a readiness or liveness
+* latch. A dependency that refuses fails each poll at once. A silent dependency fails each poll at
+* the request timeout of its client, or at the timeout of the poll when that is shorter. The
+* watchdog of the client can close the connection sooner, and then each poll fails at once.
+*/
+const latchBand = (timing, poll, failure, requestTimeout) => {
+	const error = failSpan(timing, poll, "error");
+	if (failure === "refuses") return band(s$1(error.min), s$1(error.max));
+	const wait = requestTimeout === void 0 ? s$1(poll.timeout) : Math.min(s$1(requestTimeout), s$1(poll.timeout));
+	const late = failSpan(timing, {
+		...poll,
+		timeout: seconds(wait)
+	}, "timeout");
+	return band(s$1(error.min), s$1(late.max));
+};
+/** This function gives the time from a fault to the removal of the endpoints from the rules: the latch, then the kubelet, then the propagation. */
+const removalBand = (latch, probe, propagation) => {
+	const kubelet = secondsOf(kubeletBand(probe, false));
+	const trip = secondsOf(latch);
+	const spread = secondsOf(propagation);
+	return band(trip.lo + kubelet.lo + spread.lo, trip.hi + kubelet.hi + spread.hi);
+};
+/**
+* This function gives the time from a fault to the SIGTERM of a replica whose liveness fails with
+* its readiness. The time is the latch, then the liveness probes of the kubelet, then the preStop sleep.
+*/
+const restartBand = (latch, liveness, preStop) => {
+	const kubelet = secondsOf(kubeletBand(liveness, false));
+	const trip = secondsOf(latch);
+	return band(trip.lo + kubelet.lo + s$1(preStop), trip.hi + kubelet.hi + s$1(preStop));
+};
+/**
+* This function gives the time from the start of a stall of the event loop to the removal of the
+* replica from the rules. Each probe in the stall fails at its timeout, because the loop does not answer it.
+*/
+const stallRemovalBand = (readiness, propagation) => {
+	const kubelet = secondsOf(kubeletBand(readiness, true));
+	const spread = secondsOf(propagation);
+	return band(kubelet.lo + spread.lo, kubelet.hi + spread.hi);
+};
+/** This function gives the cap of a stage in a declaration, or zero when the declaration does not name the stage. */
+const capOf = (declaration, stage) => {
+	const found = declaration.phases.find(([name]) => name === stage);
+	return seconds(found === void 0 ? 0 : s$1(found[1]));
+};
+/**
+* This function gives the shortest time from the deletion of a pod to the stop of its intake. It
+* is the preStop sleep, then the hold of `fail readiness`. The kubelet adds its own delay before
+* the preStop, thus the real time is longer.
+*/
+const intakeAfter = (declaration, preStop) => seconds(us(s$1(preStop) + s$1(capOf(declaration, "fail readiness"))));
+/** This function gives the deletion window of a declaration and a preStop sleep on a node with a propagation. */
+const deletionWindow = (declaration, preStop, propagation) => {
+	const intake = intakeAfter(declaration, preStop);
+	const spread = secondsOf(propagation);
+	return {
+		empty: s$1(intake) >= spread.hi,
+		end: band(spread.lo, spread.hi),
+		intake
+	};
+};
+/**
+* This function gives the window on a node whose kube-proxy pauses across a deletion. The pause
+* starts `lead` before the deletion and holds for `hold`. The node then syncs in one propagation
+* after the end of the pause, at `resume` after the deletion. The window is empty when the intake
+* stops after that.
+*/
+const pausedWindow = (declaration, preStop, lead, hold, propagation) => {
+	const intake = intakeAfter(declaration, preStop);
+	const resume = us(s$1(hold) - s$1(lead));
+	const spread = secondsOf(propagation);
+	return {
+		empty: s$1(intake) >= resume + spread.hi,
+		end: band(spread.lo, spread.hi),
+		intake,
+		resume: seconds(resume)
+	};
+};
+//#endregion
+//#region src/rules.ts
+/**
+* These are the rules of the ordering law that a drain in Kubernetes meets, as data. The caller of
+* `lint()` gives the rules, thus they are here and not in the service.
+*/
+/**
+* O04 with the hold of the service. The listener of a pod stays open through the preStop sleep and
+* then through the hold of `fail readiness`. Thus the sum of the two must cover the propagation of
+* the endpoints. The rule of the research is the case without a hold.
+*/
+const O04 = {
+	id: "O04",
+	rule: "the preStop sleep plus the hold of fail readiness is at least the endpoint propagation",
+	window: "new connections reach a pod whose listener is closed",
+	expr: {
+		lhs: { terms: [{
+			knob: "prestop_sleep",
+			coef: 1,
+			at: "pod"
+		}, {
+			knob: "readiness_hold",
+			coef: 1,
+			at: "process"
+		}] },
+		op: ">=",
+		rhs: { terms: [{
+			knob: "propagation_bound",
+			coef: 1,
+			at: "cluster"
+		}] }
+	}
+};
+/** O09 is W8: readiness detection plus propagation is no longer than liveness detection plus the preStop sleep. */
+const O09 = {
+	id: "O09",
+	rule: "readiness detection plus endpoint propagation is no longer than liveness detection plus the preStop sleep",
+	window: "traffic reaches a container that its liveness probe kills",
+	expr: {
+		lhs: { terms: [{
+			knob: "readiness_detection_window",
+			coef: 1,
+			at: "pod"
+		}, {
+			knob: "propagation_bound",
+			coef: 1,
+			at: "cluster"
+		}] },
+		op: "<=",
+		rhs: { terms: [{
+			knob: "liveness_detection_window",
+			coef: 1,
+			at: "pod"
+		}, {
+			knob: "prestop_sleep",
+			coef: 1,
+			at: "pod"
+		}] }
+	}
+};
+/** O10: the preStop sleep plus the drain of the process fits in the grace period. */
+const O10 = {
+	id: "O10",
+	rule: "the preStop sleep plus the shutdown of the process fits in the grace period",
+	window: "work in flight at SIGKILL is cut, and each connection closes at once",
+	expr: {
+		lhs: { terms: [{
+			knob: "prestop_sleep",
+			coef: 1,
+			at: "pod"
+		}, {
+			knob: "process_drain_time",
+			coef: 1,
+			at: "process"
+		}] },
+		op: "<=",
+		rhs: { terms: [{
+			knob: "termination_grace",
+			coef: 1,
+			at: "pod"
+		}] }
+	}
+};
+/** O16: the cooperative window is shorter than the drain, which is shorter than the grace less the preStop sleep. */
+const O16 = {
+	id: "O16",
+	rule: "the cooperative window is shorter than the drain, which is shorter than the grace less the preStop sleep",
+	window: "handlers are cut before they answer, or the drain itself gets SIGKILL",
+	expr: {
+		lhs: { terms: [{
+			knob: "cooperative_window",
+			coef: 1,
+			at: "process"
+		}] },
+		op: "<",
+		rhs: { terms: [{
+			knob: "process_drain_time",
+			coef: 1,
+			at: "process"
+		}] }
+	},
+	expr_more: [{
+		lhs: { terms: [{
+			knob: "process_drain_time",
+			coef: 1,
+			at: "process"
+		}] },
+		op: "<",
+		rhs: { terms: [{
+			knob: "termination_grace",
+			coef: 1,
+			at: "pod"
+		}, {
+			knob: "prestop_sleep",
+			coef: -1,
+			at: "pod"
+		}] }
+	}]
+};
+/** These are the rules of the drain that each kind cell gets. */
+const DRAIN_RULES = [
+	O04,
+	O09,
+	O10,
+	O16
+];
+const seconds$1 = (amount) => valueIn(amount, "s");
+const s = (value) => ({
+	value,
+	unit: "s"
+});
+/**
+* This function gives the scenario of a cell for `lint()`: the preStop sleep, the grace, the probe
+* period and the propagation. The probe windows are the period times the threshold of each probe.
+*/
+const scenarioOf = (cell, propagation) => ({
+	platform: {
+		"prestop_sleep@pod": cell.pod.preStop,
+		"termination_grace@pod": cell.pod.grace,
+		"probe_period@pod": cell.pod.readiness.period,
+		"propagation_bound@cluster": propagation.hi
+	},
+	probes: {
+		readiness: s(seconds$1(cell.pod.readiness.period) * cell.pod.readiness.threshold),
+		liveness: s(seconds$1(cell.pod.liveness.period) * cell.pod.liveness.threshold)
+	}
+});
+/** This function applies the drain rules to the description of a cell, and gives each finding. */
+const lintCell = (description, cell, propagation) => lint(description, scenarioOf(cell, propagation), DRAIN_RULES);
+/** This function gives the verdict of each drain rule, by its identifier. */
+const verdictsOf = (findings) => Object.fromEntries(findings.map((finding) => [finding.rule, finding.verdict]));
+/** This function gives the description of the service of a cell, as the app of the image composes it. Nothing connects. */
+const describeCell = (cell) => {
+	const redis = cell.declaration.redis;
+	const client = redis === null ? void 0 : redisClient("redis", {
+		host: "redis",
+		port: 6379,
+		system: "systems.component.redis",
+		knobs: redis.knobs,
+		heartbeatMisses: redis.misses
+	});
+	return compose("lab", cell.declaration, { redis: client }).describe();
+};
+//#endregion
+//#region src/run.ts
+/** This time is the interval of the poll of the rules on each node. */
+const RULES_EVERY = {
+	value: 50,
+	unit: "ms"
+};
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const slug = (name) => name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
+/** This tracker keeps the latest state of the cell from the lines: each pod, each slice, the backends of each node and each client. */
+var Tracker = class {
+	pods = /* @__PURE__ */ new Map();
+	slices = /* @__PURE__ */ new Map();
+	backends = /* @__PURE__ */ new Map();
+	clients = /* @__PURE__ */ new Map();
+	last = 0;
+	see = (line) => {
+		if (typeof line.t === "number" && line.t > this.last && line.ev === "req") this.last = line.t;
+		if (line.ev === "pod") {
+			const pod = line;
+			if (pod.type === "DELETED") this.pods.delete(pod.name);
+			else this.pods.set(pod.name, pod);
+		} else if (line.ev === "slice") {
+			if (line["type"] === "DELETED") this.slices.delete(String(line["name"]));
+			else this.slices.set(String(line["name"]), line);
+		} else if (line.ev === "rules") this.backends.set(String(line["node"]), line["b"]);
+		else if (line.ev === "req") {
+			const request = line;
+			const client = this.clients.get(request.p) ?? {
+				node: request.n,
+				ok: 0,
+				failed: 0
+			};
+			if (request.o === "ok") client.ok = Math.max(client.ok, request.t);
+			else client.failed = Math.max(client.failed, request.t);
+			this.clients.set(request.p, client);
+		}
+	};
+	servers() {
+		return [...this.pods.values()].filter((pod) => pod.labels["app"] === "lab").sort((a, b) => a.name.localeCompare(b.name));
+	}
+};
+/** This function tells why the cell is not at rest, or gives null when it is. */
+const unsteady = (ctx) => {
+	const { tracker, cell } = ctx;
+	const servers = tracker.servers();
+	if (servers.length !== ctx.expected) return `${servers.length} replicas, not ${ctx.expected}`;
+	const busy = servers.find((pod) => pod.del || !pod.ready);
+	if (busy !== void 0) return `${busy.name} is not ready`;
+	if (new Set(servers.map((pod) => pod.labels["pod-template-hash"])).size > 1) return "two versions of the replicas";
+	if (cell.declaration.redis !== null) {
+		const redis = [...tracker.pods.values()].filter((pod) => pod.labels["app"] === REDIS);
+		if (redis.length !== 1 || !redis[0].ready || redis[0].del) return "Redis is not ready";
+	}
+	const ips = servers.map((pod) => pod.ip ?? "").sort();
+	const ready = [...tracker.slices.values()].filter((slice) => slice["svc"] === "lab").flatMap((slice) => slice["eps"].filter((endpoint) => endpoint.r).map((endpoint) => endpoint.ip)).sort();
+	if (ready.join() !== ips.join()) return `the EndpointSlice has ${ready.join(" ")}, not ${ips.join(" ")}`;
+	const expected = ips.map((ip) => `${ip}:8080`).sort().join();
+	for (const node of ctx.cluster.nodes) if ((tracker.backends.get(node) ?? []).join() !== expected) return `the rules of ${node} have ${(tracker.backends.get(node) ?? []).join(" ")}`;
+	const nodes = new Set([...tracker.clients.values()].map((client) => client.node));
+	if (nodes.size !== ctx.cluster.nodes.length) return `clients answer on ${nodes.size} nodes`;
+	for (const [pod, client] of tracker.clients) {
+		if (tracker.last - client.ok > 1e3) return `client ${pod} has no answer in the last second`;
+		if (tracker.last - client.failed < 2e3) return `client ${pod} failed in the last 2 s`;
+	}
+	return null;
+};
+/** This function waits until the cell is at rest, and gives null, or the reason when it is not at rest in time. */
+const steady = async (ctx, within) => {
+	const end = Date.now() + within;
+	let why = unsteady(ctx);
+	while (why !== null && Date.now() < end) {
+		await sleep(200);
+		why = unsteady(ctx);
+	}
+	return why;
+};
+/** This function samples the load of the VM for 4 s, and names the lab containers that are not of this run. */
+const sample = async (ctx, ours) => {
+	const second = (await ctx.loader.exec([
+		"top",
+		"-b",
+		"-n",
+		"2",
+		"-d",
+		"4"
+	])).stdout.split(/^Mem:/m).at(-1) ?? "";
+	const idle = Number(/(\d+)% idle/.exec(second)?.[1] ?? "0") / 100;
+	const top = second.split("\n").filter((line) => /^\s*\d+\s/.test(line)).slice(0, 6).map((line) => line.trim().replace(/\s+/g, " "));
+	const foreign = (await command("docker", [
+		"ps",
+		"--format",
+		"{{.ID}} {{.Names}} {{.Image}}"
+	])).trim().split("\n").filter((line) => line !== "").filter((line) => !ours.has(line.split(" ")[1]) && !ours.has(line.split(" ")[0])).filter((line) => /loom-|kindest\/node|toxiproxy|haproxy|redis|systems-kind-lab|f6-/.test(line.split(" ")[2] ?? ""));
+	return {
+		t: Date.now(),
+		idle,
+		top,
+		foreign
+	};
+};
+/** This function waits until the VM is quiet: no lab container of another agent, and an idle share of the CPUs. */
+const quiet = async (ctx, ours) => {
+	for (;;) {
+		const load = await sample(ctx, /* @__PURE__ */ new Set([
+			...ours,
+			...ctx.lab.own(),
+			ctx.loader.getName().replace(/^\//, "")
+		]));
+		ctx.loads.push(load);
+		if (load.foreign.length === 0 && load.idle >= ctx.idle) return load;
+		ctx.log(`  waiting: ${Math.round(load.idle * 100)}% idle, other lab containers: ${load.foreign.join("; ") || "none"}`);
+		await sleep(3e4);
+	}
+};
+/** This function deploys a cell in its namespace, with the watcher, the poll of the rules and the clients. */
+const deploy = async (lab, cluster, cell, namespace, redisImage) => {
+	await cluster.apply([{
+		apiVersion: "v1",
+		kind: "Namespace",
+		metadata: { name: namespace }
+	}]);
+	await lab.watch([namespace]);
+	if (cell.declaration.redis !== null) {
+		await cluster.apply(redisObjects(namespace, redisImage));
+		await cluster.kubectl([
+			"-n",
+			namespace,
+			"rollout",
+			"status",
+			`deployment/${REDIS}`,
+			"--timeout=180s"
+		]);
+	}
+	await cluster.apply(serverObjects(cell, namespace, lab.image));
+	const clusterIp = (await cluster.kubectl([
+		"-n",
+		namespace,
+		"get",
+		"service",
+		"lab",
+		"-o",
+		"jsonpath={.spec.clusterIP}"
+	])).trim();
+	await cluster.apply(clientObjects(cell, namespace, lab.image, `${clusterIp}:80`));
+	await cluster.kubectl([
+		"-n",
+		namespace,
+		"rollout",
+		"status",
+		`deployment/lab`,
+		"--timeout=300s"
+	]);
+	await cluster.kubectl([
+		"-n",
+		namespace,
+		"rollout",
+		"status",
+		`daemonset/${CLIENT}`,
+		"--timeout=300s"
+	]);
+	await lab.rules({
+		mode: cluster.scenario.kubeProxy.mode,
+		ns: namespace,
+		svc: "lab",
+		ip: clusterIp,
+		port: 80,
+		every: RULES_EVERY
+	});
+	return clusterIp;
+};
+/** This function starts the prober again when the pods of the cell changed: it watches the namespace of each pod. */
+const probeThePods = async (ctx) => {
+	const pods = [...ctx.tracker.pods.values()].filter((pod) => !pod.del && pod.node !== null && pod.cs.every((status) => status.st === "running"));
+	const key = pods.map((pod) => `${pod.name}/${pod.cs.map((status) => status.id).join("+")}`).sort().join();
+	if (key === ctx.probed) return;
+	const netns = [];
+	for (const pod of pods) {
+		const container = pod.labels["app"] === "lab" ? "app" : pod.labels["app"] === "client" ? CLIENT : REDIS;
+		const pid = await ctx.lab.pidOf(pod.node, ctx.namespace, pod.name, container);
+		netns.push(await ctx.lab.netnsOf(pod.node, pid));
+	}
+	await ctx.lab.probe([...new Set(netns)]);
+	ctx.probed = key;
+};
+/** This function gives the replicas of the service at this time, in the sequence of their names. */
+const replicasOf = (ctx) => ctx.tracker.servers().map((pod) => ({
+	pod: pod.name,
+	node: pod.node ?? "",
+	ip: pod.ip ?? "",
+	version: pod.labels["pod-template-hash"] ?? ""
+}));
+/** This function does one step of a fault plan. It gives the stamp of its act, or null for an act without a stamp. */
+const doStep = async (ctx, step, replicas, background) => {
+	const { lab, cluster, namespace } = ctx;
+	const target = replicas[0];
+	const fill = (args) => args.map((arg) => arg.replaceAll("{namespace}", namespace).replaceAll(POD, target.pod).replaceAll(NODE, target.node).replaceAll(VERSION, String(ctx.version)));
+	const where = async (which) => {
+		if (which === "server") return {
+			node: target.node,
+			pid: await lab.pidOf(target.node, namespace, target.pod, "app"),
+			pod: null
+		};
+		if (which === "redis") {
+			const redis = [...ctx.tracker.pods.values()].find((pod) => pod.labels["app"] === "redis" && !pod.del);
+			return {
+				node: redis.node,
+				pid: await lab.pidOf(redis.node, namespace, redis.name, REDIS),
+				pod: redis
+			};
+		}
+		const worker = cluster.nodes[1] ?? cluster.nodes[0];
+		const proxy = JSON.parse(await cluster.kubectl([
+			"-n",
+			"kube-system",
+			"get",
+			"pods",
+			"-l",
+			"k8s-app=kube-proxy",
+			"--field-selector",
+			`spec.nodeName=${worker}`,
+			"-o",
+			"json"
+		]));
+		return {
+			node: worker,
+			pid: await lab.pidOf(worker, "kube-system", proxy.items[0].metadata.name, "kube-proxy"),
+			pod: null
+		};
+	};
+	switch (step.do) {
+		case "kubectl": {
+			const t0 = await lab.now();
+			if (step.args.includes(`LAB_VERSION={version}`)) ctx.version += 1;
+			const call = cluster.kubectl(fill(step.args));
+			if (step.background === true) {
+				background.push(call.catch((error) => ctx.log(`  kubectl ${step.args.join(" ")}: ${String(error)}`)));
+				return {
+					t0,
+					t1: t0
+				};
+			}
+			await call;
+			return {
+				t0,
+				t1: await lab.now()
+			};
+		}
+		case "signal": {
+			const at = await where(step.target);
+			return lab.act(at.node, {
+				op: "signal",
+				pid: at.pid,
+				signal: step.signal
+			});
+		}
+		case "pause": {
+			const at = await where(step.target);
+			const stamp = lab.act(at.node, {
+				op: "pause",
+				pid: at.pid,
+				ms: valueIn(step.time, "ms")
+			});
+			background.push(stamp);
+			const now = await lab.now();
+			return {
+				t0: now,
+				t1: now
+			};
+		}
+		case "inject": {
+			const at = await where(step.target);
+			let first = null;
+			for (const replica of replicas) {
+				const stamp = await lab.inject(at.node, at.pid, [step.action, replica.ip]);
+				first ??= stamp;
+			}
+			return first;
+		}
+		case "flag": {
+			const at = await where(step.target);
+			return step.on ? lab.act(at.node, {
+				op: "write",
+				path: `/proc/${at.pid}/root/tmp/lab-fail`,
+				data: "1"
+			}) : lab.act(at.node, {
+				op: "remove",
+				path: `/proc/${at.pid}/root/tmp/lab-fail`
+			});
+		}
+		case "cpu": {
+			const at = await where(step.target);
+			const cgroup = await lab.cgroupOf(at.node, at.pid);
+			return lab.act(at.node, {
+				op: "write",
+				path: `${cgroup}/cpu.max`,
+				data: step.max
+			});
+		}
+		case "stall": {
+			const at = await where(step.target);
+			return lab.act(at.node, {
+				op: "stall",
+				pid: at.pid,
+				ms: valueIn(step.time, "ms")
+			});
+		}
+		case "wait":
+			await sleep(valueIn(step.time, "ms"));
+			return null;
+	}
+};
+/** This function tells if a run is over: the trigger is done, and the cell is at rest again. */
+const over = (ctx, run, lines) => {
+	if (unsteady(ctx) !== null) return false;
+	const kind = ctx.cell.trigger.kind;
+	if ((kind === "crash" || kind === "liveness-restart") && run.target !== null) {
+		const pod = ctx.tracker.pods.get(run.target.pod);
+		return (pod === void 0 ? 0 : pod.cs.reduce((sum, status) => sum + status.rc, 0)) > (run.restarts.get(run.target.pod) ?? 0);
+	}
+	if (run.target !== null && [
+		"pod-deletion",
+		"node-drain",
+		"scale-in",
+		"freeze-process"
+	].includes(kind) && ctx.cell.template === "T07") return kind === "scale-in" || kind === "node-drain" ? run.replicas.some((replica) => !ctx.tracker.pods.has(replica.pod)) : !ctx.tracker.pods.has(run.target.pod);
+	if (kind === "rolling-update" || kind === "rollout") return run.replicas.every((replica) => !ctx.tracker.pods.has(replica.pod));
+	if (kind === "app-stall") return lines.some((line) => line.ev === "stall" && line["phase"] === "end");
+	return true;
+};
+/** This function does one run of a cell: at rest, the trigger, the heal, then the observation until the cell is at rest again. */
+const runOnce = async (ctx, env, index, warmup, ours) => {
+	const { lab, cell, cluster, namespace } = ctx;
+	const plan = planOf(cell);
+	const atRest = await steady(ctx, 3e5);
+	const empty = (why, start) => ({
+		env,
+		index,
+		warmup,
+		start,
+		end: start,
+		acts: {
+			fault: null,
+			heal: null,
+			resume: null
+		},
+		target: null,
+		paused: null,
+		replicas: [],
+		void: why,
+		lines: lab.take(),
+		dropped: {}
+	});
+	if (atRest !== null) return empty(`the cell was not at rest: ${atRest}`, await lab.now());
+	await probeThePods(ctx);
+	await quiet(ctx, ours);
+	if (await steady(ctx, 3e5) !== null) return empty("the cell was not at rest after the wait for a quiet VM", await lab.now());
+	const replicas = replicasOf(ctx);
+	const restarts = new Map(ctx.tracker.servers().map((pod) => [pod.name, pod.cs.reduce((sum, status) => sum + status.rc, 0)]));
+	const target = cell.declaration.redis !== null && [
+		"rst",
+		"blackhole",
+		"freeze-process",
+		"peer-gone-without-close",
+		"rollout"
+	].includes(cell.trigger.kind) ? null : replicas[0];
+	const paused = cell.template === "T07" && cell.trigger.kind === "freeze-process" ? cluster.nodes[1] ?? cluster.nodes[0] : null;
+	lab.take();
+	for (const key of Object.keys(lab.dropped)) delete lab.dropped[key];
+	const start = await lab.now();
+	const background = [];
+	let fault = null;
+	let resume = null;
+	if (cell.trigger.kind === "scale-in") ctx.expected = cell.pod.replicas - 1;
+	const notes = [];
+	for (const step of plan.inject) {
+		const stamp = await doStep(ctx, step, replicas, background);
+		if (stamp !== null && fault === null) fault = stamp.t0;
+	}
+	let heal = null;
+	if (plan.hold !== null) {
+		const now = await lab.now();
+		await sleep(Math.max(0, valueIn(plan.hold, "ms") - (now - (fault ?? now))));
+		for (const step of plan.heal) try {
+			const stamp = await doStep(ctx, step, replicas, background);
+			if (stamp !== null && heal === null) heal = stamp.t0;
+		} catch (error) {
+			notes.push(`the heal ${step.do} failed: ${String(error)}`);
+		}
+	}
+	const settled = await Promise.allSettled(background);
+	for (const each of settled) if (each.status === "fulfilled" && typeof each.value === "object" && each.value !== null && "t1" in each.value) resume = each.value.t1;
+	const limit = Date.now() + 24e4 + valueIn(cell.pod.grace, "ms") + valueIn(cell.pod.preStop, "ms");
+	await sleep(3e3);
+	while (Date.now() < limit && !over(ctx, {
+		target,
+		replicas,
+		restarts
+	}, lab.lines)) await sleep(250);
+	const end = await lab.now();
+	const timedOut = Date.now() >= limit;
+	for (const step of plan.restore) try {
+		await doStep(ctx, step, replicas, background);
+	} catch (error) {
+		notes.push(`the restore ${step.do} failed: ${String(error)}`);
+	}
+	ctx.expected = cell.pod.replicas;
+	for (const note of notes) ctx.log(`  ${note}`);
+	const lines = lab.take();
+	const stopped = deletes(cell) ? replicas.find((replica) => lines.some((line) => line.ev === "pod" && line["name"] === replica.pod && line["del"] === true)) ?? target : target;
+	if (lines.some((line) => line.ev === "pod" && line["cs"].some((status) => status.rc > 0))) {
+		await cluster.kubectl([
+			"-n",
+			namespace,
+			"rollout",
+			"restart",
+			`deployment/lab`
+		]);
+		await cluster.kubectl([
+			"-n",
+			namespace,
+			"rollout",
+			"status",
+			`deployment/lab`,
+			"--timeout=300s"
+		]);
+	}
+	if (cell.trigger.kind === "node-drain") await rebalance(ctx);
+	ctx.log(`  ${warmup ? "warm-up" : `run ${index}`}: ${sec(end - start)} s${timedOut ? ", not at rest in time" : ""}`);
+	return {
+		env,
+		index,
+		warmup,
+		start,
+		end,
+		acts: {
+			fault,
+			heal,
+			resume
+		},
+		target: stopped,
+		paused,
+		replicas,
+		void: null,
+		lines,
+		dropped: { ...lab.dropped },
+		notes
+	};
+};
+const sec = (ms) => Math.round(ms) / 1e3;
+/** This function moves a replica back to a node without one after a drain: it deletes one replica of the node with two. */
+const rebalance = async (ctx) => {
+	await steady(ctx, 3e5);
+	const servers = ctx.tracker.servers();
+	const byNode = /* @__PURE__ */ new Map();
+	for (const pod of servers) byNode.set(pod.node ?? "", [...byNode.get(pod.node ?? "") ?? [], pod.name]);
+	const crowded = [...byNode.values()].find((pods) => pods.length > 1);
+	if (crowded !== void 0 && byNode.size < ctx.cluster.nodes.length - 1) await ctx.cluster.kubectl([
+		"-n",
+		ctx.namespace,
+		"delete",
+		"pod",
+		crowded[0],
+		"--wait=true"
+	]);
+};
+/** This function writes the result of a cell: its raw record, and one line of its verdict in `summary.ndjson`. */
+const writeResult = (out, result) => {
+	mkdirSync(out, { recursive: true });
+	const record = {
+		format: FORMAT,
+		...result
+	};
+	writeFileSync(join(out, `${slug(result.cell.name)}.json`), JSON.stringify(record));
+	const { verdict, cell } = result;
+	writeFileSync(join(out, "summary.ndjson"), `${JSON.stringify({
+		id: cell.id,
+		name: cell.name,
+		mode: cell.mode,
+		trigger: cell.trigger.kind,
+		prediction: cell.prediction.kind,
+		decision: verdict.decision,
+		counted: verdict.counted,
+		voided: verdict.voided,
+		failed: verdict.failed,
+		median: verdict.median,
+		why: verdict.why
+	})}\n`, { flag: "a" });
+};
+/**
+* This function does the runs of a catalog of kind cells. The cells of one kube-proxy mode and node
+* count share a cluster. Each group gets new clusters in sequence, one for each environment. Each
+* environment does its share of the runs of each cell, after one warm-up. The function deletes each
+* cluster after its cells.
+*/
+const runKindCells = async (cells, options) => {
+	const environments = options.environments ?? 2;
+	const image = await labImage();
+	const redisImage = await localPin(REDIS_IMAGE);
+	const groups = /* @__PURE__ */ new Map();
+	for (const cell of cells) groups.set(`${cell.mode}-${cell.workers}`, [...groups.get(`${cell.mode}-${cell.workers}`) ?? [], cell]);
+	const results = [];
+	for (const [key, group] of groups) {
+		const partial = /* @__PURE__ */ new Map();
+		for (let env = 0; env < environments; env += 1) {
+			const s = scenario(`lab-${key}-${env}`, group[0].mode, group[0].workers);
+			const dir = join(options.work ?? join(tmpdir(), "systems-kind-work"), s.name);
+			options.log(`cluster ${s.name}: ${s.kubeProxy.mode}, ${s.workers} workers`);
+			const cluster = await createCluster(s, dir);
+			let lab = null;
+			let loader = null;
+			try {
+				await cluster.load(image);
+				await cluster.load(redisImage);
+				lab = await openKindLab(cluster, image);
+				const { GenericContainer } = await import("testcontainers");
+				const Shared = class extends GenericContainer {
+					withPidMode(mode) {
+						this.hostConfig.PidMode = mode;
+						return this;
+					}
+				};
+				loader = await new Shared(image).withPidMode("host").withNetworkMode("none").withCommand(["sleep", "infinity"]).start();
+				const manifest = await cluster.manifest();
+				const ours = /* @__PURE__ */ new Set([...cluster.nodes]);
+				for (const cell of group) {
+					const namespace = namespaceOf(cell);
+					options.log(`${cell.name} (${cell.id}), environment ${env}`);
+					const tracker = new Tracker();
+					const off = lab.listen(tracker.see);
+					const loads = [];
+					try {
+						const clusterIp = await deploy(lab, cluster, cell, namespace, redisImage);
+						for (const name of [...lab.own(), loader.getName().replace(/^\//, "")]) ours.add(name);
+						const ctx = {
+							lab,
+							cluster,
+							cell,
+							namespace,
+							tracker,
+							clusterIp,
+							loads,
+							loader,
+							idle: options.idle ?? .6,
+							log: options.log,
+							version: 1,
+							probed: "",
+							expected: cell.pod.replicas
+						};
+						const share = Math.ceil(cell.runs / environments) - (env < cell.runs % environments || cell.runs % environments === 0 ? 0 : 1);
+						const runs = [await runOnce(ctx, env, -1, true, ours)];
+						for (let index = 0; index < share; index += 1) runs.push(await runOnce(ctx, env, index, false, ours));
+						const entry = partial.get(cell.name) ?? {
+							runs: [],
+							manifests: [],
+							loads: []
+						};
+						entry.runs.push(...runs);
+						entry.manifests.push({
+							env,
+							scenario: s,
+							cluster: manifest,
+							images: {
+								lab: image,
+								redis: redisImage
+							}
+						});
+						entry.loads.push(...loads);
+						partial.set(cell.name, entry);
+						mkdirSync(join(options.out, "partial"), { recursive: true });
+						writeFileSync(join(options.out, "partial", `${slug(cell.name)}.env${env}.json`), JSON.stringify({
+							cell,
+							runs,
+							manifest,
+							loads
+						}));
+					} finally {
+						off();
+						await lab.rules(null).catch(() => void 0);
+						await lab.stopProbe();
+						await cluster.kubectl([
+							"delete",
+							"namespace",
+							namespace,
+							"--wait=true",
+							"--timeout=300s"
+						]).catch((error) => options.log(`  ${String(error)}`));
+					}
+				}
+			} finally {
+				if (loader !== null) await loader.stop({ timeout: 0 }).catch(() => void 0);
+				if (lab !== null) await lab.close();
+				await cluster.delete();
+				options.log(`cluster ${s.name} deleted`);
+			}
+		}
+		for (const cell of group) {
+			const entry = partial.get(cell.name);
+			if (entry === void 0) continue;
+			const result = {
+				cell,
+				...entry,
+				verdict: judgeKind(cell, entry.runs),
+				at: (/* @__PURE__ */ new Date()).toISOString()
+			};
+			writeResult(options.out, result);
+			rmSync(join(options.out, "partial", `${slug(cell.name)}.env0.json`), { force: true });
+			for (let env = 1; env < environments; env += 1) rmSync(join(options.out, "partial", `${slug(cell.name)}.env${env}.json`), { force: true });
+			options.log(`${cell.name}: ${result.verdict.decision}, ${result.verdict.why}`);
+			results.push(result);
+		}
+	}
+	return results;
+};
+/** This function gives the estimate of a cell in seconds, for the arithmetic of the shards. */
+const estimate = (cell, environments = 2) => {
+	const hold = cell.trigger.hold === void 0 ? 0 : valueIn(cell.trigger.hold, "s");
+	const each = 25 + ({
+		"liveness-restart": 100,
+		crash: 30,
+		"app-stall": hold + 40,
+		"cpu-throttle": hold + 40,
+		rollout: hold + 50
+	}[cell.trigger.kind] ?? (cell.template === "T18" ? hold + 35 : 15 + valueIn(cell.pod.preStop, "s")));
+	return 60 * environments + (cell.runs + environments) * each;
+};
+/**
+* This function gives an estimate of the time of a cell: its setup, then each run and each
+* warm-up. A run waits for the cell at rest, samples the load, does the trigger and waits until the
+* cell is at rest again.
+*/
+const estimatedTime = (cell, environments = 2) => ({
+	value: estimate(cell, environments),
+	unit: "s"
+});
+/**
+* This function gives the share of a shard of a catalog: cells in sequence. Thus the cells of one
+* mode stay together and share clusters. Each shard gets an equal share of the estimated time.
+*/
+const shardOf = (cells, index, count) => {
+	if (!(Number.isInteger(index) && Number.isInteger(count) && count >= 1 && index >= 0 && index < count)) throw new RangeError(`shard ${index} of ${count} is not a shard`);
+	const ordered = [...cells].sort((a, b) => a.mode === b.mode ? 0 : a.mode < b.mode ? -1 : 1);
+	const total = ordered.reduce((sum, cell) => sum + estimate(cell), 0);
+	const out = [];
+	let before = 0;
+	for (const cell of ordered) {
+		const middle = before + estimate(cell) / 2;
+		if (Math.min(count - 1, Math.floor(middle / total * count)) === index) out.push(cell);
+		before += estimate(cell);
+	}
+	return out;
+};
+//#endregion
+export { CLIENT, CONTROLLER, DOCUMENTED, DRAIN_RULES, FLAG_CHECK, FORMAT, FORMAT_V1, HEALTH_PORT, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LOG_LABEL, NODE_IMAGES, O04, O09, O10, O16, PORT, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, STAGES, WATCHDOG, backendsAt, backendsOf, bandOf, byNode, cacheFolder, calibrationOf, capOf, cellsTable, checkRun, cidrOverlap, cidrRange, clientObjects, compose, createCluster, deletes, deletionOf, deletionWindow, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, goDuration, intakeAfter, intakeOf, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, latchBand, lintCell, localPin, medianInterval, medianText, middle, namespaceOf, nodeNames, openKindLab, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, predictionText, probeOf, propagationOf, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartsAt, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverObjects, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, stageLabImage, stallRemovalBand, timeSpread, toolPath, transitionOf, unitOf, upgrade, upgradeCell, verdictsOf, writeResult };
+
+//# sourceMappingURL=index.js.map