@mark1russell7/kind 0.2.2 → 0.3.0

package/build/index.js

@@ -1,4 +1,4 @@
-import { $ as HEALTH_PORT, A as byClients, B as podsOf, C as reusesReset, D as middle, E as medianInterval, F as failuresOf, G as signalOf, H as requestsOf, I as intakeOf, J as sliceEmptyOf, K as signalsOf, L as kernelCounts, M as deletionOf, N as dropOf, O as quantileOf, P as emptyOf, Q as CLIENT, R as outcomesOf, S as referenceOf, T as RELATIVE, U as restartsAt, V as recordsOf, W as rulesOf, X as stoppingOf, Y as slicesOf, Z as transitionOf, _ as checkRun, a as calibrationOf, at as seconds, b as judgeKind, c as predictionText, d as FORMAT_V1, et as LOG_LABEL, f as judge, g as LAST_SLACK, h as KIND_NOISE, i as readRecord, it as bandOf, j as byNode, k as backendsAt, l as timeSpread, m as upgradeCell, n as kindCellRecordSchema, nt as REDIS, o as cellsTable, ot as secondsOf, p as upgrade, q as sliceDropOf, r as kindCellRecordV1Schema, rt as SERVER, s as medianText, st as showBand, t as STAGES, tt as PORT, u as FORMAT, v as deletionsOf, w as unitOf, x as propagationOf, y as factsOf, z as podsAt } from "./judge-reWPMVr4.js";
+import { $ as stoppingOf, A as middle, B as kernelCounts, C as propagationOf, D as unitOf, E as reusesReset, F as deletionOf, G as requestsOf, H as podsAt, I as dropOf, J as signalOf, K as restartsAt, L as emptyOf, M as backendsAt, N as byClients, O as RELATIVE, P as byNode, Q as slicesOf, R as failuresOf, S as observationEnd, T as referenceOf, U as podsOf, V as outcomesOf, W as recordsOf, X as sliceDropOf, Y as signalsOf, Z as sliceEmptyOf, _ as checkRun, a as calibrationOf, at as REDIS, b as judgeKind, c as predictionText, ct as seconds, d as FORMAT_V1, et as transitionOf, f as judge, g as LAST_SLACK, h as KIND_NOISE, i as readRecord, it as PORT, j as quantileOf, k as medianInterval, l as timeSpread, lt as secondsOf, m as upgradeCell, n as kindCellRecordSchema, nt as HEALTH_PORT, o as cellsTable, ot as SERVER, p as upgrade, q as rulesOf, r as kindCellRecordV1Schema, rt as LOG_LABEL, s as medianText, st as bandOf, t as STAGES, tt as CLIENT, u as FORMAT, ut as showBand, v as deletionsOf, w as propagationSource, x as observationAfter, y as factsOf, z as intakeOf } from "./judge-CRMYoEOw.js";
 import { valueIn } from "@mark1russell7/quantity";
 import { execFile, execFileSync } from "node:child_process";
 import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
@@ -6,6 +6,7 @@ import { dirname, join } from "node:path";
 import { promisify } from "node:util";
 import { createHash } from "node:crypto";
 import { fileURLToPath } from "node:url";
+import { gunzipSync } from "node:zlib";
 import { failSpan, lint, service } from "@mark1russell7/service";
 import { createRequire } from "node:module";
 import { tmpdir } from "node:os";
@@ -227,6 +228,57 @@ const KUBECTL = {
 		}
 	}
 };
+/**
+* This release of istioctl is 1.31.3, from 2026-10-22. It holds the charts of its own version, thus the
+* lab needs no other tool to install Istio. The sums are from the `.sha256` assets of the release. The
+* release has a zip file for Windows, and the lab reads only a gzipped tar file. Thus Windows has no asset.
+*/
+const ISTIOCTL = {
+	name: "istioctl",
+	version: "1.31.3",
+	entry: "istioctl",
+	assets: {
+		"linux-x64": {
+			url: "https://github.com/istio/istio/releases/download/1.31.3/istioctl-1.31.3-linux-amd64.tar.gz",
+			sha256: "382c3004ce2e9c5ed3064b092ae0da7f6bbe30c766a1be282f45725add004860"
+		},
+		"linux-arm64": {
+			url: "https://github.com/istio/istio/releases/download/1.31.3/istioctl-1.31.3-linux-arm64.tar.gz",
+			sha256: "7476801623dbb900073cb1dba6a23b6a15c564b7e92b208aed36829315fefff1"
+		},
+		"darwin-arm64": {
+			url: "https://github.com/istio/istio/releases/download/1.31.3/istioctl-1.31.3-osx-arm64.tar.gz",
+			sha256: "881c3b443d9e458e8f19feed6ffa187dcfcb03dc032cb56068f7d1d0cd976c82"
+		}
+	}
+};
+/**
+* This function gives the bytes of one file of a gzipped tar file. A header of the ustar format has
+* 512 bytes. It has the name at 0, the size in octal at 124, the type at 156 and the prefix of the
+* name at 345. The data follows the header, in blocks of 512 bytes. The match is on the last part of the
+* name. A link, a folder and an extended header are not a file, thus the search skips them.
+*/
+const tarEntry = (gzipped, entry) => {
+	const tar = gunzipSync(gzipped);
+	const text = (from, length) => {
+		const field = tar.subarray(from, from + length);
+		const end = field.indexOf(0);
+		return field.subarray(0, end === -1 ? field.length : end).toString("latin1");
+	};
+	let at = 0;
+	while (at + 512 <= tar.length) {
+		const name = text(at, 100);
+		if (name === "") break;
+		const size = Number.parseInt(text(at + 124, 12).trim() || "0", 8);
+		if (!Number.isFinite(size)) throw new Error(`the tar entry ${name} has no size in octal`);
+		const type = String.fromCharCode(tar[at + 156] ?? 0);
+		const prefix = text(at + 345, 155);
+		const full = prefix === "" ? name : `${prefix}/${name}`;
+		if ((type === "0" || type === "\0") && full.split("/").at(-1) === entry) return new Uint8Array(tar.subarray(at + 512, at + 512 + size));
+		at += 512 + Math.ceil(size / 512) * 512;
+	}
+	throw new Error(`the tar file has no file ${entry}`);
+};
 /** This folder is the cache of the tools: `.cache/` in this package, which git ignores. `KIND_CACHE` names another folder. */
 const cacheFolder = () => process.env["KIND_CACHE"] ?? fileURLToPath(new URL("../.cache/", import.meta.url));
 /** This function gives the platform key of this process, as the assets name it. */
@@ -235,29 +287,43 @@ const platformKey = (platform = process.platform, arch = process.arch) => `${pla
 const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
 /** This function gives the path of a tool in the cache, for a platform. */
 const toolPath = (tool, key = platformKey(), cache = cacheFolder()) => join(cache, "tools", `${tool.name}-${tool.version}-${key}`, key.startsWith("win32") ? `${tool.name}.exe` : tool.name);
+/** This function writes a file through a partial file, thus a stop in the middle leaves no file at the path. */
+const writeWhole = (path, bytes, executable) => {
+	mkdirSync(dirname(path), { recursive: true });
+	const partial = `${path}.partial`;
+	writeFileSync(partial, bytes);
+	if (executable) chmodSync(partial, 493);
+	renameSync(partial, path);
+};
 /**
 * This function gives the path of a tool, after a download when the cache does not have it. It
-* refuses bytes whose sha256 is not the pinned value, from the cache or from the download.
+* refuses bytes whose sha256 is not the pinned value, from the cache or from the download. For a
+* tool in a tar file, the cache keeps the tar file beside the tool. Each use examines the tar file,
+* and writes the tool again when its bytes are not those of the entry.
 */
 const ensureTool = async (tool, key = platformKey(), cache = cacheFolder()) => {
 	const asset = tool.assets[key];
 	if (asset === void 0) throw new Error(`${tool.name} ${tool.version} has no pinned asset for ${key}`);
 	const path = toolPath(tool, key, cache);
-	if (existsSync(path)) {
-		const found = sha256(readFileSync(path));
-		if (found !== asset.sha256) throw new Error(`${path} has the sha256 ${found}, not the pinned ${asset.sha256}`);
-		return path;
+	const executable = !key.startsWith("win32");
+	const kept = tool.entry === void 0 ? path : `${path}.tar.gz`;
+	let bytes;
+	if (existsSync(kept)) {
+		bytes = readFileSync(kept);
+		const found = sha256(bytes);
+		if (found !== asset.sha256) throw new Error(`${kept} has the sha256 ${found}, not the pinned ${asset.sha256}`);
+		if (tool.entry === void 0) return path;
+	} else {
+		const response = await fetch(asset.url, { redirect: "follow" });
+		if (!response.ok) throw new Error(`the download of ${asset.url} answered ${response.status}`);
+		bytes = new Uint8Array(await response.arrayBuffer());
+		const found = sha256(bytes);
+		if (found !== asset.sha256) throw new Error(`${asset.url} has the sha256 ${found}, not the pinned ${asset.sha256}`);
+		writeWhole(kept, bytes, executable && tool.entry === void 0);
+		if (tool.entry === void 0) return path;
 	}
-	const response = await fetch(asset.url, { redirect: "follow" });
-	if (!response.ok) throw new Error(`the download of ${asset.url} answered ${response.status}`);
-	const bytes = new Uint8Array(await response.arrayBuffer());
-	const found = sha256(bytes);
-	if (found !== asset.sha256) throw new Error(`${asset.url} has the sha256 ${found}, not the pinned ${asset.sha256}`);
-	mkdirSync(dirname(path), { recursive: true });
-	const partial = `${path}.partial`;
-	writeFileSync(partial, bytes);
-	if (!key.startsWith("win32")) chmodSync(partial, 493);
-	renameSync(partial, path);
+	const binary = tarEntry(bytes, tool.entry);
+	if (!existsSync(path) || sha256(readFileSync(path)) !== sha256(binary)) writeWhole(path, binary, executable);
 	return path;
 };
 //#endregion
@@ -507,6 +573,37 @@ const compose = (name, declaration, parts = {}) => {
 	return svc;
 };
 //#endregion
+//#region src/eds.ts
+/**
+* This function gives the name of the outbound cluster of a Service port in a sidecar of Istio:
+* `outbound|<port>||<service>.<namespace>.svc.cluster.local`.
+*/
+const outboundCluster = (service, namespace, port) => `outbound|${port}||${service}.${namespace}.svc.cluster.local`;
+/** This function gives the filter of `/clusters` that selects one cluster: a regular expression of RE2 with anchors. */
+const clusterFilter = (cluster) => `^${cluster.replace(/[.|\\^$*+?()[\]{}]/g, (c) => `\\${c}`)}$`;
+/**
+* This function gives the hosts of a cluster from the text of `/clusters`. Each host of the text has
+* a line `<cluster>::<ip:port>::health_flags::<flags>`. The flags are `healthy`, or the names of the
+* flags that the host has (`clusters_handler.cc` and `host_utility.cc` of Envoy v1.39.4).
+*/
+const hostsOf = (text, cluster) => {
+	const healthy = [];
+	const other = {};
+	const prefix = `${cluster}::`;
+	for (const line of text.split("\n")) {
+		if (!line.startsWith(prefix)) continue;
+		const parts = line.slice(prefix.length).split("::");
+		if (parts.length !== 3 || parts[1] !== "health_flags") continue;
+		const [host, , flags] = parts;
+		if (flags.trim() === "healthy") healthy.push(host);
+		else other[host] = flags.trim();
+	}
+	return {
+		b: healthy.sort(),
+		u: other
+	};
+};
+//#endregion
 //#region src/faults.ts
 const needs = (time, what) => {
 	if (time === void 0) throw new Error(`the trigger needs ${what}`);
@@ -818,7 +915,8 @@ const FILES = [
 	["compose.ts", "src/compose.ts"],
 	["backends.ts", "src/backends.ts"],
 	["conntrack.ts", "src/conntrack.ts"],
-	["drops.ts", "src/drops.ts"]
+	["drops.ts", "src/drops.ts"],
+	["eds.ts", "src/eds.ts"]
 ];
 const shell = process.platform === "win32";
 /** This function gives each file of a staged context, with its path in the context, in a stable sequence. */
@@ -912,6 +1010,132 @@ const labImage = () => {
 	return built;
 };
 //#endregion
+//#region src/istio.ts
+/**
+* This module installs Istio 1.31.3 in a kind cluster with istioctl, from images pinned by digest.
+* The release notes of 1.31.2 and 1.31.3 change no default that a cell of T15 reads. Each value
+* of the install that is not a default of Istio is in `istioOperator`, with its reason.
+*/
+/** This version is the release of Istio of the lab: the tag of istioctl and of each image. */
+const ISTIO_VERSION = "1.31.3";
+/**
+* These are the images of the flavor `sidecar`, pinned by the digest of their index on Docker Hub
+* (read on 2026-10-24). `gcr.io/istio-release` has no final tag of 1.31. The lab pulls each one by
+* its digest, tags it with that digest (`localPin`), and loads it into each node. Thus no node pulls
+* from a registry.
+*/
+const ISTIO_IMAGES = {
+	pilot: "istio/pilot:1.31.3@sha256:6ad1e816c4d4767ba3f30a6286a96d121851ddb23d33e20736758acb5f3f4ea9",
+	proxy: "istio/proxyv2:1.31.3@sha256:eb74eacea98d3fb1e239ea84016085155478586a554158676b9b67d960151084"
+};
+/**
+* This format is the JSON access log of each sidecar. Each field is a command of the formatter of
+* Envoy v1.39.4. The fields give the status, the response flags and their details, and the gRPC
+* status as a number. They also give the tries of the router, the upstream host and cluster, and
+* the reasons of a failed connection. The watcher reads `code`, `flags`, `grpc` and `attempts`
+* (`image/watch.mjs`).
+*/
+const ACCESS_LOG_FORMAT = {
+	start: "%START_TIME%",
+	code: "%RESPONSE_CODE%",
+	flags: "%RESPONSE_FLAGS%",
+	details: "%RESPONSE_CODE_DETAILS%",
+	grpc: "%GRPC_STATUS_NUMBER%",
+	attempts: "%UPSTREAM_REQUEST_ATTEMPT_COUNT%",
+	host: "%UPSTREAM_HOST%",
+	cluster: "%UPSTREAM_CLUSTER%",
+	local: "%DOWNSTREAM_LOCAL_ADDRESS%",
+	remote: "%DOWNSTREAM_REMOTE_ADDRESS%",
+	duration: "%DURATION%",
+	failure: "%UPSTREAM_TRANSPORT_FAILURE_REASON%",
+	termination: "%CONNECTION_TERMINATION_DETAILS%"
+};
+/**
+* This function gives the IstioOperator of a flavor. The values that are not the defaults of Istio:
+* - The profile `minimal` is istiod only. The profile `default` also has an ingress gateway, which no cell uses.
+* - Each image is a full reference, thus the charts use it as it is (`contains "/"` in the
+*   templates of istiod and of the injector). The init container of the injector has the image of the proxy too (`proxy_init`).
+* - `imagePullPolicy: Never`: each image is on each node. An image that the lab did not load fails the install at once.
+* - istiod requests 256 MiB, not 2 GiB, and has one replica with no autoscaler. Each node of kind
+*   reports the whole runner, and no cell scales istiod.
+* - The access log of each sidecar goes to its output as JSON (`ACCESS_LOG_FORMAT`).
+*
+* The flavor `ambient` comes in a later step, thus it is refused.
+*/
+const istioOperator = (flavor, images) => {
+	if (flavor !== "sidecar") throw new Error(`the lab installs the mesh flavor sidecar only, not ${flavor}`);
+	return {
+		apiVersion: "install.istio.io/v1alpha1",
+		kind: "IstioOperator",
+		metadata: {
+			name: "lab",
+			namespace: "istio-system"
+		},
+		spec: {
+			profile: "minimal",
+			hub: "docker.io/istio",
+			tag: ISTIO_VERSION,
+			meshConfig: {
+				accessLogFile: "/dev/stdout",
+				accessLogEncoding: "JSON",
+				accessLogFormat: JSON.stringify(ACCESS_LOG_FORMAT)
+			},
+			values: {
+				global: {
+					imagePullPolicy: "Never",
+					proxy: { image: images.proxy },
+					proxy_init: { image: images.proxy }
+				},
+				pilot: {
+					image: images.pilot,
+					autoscaleEnabled: false,
+					replicaCount: 1,
+					resources: { requests: {
+						cpu: "500m",
+						memory: "256Mi"
+					} }
+				}
+			}
+		}
+	};
+};
+/**
+* This function installs the mesh of a flavor in a cluster with `istioctl install` and the operator
+* of the flavor. Then it waits for istiod. The kubeconfig of the cluster goes to istioctl.
+*/
+const installMesh = async (cluster, flavor, images) => {
+	const istioctl = await ensureTool(ISTIOCTL);
+	const file = join(cluster.dir, `istio-${flavor}.json`);
+	writeFileSync(file, `${JSON.stringify(istioOperator(flavor, images), null, 2)}\n`);
+	await command(istioctl, [
+		"install",
+		"--kubeconfig",
+		cluster.kubeconfig,
+		"-y",
+		"-f",
+		file,
+		"--readiness-timeout",
+		"10m0s"
+	], { timeout: 9e5 });
+	await cluster.kubectl([
+		"-n",
+		"istio-system",
+		"rollout",
+		"status",
+		"deployment/istiod",
+		"--timeout=300s"
+	]);
+};
+/** This function gives the key of the mesh of a cell for its cluster: its flavor, or `none`. */
+const meshKey = (cell) => cell.mesh?.flavor ?? "none";
+/** This function gives the outbound cluster of the server of a cell in each client sidecar: the port 80 of its Service. */
+const serverCluster = (namespace) => outboundCluster("lab", namespace, 80);
+/** This time is the interval of the EDS poll of each client sidecar, the same as the poll of the rules. */
+const EDS_EVERY = {
+	value: 50,
+	unit: "ms"
+};
+//#endregion
 //#region src/conntrack.ts
 /** This function gives the count of entries that `conntrack -D` deleted, from its last line: "N flow entries have been deleted". */
 const deletedOf = (text) => {
@@ -1093,6 +1317,29 @@ const openKindLab = async (cluster, image) => {
 				`printf '%s' '${file}' > /tmp/rules.json`
 			]);
 		},
+		async eds(spec) {
+			for (const [node, { agent }] of sidecars) {
+				const pods = spec === null ? [] : spec.pods.filter((pod) => pod.node === node).map(({ pod, pid }) => ({
+					pod,
+					pid
+				}));
+				const file = spec === null || pods.length === 0 ? null : JSON.stringify({
+					cluster: spec.cluster,
+					every: valueIn(spec.every, "ms"),
+					pods
+				});
+				if (file === null) await output(agent, [
+					"rm",
+					"-f",
+					"/tmp/eds.json"
+				]);
+				else await output(agent, [
+					"sh",
+					"-c",
+					`printf '%s' '${file}' > /tmp/eds.json.partial && mv /tmp/eds.json.partial /tmp/eds.json`
+				]);
+			}
+		},
 		async forgetHandshakes(ip, port) {
 			const out = {};
 			for (const [node, { agent }] of sidecars) {
@@ -1251,6 +1498,18 @@ const whole = (amount, what) => {
 /** This function gives the namespace of a cell: `lab-` and 10 digits of the sha256 of its name. */
 const namespaceOf = (cell) => `lab-${createHash("sha256").update(cell.name).digest("hex").slice(0, 10)}`;
 /**
+* This function gives the Namespace object of a cell. In the sidecar mode of a mesh, the label
+* `istio-injection=enabled` makes the injector of Istio put a sidecar in each pod of the namespace.
+*/
+const namespaceObject = (cell, namespace) => ({
+	apiVersion: "v1",
+	kind: "Namespace",
+	metadata: {
+		name: namespace,
+		...cell.mesh?.mode === "sidecar" ? { labels: { "istio-injection": "enabled" } } : {}
+	}
+});
+/**
 * This function gives a probe of the pod, with each field of the cell. The probe of the service is an
 * HTTP GET of its health server (`path`). The probe of grpc-go (`path` null) is the gRPC probe of the
 * kubelet. It is a Check of the service "" on the port of the server, and the answer is SERVING until
@@ -1808,11 +2067,17 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const slug = (name) => name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
 /** This function tells if a cell has Redis: only a declaration of the service can have it. */
 const hasRedis = (cell) => cell.declaration !== null && cell.declaration.redis !== null;
-/** This tracker keeps the latest state of the cell from the lines: each pod, each slice, the backends of each node and each client. */
+/** This function gives the key of the cluster of a cell: its kube-proxy mode, its node count, and the flavor of its mesh. */
+const clusterKey = (cell) => cell.mesh === void 0 ? `${cell.mode}-${cell.workers}` : `${cell.mode}-${cell.workers}-${cell.mesh.flavor}`;
+/**
+* This tracker keeps the latest state of the cell from the lines: each pod, each slice and each
+* client. It also has, for each node, the backends of its rules and the hosts of its client sidecar.
+*/
 var Tracker = class {
 	pods = /* @__PURE__ */ new Map();
 	slices = /* @__PURE__ */ new Map();
 	backends = /* @__PURE__ */ new Map();
+	hosts = /* @__PURE__ */ new Map();
 	clients = /* @__PURE__ */ new Map();
 	/** This map is the backend of the last answer of each client, by client. */
 	answers = /* @__PURE__ */ new Map();
@@ -1827,6 +2092,7 @@ var Tracker = class {
 			if (line["type"] === "DELETED") this.slices.delete(String(line["name"]));
 			else this.slices.set(String(line["name"]), line);
 		} else if (line.ev === "rules") this.backends.set(String(line["node"]), line["b"]);
+		else if (line.ev === "eds") this.hosts.set(String(line["node"]), line["b"]);
 		else if (line.ev === "req") {
 			const request = line;
 			const client = this.clients.get(request.p) ?? {
@@ -1861,6 +2127,9 @@ const unsteady = (ctx) => {
 	if (ready.join() !== ips.join()) return `the EndpointSlice has ${ready.join(" ")}, not ${ips.join(" ")}`;
 	const expected = ips.map((ip) => `${ip}:8080`).sort().join();
 	for (const node of ctx.cluster.nodes) if ((tracker.backends.get(node) ?? []).join() !== expected) return `the rules of ${node} have ${(tracker.backends.get(node) ?? []).join(" ")}`;
+	if (cell.mesh?.mode === "sidecar") {
+		for (const node of ctx.cluster.nodes) if ((tracker.hosts.get(node) ?? []).join() !== expected) return `the client sidecar of ${node} has ${(tracker.hosts.get(node) ?? []).join(" ") || "no host"}`;
+	}
 	const nodes = new Set([...tracker.clients.values()].map((client) => client.node));
 	if (nodes.size !== ctx.cluster.nodes.length) return `clients answer on ${nodes.size} nodes`;
 	for (const [pod, client] of tracker.clients) {
@@ -1923,11 +2192,7 @@ const quiet = async (ctx, ours) => {
 * the backoff of grpc-js longer, up to 120 s.
 */
 const deploy = async (lab, cluster, cell, namespace, images) => {
-	await cluster.apply([{
-		apiVersion: "v1",
-		kind: "Namespace",
-		metadata: { name: namespace }
-	}]);
+	await cluster.apply([namespaceObject(cell, namespace)]);
 	await lab.watch([namespace]);
 	if (hasRedis(cell)) {
 		await cluster.apply(redisObjects(namespace, images.redis));
@@ -1979,8 +2244,40 @@ const deploy = async (lab, cluster, cell, namespace, images) => {
 		port: 80,
 		every: RULES_EVERY
 	});
+	if (cell.mesh?.mode === "sidecar") await pollEndpoints(lab, cluster, namespace);
 	return clusterIp;
 };
+/**
+* This function starts the EDS poll of the client sidecar on each node: the hosts of the server
+* cluster, each 50 ms. The agent enters the network namespace of the client pod through the PID of
+* its client process. The sidecar shares that namespace.
+*/
+const pollEndpoints = async (lab, cluster, namespace) => {
+	const listed = JSON.parse(await cluster.kubectl([
+		"-n",
+		namespace,
+		"get",
+		"pods",
+		"-l",
+		`app=${CLIENT}`,
+		"-o",
+		"json"
+	]));
+	const pods = [];
+	for (const item of listed.items.filter((each) => each.metadata.deletionTimestamp === void 0)) {
+		const node = item.spec.nodeName ?? "";
+		pods.push({
+			node,
+			pod: item.metadata.name,
+			pid: await lab.pidOf(node, namespace, item.metadata.name, CLIENT)
+		});
+	}
+	await lab.eds({
+		cluster: serverCluster(namespace),
+		every: EDS_EVERY,
+		pods
+	});
+};
 /** This function starts the prober again when the pods of the cell changed: it watches the namespace of each pod. */
 const probeThePods = async (ctx) => {
 	const pods = [...ctx.tracker.pods.values()].filter((pod) => !pod.del && pod.node !== null && pod.cs.every((status) => status.st === "running"));
@@ -2143,12 +2440,18 @@ const over = (ctx, run, lines) => {
 		"node-drain",
 		"scale-in",
 		"freeze-process"
-	].includes(kind) && ctx.cell.template === "T07") return kind === "scale-in" || kind === "node-drain" ? run.replicas.some((replica) => !ctx.tracker.pods.has(replica.pod)) : !ctx.tracker.pods.has(run.target.pod);
+	].includes(kind) && (ctx.cell.template === "T07" || ctx.cell.template === "T15")) return kind === "scale-in" || kind === "node-drain" ? run.replicas.some((replica) => !ctx.tracker.pods.has(replica.pod)) : !ctx.tracker.pods.has(run.target.pod);
 	if (kind === "rolling-update" || kind === "rollout") return run.replicas.every((replica) => !ctx.tracker.pods.has(replica.pod));
 	if (kind === "app-stall") return lines.some((line) => line.ev === "stall" && line["phase"] === "end");
 	return true;
 };
-/** This function does one run of a cell: at rest, the trigger, the heal, then the observation until the cell is at rest again. */
+/** This function gives the replica that the trigger stopped. For a scale-in or a drain, the controller chooses it. */
+const stoppedOf = (cell, replicas, target, lines) => deletes(cell) ? replicas.find((replica) => lines.some((line) => line.ev === "pod" && line["name"] === replica.pod && line["del"] === true)) ?? target : target;
+/**
+* This function does one run of a cell: at rest, the trigger, the heal, then the observation. The
+* observation lasts until the cell is at rest again, and until each predicted window can close
+* (`observationEnd`).
+*/
 const runOnce = async (ctx, env, index, warmup, ours) => {
 	const { lab, cell, cluster, namespace } = ctx;
 	const plan = planOf(cell);
@@ -2220,6 +2523,21 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
 		replicas,
 		restarts
 	}, lab.lines)) await sleep(250);
+	const until = observationEnd(cell, {
+		start,
+		acts: {
+			fault,
+			heal,
+			resume
+		},
+		target: stoppedOf(cell, replicas, target, lab.lines),
+		replicas,
+		lines: lab.lines
+	});
+	if (until !== null) {
+		const wait = Math.min(until - await lab.now(), limit - Date.now());
+		if (wait > 0) await sleep(wait);
+	}
 	const end = await lab.now();
 	const timedOut = Date.now() >= limit;
 	for (const step of plan.restore) try {
@@ -2230,7 +2548,7 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
 	ctx.expected = cell.pod.replicas;
 	for (const note of notes) ctx.log(`  ${note}`);
 	const lines = lab.take();
-	const stopped = deletes(cell) ? replicas.find((replica) => lines.some((line) => line.ev === "pod" && line["name"] === replica.pod && line["del"] === true)) ?? target : target;
+	const stopped = stoppedOf(cell, replicas, target, lines);
 	if (lines.some((line) => line.ev === "pod" && line["cs"].some((status) => status.rc > 0))) {
 		await cluster.kubectl([
 			"-n",
@@ -2327,17 +2645,22 @@ const runKindCells = async (cells, options) => {
 	const images = {
 		lab: image,
 		redis: redisImage,
-		grpc: grpcImage
+		grpc: grpcImage,
+		mesh: cells.some((cell) => cell.mesh !== void 0) ? {
+			pilot: await localPin(ISTIO_IMAGES.pilot),
+			proxy: await localPin(ISTIO_IMAGES.proxy)
+		} : void 0
 	};
 	const groups = /* @__PURE__ */ new Map();
-	for (const cell of cells) groups.set(`${cell.mode}-${cell.workers}`, [...groups.get(`${cell.mode}-${cell.workers}`) ?? [], cell]);
+	for (const cell of cells) groups.set(clusterKey(cell), [...groups.get(clusterKey(cell)) ?? [], cell]);
 	const results = [];
 	for (const [key, group] of groups) {
 		const partial = /* @__PURE__ */ new Map();
 		for (let env = 0; env < environments; env += 1) {
 			const s = scenario(`lab-${key}-${env}`, group[0].mode, group[0].workers);
 			const dir = join(options.work ?? join(tmpdir(), "systems-kind-work"), s.name);
-			options.log(`cluster ${s.name}: ${s.kubeProxy.mode}, ${s.workers} workers`);
+			const mesh = group[0].mesh;
+			options.log(`cluster ${s.name}: ${s.kubeProxy.mode}, ${s.workers} workers${mesh === void 0 ? "" : `, Istio ${mesh.flavor}`}`);
 			const cluster = await createCluster(s, dir);
 			let lab = null;
 			let loader = null;
@@ -2345,6 +2668,13 @@ const runKindCells = async (cells, options) => {
 				await cluster.load(image);
 				await cluster.load(redisImage);
 				if (grpcImage !== void 0 && group.some((cell) => stackOf(cell) === "grpc")) await cluster.load(grpcImage);
+				if (mesh !== void 0) {
+					if (images.mesh === void 0) throw new Error(`${group[0].name}: the run has no images of Istio`);
+					for (const each of [images.mesh.pilot, images.mesh.proxy]) await cluster.load(each);
+					const began = Date.now();
+					await installMesh(cluster, mesh.flavor, images.mesh);
+					options.log(`  Istio ${mesh.flavor} installed in ${Math.round((Date.now() - began) / 100) / 10} s`);
+				}
 				lab = await openKindLab(cluster, image);
 				const { GenericContainer } = await import("testcontainers");
 				const Shared = class extends GenericContainer {
@@ -2396,7 +2726,11 @@ const runKindCells = async (cells, options) => {
 							images: {
 								lab: image,
 								redis: redisImage,
-								...stackOf(cell) === "grpc" ? { grpc: grpcImage } : {}
+								...stackOf(cell) === "grpc" ? { grpc: grpcImage } : {},
+								...cell.mesh === void 0 ? {} : {
+									"istio-pilot": ISTIO_IMAGES.pilot,
+									"istio-proxy": ISTIO_IMAGES.proxy
+								}
 							}
 						});
 						entry.loads.push(...loads);
@@ -2411,6 +2745,7 @@ const runKindCells = async (cells, options) => {
 					} finally {
 						off();
 						await lab.rules(null).catch(() => void 0);
+						await lab.eds(null).catch(() => void 0);
 						await lab.stopProbe();
 						await cluster.kubectl([
 							"delete",
@@ -2449,14 +2784,17 @@ const runKindCells = async (cells, options) => {
 /** This function gives the estimate of a cell in seconds, for the arithmetic of the shards. */
 const estimate = (cell, environments = 2) => {
 	const hold = cell.trigger.hold === void 0 ? 0 : valueIn(cell.trigger.hold, "s");
-	const each = 25 + ({
+	const trigger = {
 		"liveness-restart": 100,
 		crash: 30,
 		"app-stall": hold + 40,
 		"cpu-throttle": hold + 40,
 		rollout: hold + 50
-	}[cell.trigger.kind] ?? (cell.template === "T18" ? hold + 35 : 15 + valueIn(cell.pod.preStop, "s")));
-	return 60 * environments + (cell.runs + environments) * each;
+	};
+	const after = observationAfter(cell);
+	const observed = after === null ? 0 : hold + valueIn(after, "s");
+	const each = 25 + Math.max(observed, trigger[cell.trigger.kind] ?? (cell.template === "T18" ? hold + 35 : 15 + valueIn(cell.pod.preStop, "s")));
+	return (cell.mesh === void 0 ? 60 : 180) * environments + (cell.runs + environments) * each;
 };
 /**
 * This function gives an estimate of the time of a cell: its setup, then each run and each
@@ -2469,11 +2807,13 @@ const estimatedTime = (cell, environments = 2) => ({
 });
 /**
 * This function gives the share of a shard of a catalog: cells in sequence. Thus the cells of one
-* mode stay together and share clusters. Each shard gets an equal share of the estimated time.
+* mode and one mesh flavor stay together and share clusters. Each shard gets an equal share of the
+* estimated time. In one mode, the cells with no mesh come first, in the sequence of the catalog.
 */
 const shardOf = (cells, index, count) => {
 	if (!(Number.isInteger(index) && Number.isInteger(count) && count >= 1 && index >= 0 && index < count)) throw new RangeError(`shard ${index} of ${count} is not a shard`);
-	const ordered = [...cells].sort((a, b) => a.mode === b.mode ? 0 : a.mode < b.mode ? -1 : 1);
+	const rank = (cell) => `${cell.mode}/${meshKey(cell) === "none" ? "" : meshKey(cell)}`;
+	const ordered = [...cells].sort((a, b) => rank(a) === rank(b) ? 0 : rank(a) < rank(b) ? -1 : 1);
 	const total = ordered.reduce((sum, cell) => sum + estimate(cell), 0);
 	const out = [];
 	let before = 0;
@@ -2485,6 +2825,6 @@ const shardOf = (cells, index, count) => {
 	return out;
 };
 //#endregion
-export { CLIENT, CONTROLLER, DOCUMENTED, DRAIN_RULES, FLAG_CHECK, FORMAT, FORMAT_V1, HEALTH_PORT, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LOG_LABEL, NODE_IMAGES, O04, O09, O10, O16, PORT, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, STAGES, WATCHDOG, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, checkRun, cidrOverlap, cidrRange, clientObjects, compose, createCluster, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, goDuration, intakeAfter, intakeOf, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, latchBand, lintCell, localPin, medianInterval, medianText, middle, namespaceOf, nodeNames, openKindLab, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, predictionText, probeOf, propagationOf, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverObjects, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, stackOf, stageLabImage, stallRemovalBand, stoppingOf, timeSpread, toolPath, transitionOf, unitOf, upgrade, upgradeCell, verdictsOf, writeResult };
+export { ACCESS_LOG_FORMAT, CLIENT, CONTROLLER, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HEALTH_PORT, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LOG_LABEL, NODE_IMAGES, O04, O09, O10, O16, PORT, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, STAGES, WATCHDOG, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, checkRun, cidrOverlap, cidrRange, clientObjects, clusterFilter, compose, createCluster, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, goDuration, hostsOf, installMesh, intakeAfter, intakeOf, istioOperator, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, latchBand, lintCell, localPin, medianInterval, medianText, meshKey, middle, namespaceObject, namespaceOf, nodeNames, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, stackOf, stageLabImage, stallRemovalBand, stoppingOf, tarEntry, timeSpread, toolPath, transitionOf, unitOf, upgrade, upgradeCell, verdictsOf, writeResult };
 
 //# sourceMappingURL=index.js.map