@mark1russell7/kind 0.11.12 → 0.12.1

package/build/index.js

@@ -1,8 +1,8 @@
-import { $ as rulesOf, A as RELATIVE, B as dropOf, C as observationEnd, D as reusesReset, E as referenceOf, F as TCP_STATE, G as kernelCounts, H as failuresOf, I as backendsAt, J as podsOf, K as outcomesOf, L as byClients, M as middle, N as quantileOf, O as unitOf, P as HBONE_PORT, Q as restartsAt, R as byNode, S as observationAfter, St as showBand, T as propagationSource, U as intakeOf, V as emptyOf, W as isTunnelState, X as requestsOf, Y as recordsOf, Z as restartOf, _ as checkRun, _t as REDIS, a as calibrationOf, at as sliceReadyOf, b as factsOf, bt as seconds, c as predictionText, ct as stoppingOf, d as FORMAT_V1, dt as ztunnelPodsOf, et as signalOf, f as judge, ft as ztunnelReplaced, g as LAST_SLACK, gt as PORT, h as KIND_NOISE, ht as LOG_LABEL, i as readRecord, it as sliceReadyAt, j as medianInterval, k as updatesOf, l as timeSpread, lt as transitionOf, m as upgradeCell, mt as HEALTH_PORT, n as kindCellRecordSchema, nt as sliceDropOf, o as cellsTable, ot as slicesOf, p as upgrade, pt as CLIENT, q as podsAt, r as kindCellRecordV1Schema, rt as sliceEmptyOf, s as medianText, st as socketsOf, t as STAGES, tt as signalsOf, u as FORMAT, ut as tunnelsOf, v as cyclesOf, vt as SERVER, w as propagationOf, x as judgeKind, xt as secondsOf, y as deletionsOf, yt as bandOf, z as deletionOf } from "./judge-n0rL_RR0.js";
+import { $ as rulesOf, A as RELATIVE, B as dropOf, C as observationEnd, Ct as showBand, D as reusesReset, E as referenceOf, F as TCP_STATE, G as kernelCounts, H as failuresOf, I as backendsAt, J as podsOf, K as outcomesOf, L as byClients, M as middle, N as quantileOf, O as unitOf, P as HBONE_PORT, Q as restartsAt, R as byNode, S as observationAfter, St as secondsOf, T as propagationSource, U as intakeOf, V as emptyOf, W as isTunnelState, X as requestsOf, Y as recordsOf, Z as restartOf, _ as checkRun, _t as PORT, a as calibrationOf, at as sliceReadyAt, b as factsOf, bt as bandOf, c as predictionText, ct as socketsOf, d as FORMAT_V1, dt as tunnelsOf, et as sessionsAt, f as judge, ft as ztunnelPodsOf, g as LAST_SLACK, gt as LOG_LABEL, h as KIND_NOISE, ht as HEALTH_PORT, i as readRecord, it as sliceEmptyOf, j as medianInterval, k as updatesOf, l as timeSpread, lt as stoppingOf, m as upgradeCell, mt as CLIENT, n as kindCellRecordSchema, nt as signalsOf, o as cellsTable, ot as sliceReadyOf, p as upgrade, pt as ztunnelReplaced, q as podsAt, r as kindCellRecordV1Schema, rt as sliceDropOf, s as medianText, st as slicesOf, t as STAGES, tt as signalOf, u as FORMAT, ut as transitionOf, v as cyclesOf, vt as REDIS, w as propagationOf, x as judgeKind, xt as seconds, y as deletionsOf, yt as SERVER, z as deletionOf } from "./judge-BdykXjF2.js";
 import { valueIn } from "@mark1russell7/quantity";
-import { execFile, execFileSync } from "node:child_process";
 import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
+import { execFile, execFileSync } from "node:child_process";
 import { promisify } from "node:util";
 import { createHash } from "node:crypto";
 import { fileURLToPath } from "node:url";
@@ -27,8 +27,10 @@ const hexIp = (hex) => [
 * - iptables: the `KUBE-SVC` chain of the Service jumps to a `KUBE-SEP` chain for each backend, and that chain does the DNAT.
 * - nftables: the `service-` chain of the Service does the DNAT to each backend, or it names an `endpoint-` chain for each one.
 * - IPVS: `/proc/net/ip_vs` has the line of the virtual service, then a line for each real server.
+* - Cilium: the JSON of the services of the agent (`ciliumBackendsOf`).
 */
 const backendsOf = (mode, text, rule) => {
+	if (mode === "cilium") return ciliumBackendsOf(text, rule);
 	const out = /* @__PURE__ */ new Set();
 	const lines = text.split("\n");
 	if (mode === "iptables") {
@@ -70,6 +72,45 @@ const backendsOf = (mode, text, rule) => {
 	}
 	return [...out].sort();
 };
+/** This function gives the services in a reply of the agent of Cilium. It refuses a reply that is not a JSON list. */
+const ciliumServices = (text) => {
+	const data = JSON.parse(text);
+	if (!Array.isArray(data)) throw new Error("the reply of the agent of Cilium is not a list of services");
+	return data;
+};
+/**
+* This function gives the backends of a Service in the reply of `GET /v1/service` of the agent of
+* Cilium 1.20, as `ip:port`, sorted. The model is `ServiceSpec` of `api/v1/openapi.yaml`. The
+* frontend is the ClusterIP and the port of the rule, with the scope `external` or no scope.
+*
+* The backends are those in the state `active`, or with no state. If no backend is active, the
+* backends are those in the state `terminating`. The BPF reconciler gives the slots of a service
+* in this sequence (`pkg/loadbalancer/reconciler/bpf_reconciler.go`). Thus a new connection goes
+* only to them. kube-proxy does the same with the endpoints that serve and stop. The other
+* states take no new connection: `terminating-not-serving`, `quarantined` and `maintenance`.
+*/
+const ciliumBackendsOf = (text, rule) => {
+	const backends = ciliumServices(text).filter((service) => {
+		const front = service.spec?.["frontend-address"];
+		const protocol = (front?.protocol ?? "tcp").toLowerCase();
+		return front?.ip === rule.ip && front.port === rule.port && (protocol === "tcp" || protocol === "any") && (front.scope ?? "external") === "external";
+	}).flatMap((service) => service.spec?.["backend-addresses"] ?? []);
+	const of = (states) => [...new Set(backends.filter((b) => states.includes(b.state) && b.ip !== void 0).map((b) => `${b.ip}:${b.port ?? 0}`))].sort();
+	const active = of(["active", void 0]);
+	return active.length > 0 ? active : of(["terminating"]);
+};
+/**
+* This function tells if a dump of the rules names the Service of a rule. The rules of kube-proxy
+* name it as `namespace/name`. The JSON of Cilium names it in the flags of its frontend.
+*/
+const namesService = (mode, text, rule) => {
+	if (mode !== "cilium") return text.includes(`${rule.ns}/${rule.svc}`);
+	try {
+		return ciliumServices(text).some((service) => service.spec?.flags?.namespace === rule.ns && service.spec.flags.name === rule.svc);
+	} catch {
+		return false;
+	}
+};
 //#endregion
 //#region src/scenario.ts
 const KUBE_PROXY_MODES = [
@@ -77,6 +118,9 @@ const KUBE_PROXY_MODES = [
 	"ipvs",
 	"nftables"
 ];
+const DATAPLANES = [...KUBE_PROXY_MODES, "cilium"];
+/** This function tells if a dataplane is a mode of kube-proxy. */
+const isKubeProxyMode = (mode) => mode !== "cilium";
 /**
 * These are the node images of kind v0.33.0, from its release notes. The image of Kubernetes 1.37
 * is v1.37.0. The research names v1.37.1 for some defaults, thus a default that changed in the
@@ -100,12 +144,16 @@ const ZONE_KEPT = "zone-a";
 const ZONE_LOST = "zone-b";
 /** This function gives the zone of the node at the position `at` of `nodeNames`, in a cluster of `count` nodes. */
 const zoneOf = (at, count) => at === count - 1 && at > 0 ? ZONE_LOST : ZONE_KEPT;
-/** This function gives a scenario with the defaults of Kubernetes for kube-proxy, as explicit values. */
+/**
+* This function gives a scenario with the defaults of Kubernetes for kube-proxy, as explicit values.
+* The dataplane `cilium` has no kube-proxy, and its CNI is Cilium.
+*/
 const scenario = (name, mode, workers, kubernetes = "1.37", zoned = false) => ({
 	name,
 	kubernetes,
 	workers,
-	kubeProxy: {
+	dataplane: mode,
+	kubeProxy: isKubeProxyMode(mode) ? {
 		mode,
 		minSyncPeriod: {
 			value: 1,
@@ -115,20 +163,27 @@ const scenario = (name, mode, workers, kubernetes = "1.37", zoned = false) => ({
 			value: 30,
 			unit: "s"
 		}
-	},
-	cni: "kindnet",
+	} : null,
+	cni: isKubeProxyMode(mode) ? "kindnet" : "cilium",
 	podSubnet: "10.244.0.0/16",
 	serviceSubnet: "10.96.0.0/16",
 	...zoned ? { zoned } : {}
 });
 const NAME = /^[a-z0-9]([a-z0-9-]{0,30}[a-z0-9])?$/;
-/** This function gives the problems of a scenario: a name that kind refuses, no image for the version, or a count out of range. */
+/**
+* This function gives the problems of a scenario. A problem is a name that kind refuses, no image for
+* the version, or a count out of range. A kube-proxy or a CNI that does not agree with the dataplane is a problem too.
+*/
 const scenarioProblems = (s) => {
 	const problems = [];
 	if (!NAME.test(s.name)) problems.push(`the name ${s.name} is not lowercase letters, digits and dashes of 1 to 32 characters`);
 	if (NODE_IMAGES[s.kubernetes] === void 0) problems.push(`no node image is pinned for Kubernetes ${s.kubernetes}`);
 	if (!Number.isInteger(s.workers) || s.workers < 0 || s.workers > 8) problems.push(`${s.workers} workers is not a whole number from 0 to 8`);
 	if (cidrOverlap(s.podSubnet, s.serviceSubnet)) problems.push(`the pod subnet ${s.podSubnet} overlaps the service subnet ${s.serviceSubnet}`);
+	if (isKubeProxyMode(s.dataplane)) {
+		if (s.kubeProxy?.mode !== s.dataplane) problems.push(`the dataplane ${s.dataplane} needs kube-proxy in the mode ${s.dataplane}`);
+		if (s.cni !== "kindnet") problems.push(`the dataplane ${s.dataplane} needs the CNI kindnet`);
+	} else if (s.kubeProxy !== null || s.cni !== "cilium") problems.push("the dataplane cilium needs no kube-proxy and the CNI cilium");
 	return problems;
 };
 /** This function gives the names of the node containers, as kind names them: the control plane, then each worker. */
@@ -139,17 +194,26 @@ const goDuration = (amount) => {
 	if (!Number.isInteger(ms) || ms < 0) throw new RangeError(`${ms} ms is not a whole number of milliseconds`);
 	return ms % 1e3 === 0 ? `${ms / 1e3}s` : `${ms}ms`;
 };
-/** This function gives the KubeProxyConfiguration patch of a scenario, as kubeadm reads it. */
-const kubeProxyPatch = (s) => [
-	"apiVersion: kubeproxy.config.k8s.io/v1alpha1",
-	"kind: KubeProxyConfiguration",
-	`mode: ${s.kubeProxy.mode}`,
-	`${s.kubeProxy.mode}:`,
-	`  minSyncPeriod: ${goDuration(s.kubeProxy.minSyncPeriod)}`,
-	`  syncPeriod: ${goDuration(s.kubeProxy.syncPeriod)}`,
-	""
-].join("\n");
-/** This function generates the configuration of kind from a scenario. It refuses a scenario with a problem. */
+/** This function gives the KubeProxyConfiguration patch of a scenario, as kubeadm reads it. It refuses a scenario with no kube-proxy. */
+const kubeProxyPatch = (s) => {
+	const proxy = s.kubeProxy;
+	if (proxy === null) throw new Error(`the scenario ${s.name} has no kube-proxy`);
+	return [
+		"apiVersion: kubeproxy.config.k8s.io/v1alpha1",
+		"kind: KubeProxyConfiguration",
+		`mode: ${proxy.mode}`,
+		`${proxy.mode}:`,
+		`  minSyncPeriod: ${goDuration(proxy.minSyncPeriod)}`,
+		`  syncPeriod: ${goDuration(proxy.syncPeriod)}`,
+		""
+	].join("\n");
+};
+/**
+* This function generates the configuration of kind from a scenario. It refuses a scenario with a
+* problem. The dataplane `cilium` gets no kindnet, as the kind guide of Cilium 1.20.4 tells
+* (`Documentation/installation/kind-configure.rst`). It also gets no kube-proxy, because Cilium
+* replaces it (`Documentation/network/kubernetes/kubeproxy-free.rst`).
+*/
 const kindConfig = (s) => {
 	const problems = scenarioProblems(s);
 	if (problems.length > 0) throw new Error(`the scenario ${s.name}: ${problems.join("; ")}`);
@@ -159,8 +223,8 @@ const kindConfig = (s) => {
 		apiVersion: "kind.x-k8s.io/v1alpha4",
 		networking: {
 			ipFamily: "ipv4",
-			kubeProxyMode: s.kubeProxy.mode,
-			disableDefaultCNI: false,
+			kubeProxyMode: s.kubeProxy?.mode ?? "none",
+			disableDefaultCNI: s.cni !== "kindnet",
 			podSubnet: s.podSubnet,
 			serviceSubnet: s.serviceSubnet
 		},
@@ -172,7 +236,7 @@ const kindConfig = (s) => {
 				...s.zoned === true ? { [ZONE_LABEL]: zoneOf(at, all.length) } : {}
 			}
 		})),
-		kubeadmConfigPatches: [kubeProxyPatch(s)]
+		kubeadmConfigPatches: s.kubeProxy === null ? [] : [kubeProxyPatch(s)]
 	};
 };
 /** This function gives the first and the last address of an IPv4 CIDR, as numbers. */
@@ -266,6 +330,30 @@ const ISTIOCTL = {
 	}
 };
 /**
+* This release of Helm is v3.22.1, from 2026-10-15. The lab installs Cilium from its chart with it.
+* The guide of Cilium 1.20.4 installs with Helm 3. The sums are from the `.sha256sum` files of
+* `get.helm.sh`. The tar file for Windows has `helm.exe`, not `helm`, thus Windows has no asset.
+*/
+const HELM = {
+	name: "helm",
+	version: "v3.22.1",
+	entry: "helm",
+	assets: {
+		"linux-x64": {
+			url: "https://get.helm.sh/helm-v3.22.1-linux-amd64.tar.gz",
+			sha256: "896cf63684a42a723d61c980369d6e588330927deb13676ca0d85006dd6da0ea"
+		},
+		"linux-arm64": {
+			url: "https://get.helm.sh/helm-v3.22.1-linux-arm64.tar.gz",
+			sha256: "4c29e286b279d170fa048ddfc3e4e83dcd069ca9c52102766ba672d7149e694d"
+		},
+		"darwin-arm64": {
+			url: "https://get.helm.sh/helm-v3.22.1-darwin-arm64.tar.gz",
+			sha256: "e73da8a0b0d5193774fdff17ce6319c7436bb22ee229b45fd64f3c40368c753a"
+		}
+	}
+};
+/**
 * This function gives the bytes of one file of a gzipped tar file. A header of the ustar format has
 * 512 bytes. It has the name at 0, the size in octal at 124, the type at 156 and the prefix of the
 * name at 345. The data follows the header, in blocks of 512 bytes. The match is on the last part of the
@@ -485,16 +573,17 @@ const createCluster = async (scenario, dir) => {
 				"-o",
 				"json"
 			]));
-			const proxy = JSON.parse(await kubectl([
+			const config = scenario.kubeProxy === null ? "cilium-config" : "kube-proxy";
+			const data = JSON.parse(await kubectl([
 				"-n",
 				"kube-system",
 				"get",
 				"configmap",
-				"kube-proxy",
+				config,
 				"-o",
 				"json"
-			]));
-			const mode = /^mode:\s*(\S*)/m.exec(proxy.data["config.conf"] ?? "")?.[1] ?? "";
+			])).data;
+			const mode = scenario.kubeProxy === null ? ciliumMode(data) : /^mode:\s*(\S*)/m.exec(data["config.conf"] ?? "")?.[1] ?? "";
 			const version = JSON.parse(await kubectl([
 				"version",
 				"-o",
@@ -535,6 +624,7 @@ const createCluster = async (scenario, dir) => {
 			]).catch(() => void 0);
 		}
 	};
+	const wait = scenario.cni === "kindnet" ? ["--wait", "180s"] : [];
 	await command(kind, [
 		"create",
 		"cluster",
@@ -544,8 +634,7 @@ const createCluster = async (scenario, dir) => {
 		file,
 		"--kubeconfig",
 		kubeconfig,
-		"--wait",
-		"180s"
+		...wait
 	], {
 		env,
 		timeout: 9e5
@@ -553,6 +642,178 @@ const createCluster = async (scenario, dir) => {
 	return cluster;
 };
 //#endregion
+//#region src/cilium.ts
+/**
+* This module installs Cilium 1.20.4 in a kind cluster with Helm, from images pinned by digest.
+* Cilium is the CNI and replaces kube-proxy (the dataplane `cilium`). Each value of the install
+* that is not a default of the chart is in `ciliumValues`, with its reason.
+*/
+/** This version is the release of Cilium of the lab, from 2026-10-23: the version of the chart and the tag of each image. */
+const CILIUM_VERSION = "1.20.4";
+/** This chart is the Helm chart of Cilium 1.20.4. The sha256 is of the file on `helm.cilium.io` (read on 2026-10-27). */
+const CILIUM_CHART = {
+	url: "https://helm.cilium.io/cilium-1.20.4.tgz",
+	sha256: "505abff8385811df1a949776fc42e9f10697989a332acc86d6e8933efcb855ed"
+};
+/**
+* These are the images that the install needs, pinned by the digests that the chart of 1.20.4 names
+* (`values.yaml`). The agent, the operator and Envoy are the only images: the install has no
+* node init, no preflight and no Hubble relay. The lab pulls each one by its digest, tags it
+* (`localPin`), and loads it into each node. Thus no node pulls from a registry.
+*/
+const CILIUM_IMAGES = {
+	agent: "quay.io/cilium/cilium:v1.20.4@sha256:20fe92e4c86641233a450098c158d2d29085f3419931ab6bf681d2485dedf16e",
+	operator: "quay.io/cilium/operator-generic:v1.20.4@sha256:7a7a3489dc66ad67e420d285c8a8d2e169b0957ad15e120f593d27081bf38d82",
+	envoy: "quay.io/cilium/cilium-envoy:v1.39.5-1792733054-8c73b2816bc8239e27f1fd53b271884688137459@sha256:547ab45f6092f681d9f54f3be9fa8fb1e705c9c5dc2df388efe3a52aa6965629"
+};
+/**
+* This function gives the Helm values of the install in a cluster. The values that are not the
+* defaults of the chart:
+* - `kubeProxyReplacement: true`, and the address of the API server on the control plane. With no
+*   kube-proxy, the agent cannot reach the API through its Service
+*   (`Documentation/network/kubernetes/kubeproxy-free.rst`).
+* - `ipam.mode: kubernetes`: the agent takes the pod CIDR of each node from the node, as the kind
+*   guide tells (`Documentation/installation/kind.rst`). Thus the pod subnet of the scenario holds.
+* - Each image is an `override`: the helper `cilium.image` of the chart then uses it as it is
+*   (`templates/_helpers.tpl`). `pullPolicy: Never`: each image is on each node.
+* - The operator has one replica, not two. Its anti-affinity puts each replica on a different node,
+*   and no cell examines a failover of the operator.
+*
+* Socket LB stays at its default. KPR sets it on (`pkg/kpr/kpr.go`). It selects a backend at
+* `connect()`. A connection keeps its backend, as with the conntrack NAT of kube-proxy. On the
+* deletion of a backend, the agent destroys only its UDP sockets, because
+* `LBSockTerminateAllProtos` is off by default (`pkg/loadbalancer/reconciler/termination.go`).
+*/
+const ciliumValues = (clusterName, images) => ({
+	kubeProxyReplacement: true,
+	k8sServiceHost: `${clusterName}-control-plane`,
+	k8sServicePort: 6443,
+	ipam: { mode: "kubernetes" },
+	image: {
+		override: images.agent,
+		pullPolicy: "Never"
+	},
+	operator: {
+		replicas: 1,
+		image: {
+			override: images.operator,
+			pullPolicy: "Never"
+		}
+	},
+	envoy: { image: {
+		override: images.envoy,
+		pullPolicy: "Never"
+	} }
+});
+/** These are the workloads in kube-system that the install waits for, then the DNS that waits for the CNI. */
+const CILIUM_WORKLOADS = [
+	"daemonset/cilium",
+	"deployment/cilium-operator",
+	"daemonset/cilium-envoy"
+];
+/** This function gives the path of the chart in the cache, after a download when the cache does not have it. It refuses a file whose sha256 is not the pinned value. */
+const ensureChart = async (cache = cacheFolder()) => {
+	const path = join(cache, "charts", `cilium-${CILIUM_VERSION}.tgz`);
+	const bytes = existsSync(path) ? readFileSync(path) : await download(CILIUM_CHART.url);
+	const found = sha256(bytes);
+	if (found !== CILIUM_CHART.sha256) throw new Error(`the chart of Cilium has the sha256 ${found}, not the pinned ${CILIUM_CHART.sha256}`);
+	if (!existsSync(path)) {
+		mkdirSync(join(cache, "charts"), { recursive: true });
+		writeFileSync(path, bytes);
+	}
+	return path;
+};
+/**
+* This function installs Cilium in a cluster with no CNI and no kube-proxy. Helm keeps its own
+* files in the folder of the cluster. The function waits for each workload, for each node to be
+* ready, and for CoreDNS. If the install fails, the error has the pods of kube-system and the last
+* lines of the log of the agent.
+*/
+const installCilium = async (cluster, images) => {
+	const helm = await ensureTool(HELM);
+	const chart = await ensureChart();
+	const file = join(cluster.dir, "cilium-values.json");
+	writeFileSync(file, `${JSON.stringify(ciliumValues(cluster.scenario.name, images), null, 2)}\n`);
+	const home = join(cluster.dir, "helm");
+	const env = {
+		HELM_CACHE_HOME: join(home, "cache"),
+		HELM_CONFIG_HOME: join(home, "config"),
+		HELM_DATA_HOME: join(home, "data")
+	};
+	try {
+		await command(helm, [
+			"install",
+			"cilium",
+			chart,
+			"-n",
+			"kube-system",
+			"--kubeconfig",
+			cluster.kubeconfig,
+			"-f",
+			file,
+			"--wait",
+			"--timeout",
+			"10m"
+		], {
+			env,
+			timeout: 9e5
+		});
+		for (const workload of CILIUM_WORKLOADS) await cluster.kubectl([
+			"-n",
+			"kube-system",
+			"rollout",
+			"status",
+			workload,
+			"--timeout=300s"
+		]);
+		await cluster.kubectl([
+			"wait",
+			"--for=condition=Ready",
+			"nodes",
+			"--all",
+			"--timeout=300s"
+		]);
+		await cluster.kubectl([
+			"-n",
+			"kube-system",
+			"rollout",
+			"status",
+			"deployment/coredns",
+			"--timeout=300s"
+		]);
+	} catch (error) {
+		const state = async (args) => (await cluster.kubectl([
+			"-n",
+			"kube-system",
+			...args
+		]).catch((why) => String(why))).slice(-4e3);
+		const pods = await state([
+			"get",
+			"pods",
+			"-o",
+			"wide"
+		]);
+		const agent = await state([
+			"logs",
+			"daemonset/cilium",
+			"-c",
+			"cilium-agent",
+			"--tail=80"
+		]);
+		throw new Error(`the install of Cilium failed: ${String(error)}\nthe pods of kube-system:\n${pods}\nthe log of the agent:\n${agent}`);
+	}
+};
+/** This is the setting of Cilium that a cluster manifest records: the version and the keys of `cilium-config` that the cells depend on. */
+const CILIUM_CONFIG_KEYS = [
+	"kube-proxy-replacement",
+	"bpf-lb-sock",
+	"bpf-lb-algorithm",
+	"routing-mode",
+	"ipam"
+];
+/** This function gives the mode of a cluster manifest of the dataplane `cilium`, from the data of the configmap `cilium-config`. */
+const ciliumMode = (data) => `cilium ${CILIUM_VERSION} ${CILIUM_CONFIG_KEYS.map((key) => `${key}=${data[key] ?? "default"}`).join(" ")}`;
+//#endregion
 //#region src/compose.ts
 /** This name is the controller of the service, and its one route reads Redis when the service has Redis. */
 const CONTROLLER = "work";
@@ -909,12 +1170,50 @@ const middleboxCommands = (peers, idle, port) => {
 */
 const linkOf = (cell) => cell.trigger.partition === true ? "server" : cell.template === "T15" ? "servers" : "redis";
 /**
+* This function gives the placement of the sessions of the clients: the backend of each slot of each
+* client, by client. The clients come in the sequence of their names. The sessions on the target go
+* one to each client in turn: the first slot of each client, then the second slot of each client. Thus
+* no client has two more sessions on the target than another. The other slots go to the other
+* replicas in turn.
+*/
+const placementOf = (clients, connections, onTarget, target, others) => {
+	if (!Number.isInteger(connections) || connections < 1) throw new RangeError(`a placement needs one session of each client at the least, not ${connections}`);
+	if (!Number.isInteger(onTarget) || onTarget < 0 || onTarget > clients.length * connections) throw new RangeError(`${onTarget} sessions on the target is not a count of the ${clients.length * connections} sessions of the clients`);
+	if (others.length === 0 && onTarget < clients.length * connections) throw new RangeError("a session off the target needs another replica");
+	const sorted = [...clients].sort((a, b) => a.localeCompare(b));
+	let other = 0;
+	return new Map(sorted.map((client, at) => [client, Array.from({ length: connections }, (_, slot) => slot * sorted.length + at < onTarget ? target : others[other++ % others.length])]));
+};
+/**
+* This function puts the step `place` before the first SIGHUP of the clients, for a workload with a
+* placement (`Workload.onTarget`). A plan with no such SIGHUP cannot place the sessions.
+*/
+const placed = (cell, plan) => {
+	const onTarget = cell.workload.onTarget;
+	if (onTarget === void 0) return plan;
+	if (cell.workload.connections === void 0) throw new Error(`${cell.name}: a placement needs the sessions of each client (connections)`);
+	const prepare = plan.prepare ?? [];
+	const at = prepare.findIndex((step) => step.do === "signal" && step.target === "clients" && step.signal === "SIGHUP");
+	if (at === -1) throw new Error(`${cell.name}: the trigger ${cell.trigger.kind} has no SIGHUP of the clients before its fault, thus it cannot place the sessions`);
+	return {
+		...plan,
+		prepare: [
+			...prepare.slice(0, at),
+			{
+				do: "place",
+				onTarget
+			},
+			...prepare.slice(at)
+		]
+	};
+};
+/**
 * This function gives the fault plan of a cell. A trigger with flaps (`Trigger.flaps`) gives them to
 * its plan: the runner then does the inject and the heal again for each flap. A plan with no hold or
 * no heal cannot flap.
 */
 const planOf = (cell) => {
-	const plan = stepsOf(cell);
+	const plan = placed(cell, stepsOf(cell));
 	const flaps = cell.trigger.flaps;
 	if (flaps === void 0 || flaps.length === 0) return plan;
 	if (plan.hold === null || plan.heal.length === 0) throw new Error(`the trigger ${cell.trigger.kind} has no hold and heal, thus it cannot flap`);
@@ -1494,7 +1793,8 @@ const FILES = [
 	["drops.ts", "src/drops.ts"],
 	["eds.ts", "src/eds.ts"],
 	["lds.ts", "src/lds.ts"],
-	["client-lines.ts", "src/client-lines.ts"]
+	["client-lines.ts", "src/client-lines.ts"],
+	["place.ts", "src/place.ts"]
 ];
 const shell = process.platform === "win32";
 /** This function tells if the package is in the workspace of its repository. From npm, it is not. */
@@ -3153,7 +3453,7 @@ const deploy = async (lab, cluster, cell, namespace, images) => {
 		"--timeout=300s"
 	]);
 	await lab.rules({
-		mode: cluster.scenario.kubeProxy.mode,
+		mode: cluster.scenario.dataplane,
 		ns: namespace,
 		svc: "lab",
 		ip: clusterIp,
@@ -3327,6 +3627,32 @@ const signalClients = async (ctx, signal) => {
 		t1: Math.max(...stamps.map((stamp) => stamp.t1))
 	};
 };
+/** This path in the root of a client container holds its placement: the backend of each of its slots, as JSON. */
+const PLACE_FILE = "/tmp/lab-place";
+/**
+* This function writes the placement of the sessions of each client pod (`placementOf`). The target
+* is the first replica, and the other slots go to the other replicas. The next SIGHUP of the clients
+* reads it. The nodes go at the same time, and the stamp is the first act and the last end.
+*/
+const placeClients = async (ctx, onTarget, replicas) => {
+	const connections = ctx.cell.workload.connections;
+	if (connections === void 0) throw new Error("a placement needs the sessions of each client (connections)");
+	const clients = [...ctx.tracker.pods.values()].filter((pod) => pod.labels["app"] === "client" && !pod.del && pod.node !== null);
+	if (clients.length === 0 || replicas.length === 0) throw new Error(`a placement needs clients and replicas: ${clients.length} clients, ${replicas.length} replicas`);
+	const placement = placementOf(clients.map((pod) => pod.name), connections, onTarget, replicas[0].pod, replicas.slice(1).map((replica) => replica.pod));
+	const stamps = await Promise.all(clients.map(async (pod) => {
+		const pid = await ctx.lab.pidOf(pod.node, ctx.namespace, pod.name, CLIENT);
+		return ctx.lab.act(pod.node, {
+			op: "write",
+			path: `/proc/${pid}/root${PLACE_FILE}`,
+			data: JSON.stringify(placement.get(pod.name))
+		});
+	}));
+	return {
+		t0: Math.min(...stamps.map((stamp) => stamp.t0)),
+		t1: Math.max(...stamps.map((stamp) => stamp.t1))
+	};
+};
 /**
 * This function gives the network namespace of each pod of a target, as a node and a PID in it. The
 * pods are each client pod (`clients`), or each replica (`servers`).
@@ -3388,6 +3714,7 @@ const doStep = async (ctx, step, replicas, background) => {
 				pod: redis
 			};
 		}
+		if (cluster.scenario.kubeProxy === null) throw new Error(`${ctx.cell.name}: the target kube-proxy needs kube-proxy, and the dataplane ${cluster.scenario.dataplane} has none`);
 		const worker = cluster.nodes[1] ?? cluster.nodes[0];
 		const proxy = JSON.parse(await cluster.kubectl([
 			"-n",
@@ -3535,6 +3862,7 @@ const doStep = async (ctx, step, replicas, background) => {
 			op: "zone",
 			lost: step.lost
 		});
+		case "place": return placeClients(ctx, step.onTarget, replicas);
 		case "netem": {
 			if (step.target !== "redis") throw new Error(`a netem takes the target redis, not ${step.target}`);
 			const at = await where(step.target);
@@ -3817,16 +4145,22 @@ const runKindCells = async (cells, options) => {
 	const flavors = new Set(cells.flatMap((cell) => cell.mesh === void 0 ? [] : [cell.mesh.flavor]));
 	const pinned = /* @__PURE__ */ new Map();
 	for (const name of new Set([...flavors].flatMap((flavor) => MESH_IMAGES[flavor]))) pinned.set(name, await localPin(ISTIO_IMAGES[name]));
+	const meshImages = flavors.size === 0 ? void 0 : {
+		pilot: pinned.get("pilot"),
+		proxy: pinned.get("proxy"),
+		cni: pinned.get("cni"),
+		ztunnel: pinned.get("ztunnel")
+	};
+	const cilium = cells.some((cell) => cell.mode === "cilium") ? {
+		agent: await localPin(CILIUM_IMAGES.agent),
+		operator: await localPin(CILIUM_IMAGES.operator),
+		envoy: await localPin(CILIUM_IMAGES.envoy)
+	} : void 0;
 	const images = {
 		lab: image,
 		redis: redisImage,
 		grpc: grpcImage,
-		mesh: flavors.size === 0 ? void 0 : {
-			pilot: pinned.get("pilot"),
-			proxy: pinned.get("proxy"),
-			cni: pinned.get("cni"),
-			ztunnel: pinned.get("ztunnel")
-		}
+		mesh: meshImages
 	};
 	const groups = /* @__PURE__ */ new Map();
 	for (const cell of cells) groups.set(clusterKey(cell), [...groups.get(clusterKey(cell)) ?? [], cell]);
@@ -3837,11 +4171,22 @@ const runKindCells = async (cells, options) => {
 			const s = scenario(`lab-${key}-${env}`, group[0].mode, group[0].workers, void 0, zoned(group[0]));
 			const dir = join(options.work ?? join(tmpdir(), "systems-kind-work"), s.name);
 			const mesh = group[0].mesh;
-			options.log(`cluster ${s.name}: ${s.kubeProxy.mode}, ${s.workers} workers${mesh === void 0 ? "" : `, Istio ${mesh.flavor}`}`);
+			options.log(`cluster ${s.name}: ${s.dataplane}, ${s.workers} workers${mesh === void 0 ? "" : `, Istio ${mesh.flavor}`}`);
 			const cluster = await createCluster(s, dir);
 			let lab = null;
 			let loader = null;
 			try {
+				if (s.cni === "cilium") {
+					if (cilium === void 0) throw new Error(`${group[0].name}: the run has no images of Cilium`);
+					for (const each of [
+						cilium.agent,
+						cilium.operator,
+						cilium.envoy
+					]) await cluster.load(each);
+					const began = Date.now();
+					await installCilium(cluster, cilium);
+					options.log(`  Cilium ${CILIUM_VERSION} installed in ${Math.round((Date.now() - began) / 100) / 10} s`);
+				}
 				await cluster.load(image);
 				await cluster.load(redisImage);
 				if (grpcImage !== void 0 && group.some((cell) => stackOf(cell) === "grpc")) await cluster.load(grpcImage);
@@ -3912,7 +4257,8 @@ const runKindCells = async (cells, options) => {
 								lab: image,
 								redis: redisImage,
 								...stackOf(cell) === "grpc" ? { grpc: grpcImage } : {},
-								...cell.mesh === void 0 ? {} : Object.fromEntries(MESH_IMAGES[cell.mesh.flavor].map((name) => [`istio-${name}`, ISTIO_IMAGES[name]]))
+								...cell.mesh === void 0 ? {} : Object.fromEntries(MESH_IMAGES[cell.mesh.flavor].map((name) => [`istio-${name}`, ISTIO_IMAGES[name]])),
+								...s.cni === "cilium" ? Object.fromEntries(Object.entries(CILIUM_IMAGES).map(([name, ref]) => [`cilium-${name}`, ref])) : {}
 							}
 						});
 						entry.loads.push(...loads);
@@ -4017,6 +4363,6 @@ const shardOf = (cells, index, count) => {
 	return out;
 };
 //#endregion
-export { ACCESS_LOG_FORMAT, CHANNEL_USE, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, MIDDLEBOX, NETEM_DRAIN, NETNS_DEFAULTS, NODE_IMAGES, NODE_LIFECYCLE, O04, O09, O10, O16, POD_LINK, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, SPREAD_LEAD, STAGES, SYSCTL, TCP_STATE, WATCHDOG, ZONE_KEPT, ZONE_LABEL, ZONE_LOST, ZTUNNEL_PODS, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, certificateArgs, checkRun, cidrOverlap, cidrRange, clientObjects, clusterFilter, compose, createCluster, cyclesOf, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, faultSpan, followsZtunnel, goDuration, hostsOf, installMesh, intakeAfter, intakeOf, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, labSource, latchBand, leafCertificates, learnWindow, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, middleboxCommands, namespaceObject, namespaceOf, nodeLossBand, nodeNames, nodeReturnBand, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, setZtunnelEnv, sha256, shardOf, sharedGate, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stopOf, stoppingOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, writeResult, zoneOf, ztunnelCertificates, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
+export { ACCESS_LOG_FORMAT, CHANNEL_USE, CILIUM_CHART, CILIUM_CONFIG_KEYS, CILIUM_IMAGES, CILIUM_VERSION, CILIUM_WORKLOADS, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DATAPLANES, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, HELM, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, MIDDLEBOX, NETEM_DRAIN, NETNS_DEFAULTS, NODE_IMAGES, NODE_LIFECYCLE, O04, O09, O10, O16, POD_LINK, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, SPREAD_LEAD, STAGES, SYSCTL, TCP_STATE, WATCHDOG, ZONE_KEPT, ZONE_LABEL, ZONE_LOST, ZTUNNEL_PODS, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, certificateArgs, checkRun, cidrOverlap, cidrRange, ciliumBackendsOf, ciliumMode, ciliumValues, clientObjects, clusterFilter, compose, createCluster, cyclesOf, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureChart, ensureTool, estimatedTime, factsOf, failuresOf, faultSpan, followsZtunnel, goDuration, hostsOf, installCilium, installMesh, intakeAfter, intakeOf, isKubeProxyMode, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, labSource, latchBand, leafCertificates, learnWindow, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, middleboxCommands, namesService, namespaceObject, namespaceOf, nodeLossBand, nodeNames, nodeReturnBand, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, pausedWindow, placementOf, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, sessionsAt, setZtunnelEnv, sha256, shardOf, sharedGate, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stopOf, stoppingOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, writeResult, zoneOf, ztunnelCertificates, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
 
 //# sourceMappingURL=index.js.map