@mark1russell7/kind 0.13.1 → 0.13.2

package/build/index.js

@@ -1,4 +1,4 @@
-import { $ as signalsOf, A as HBONE_PORT, B as isTunnelState, C as observationEnd, Ct as drainBand, D as reusesReset, Dt as secondsOf, E as referenceOf, Et as seconds, F as deletionOf, G as podsOf, H as outcomesOf, I as dropOf, J as restartOf, K as recordsOf, L as emptyOf, M as backendsAt, N as byClients, O as unitOf, Ot as showBand, P as byNode, Q as signalOf, R as failuresOf, S as observationAfter, St as bandOf, T as propagationSource, Tt as requestBand, U as periodOf, V as kernelCounts, W as podsAt, X as rulesOf, Y as restartsAt, Z as sessionsAt, _ as checkRun, _t as LOG_LABEL, a as calibrationOf, at as socketsOf, b as factsOf, bt as SERVER, c as predictionText, ct as tunnelsOf, d as FORMAT_V1, dt as RELATIVE, et as sliceDropOf, f as judge, ft as medianInterval, g as LAST_SLACK, gt as HEALTH_PORT, h as KIND_NOISE, ht as CLIENT, i as readRecord, it as slicesOf, j as TCP_STATE, k as updatesOf, l as timeSpread, lt as ztunnelPodsOf, m as upgradeCell, mt as quantileOf, n as kindCellRecordSchema, nt as sliceReadyAt, o as cellsTable, ot as stoppingOf, p as upgrade, pt as middle, q as requestsOf, r as kindCellRecordV1Schema, rt as sliceReadyOf, s as medianText, st as transitionOf, t as STAGES, tt as sliceEmptyOf, u as FORMAT, ut as ztunnelReplaced, v as cyclesOf, vt as PORT, w as propagationOf, wt as overshootBand, x as judgeKind, xt as backlogEnd, y as deletionsOf, yt as REDIS, z as intakeOf } from "./judge-DA754h1o.js";
+import { $ as signalsOf, A as HBONE_PORT, At as secondsOf, B as isTunnelState, C as observationEnd, Ct as VIP_ANNOTATION, D as reusesReset, Dt as overshootBand, E as referenceOf, Et as drainBand, F as deletionOf, G as podsOf, H as outcomesOf, I as dropOf, J as restartOf, K as recordsOf, L as emptyOf, M as backendsAt, N as byClients, O as unitOf, Ot as requestBand, P as byNode, Q as signalOf, R as failuresOf, S as observationAfter, St as SERVER, T as propagationSource, Tt as bandOf, U as periodOf, V as kernelCounts, W as podsAt, X as rulesOf, Y as restartsAt, Z as sessionsAt, _ as checkRun, _t as LB_CLASS, a as calibrationOf, at as socketsOf, b as factsOf, bt as PORT, c as predictionText, ct as tunnelsOf, d as FORMAT_V1, dt as RELATIVE, et as sliceDropOf, f as judge, ft as medianInterval, g as LAST_SLACK, gt as HEALTH_PORT, h as KIND_NOISE, ht as CLIENT, i as readRecord, it as slicesOf, j as TCP_STATE, jt as showBand, k as updatesOf, kt as seconds, l as timeSpread, lt as ztunnelPodsOf, m as upgradeCell, mt as quantileOf, n as kindCellRecordSchema, nt as sliceReadyAt, o as cellsTable, ot as stoppingOf, p as upgrade, pt as middle, q as requestsOf, r as kindCellRecordV1Schema, rt as sliceReadyOf, s as medianText, st as transitionOf, t as STAGES, tt as sliceEmptyOf, u as FORMAT, ut as ztunnelReplaced, v as cyclesOf, vt as LOG_LABEL, w as propagationOf, wt as backlogEnd, x as judgeKind, xt as REDIS, y as deletionsOf, yt as OUTSIDE, z as intakeOf } from "./judge-C7Zr4KoS.js";
 import { valueIn } from "@mark1russell7/quantity";
 import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
@@ -2223,6 +2223,413 @@ const ztunnelCertificates = async (cluster) => {
 	return out;
 };
 //#endregion
+//#region src/kubevip.ts
+/**
+* This module installs kube-vip 1.3.1 in a kind cluster (SYS-51). The install gives a Service of type
+* LoadBalancer an address on the Docker network of the nodes: the VIP. In ARP mode, one node holds
+* the VIP on its link, and it sends a gratuitous ARP for it each `vip_arpRate`. A Lease elects that
+* node. Each value of the install is in `kubeVipObjects`, with its reason.
+*
+* The lab uses a VIP of a Service, not a VIP of the control plane. A VIP of the control plane needs
+* three control-plane nodes and TLS on port 6443. A VIP of a Service needs one DaemonSet on the
+* workers, and it serves the server of the cell.
+*
+* Each term of the election comes from the source of v1.3.1:
+* - With `svc_election` off, one election of the lease `plndr-svcs-lock` gives each VIP to one
+*   node (pkg/manager/worker/common.go, `GlobalLeader`). The lease is in the namespace of the pod.
+* - The election is `leaderelection.RunOrDie` of client-go v0.37.1, with a Lease lock and
+*   `ReleaseOnCancel` (pkg/election/election.go). The identity is the name of the node.
+* - A leader that loses its lease stops the process (`OnStoppedLeading`), and the kubelet starts it again.
+* - The ARP manager sends a gratuitous ARP for each VIP at each tick of one ticker of
+*   `vip_arpRate`. A new VIP gets its first one at the next tick, not at once (pkg/arp/arp.go).
+*/
+/** This version is the release of kube-vip of the lab, from 2026-10-19: the tag of its image. */
+const KUBE_VIP_VERSION = "1.3.1";
+/**
+* This image is kube-vip 1.3.1 for linux/amd64, pinned by the digest of its manifest. The digest of
+* the index of the tag is sha256:1962f9daf226503f7e6b107f595cb28b7917fd2d0113957075e19a3e5a6c1e33.
+* The manifest of one platform loads into kind with no index.
+*/
+const KUBE_VIP_IMAGE = "ghcr.io/kube-vip/kube-vip:v1.3.1@sha256:47353c7bde7fec21642cf621bc3fda2b999c051342912efda468999a399be667";
+/** This is the name of kube-vip: of its ServiceAccount, its DaemonSet and its container. */
+const KUBE_VIP = "kube-vip";
+/** This is the Lease of the election of the VIPs of the Services (`svc_leasename`, cmd/kube-vip.go of v1.3.1). */
+const KUBE_VIP_LEASE = {
+	namespace: "kube-system",
+	name: "plndr-svcs-lock"
+};
+/**
+* These are the times of the election, as the install sets them. Each one is the default of the
+* flags of v1.3.1 (cmd/kube-vip.go, lines 91–94). They are a lease of 15 s, a renew deadline of 10 s
+* and a retry period of 2 s. The install sets each one in the environment, thus a new default cannot
+* change a cell.
+*/
+const KUBE_VIP_ELECTION = {
+	lease: {
+		value: 15,
+		unit: "s"
+	},
+	renew: {
+		value: 10,
+		unit: "s"
+	},
+	retry: {
+		value: 2,
+		unit: "s"
+	}
+};
+/** This time is the interval of the gratuitous ARPs, the default of the flag `arpRate` of v1.3.1 (cmd/kube-vip.go, line 61). */
+const KUBE_VIP_ARP_RATE = {
+	value: 3e3,
+	unit: "ms"
+};
+/** This label of a node selects the nodes of kube-vip: the DaemonSet has a pod only on a node with it. */
+const VIP_LABEL = "lab.systems/vip";
+/** This label of a pod of kube-vip selects them. */
+const APP = { "app.kubernetes.io/name": KUBE_VIP };
+const whole$1 = (amount) => String(Math.round(valueIn(amount, "s")));
+/**
+* This function gives the objects of kube-vip in kube-system. They are its ServiceAccount, its
+* ClusterRole and binding, and its DaemonSet. The rules of the role are those of the manifest that
+* v1.3.1 generates (pkg/kubevip/config_generator.go, lines 95–134). The DaemonSet:
+* - is on the host network, with NET_ADMIN and NET_RAW. It adds the VIP to `eth0` and sends ARP.
+* - has a pod only on a node with `VIP_LABEL`. Thus the lab chooses the holder of the VIP.
+* - sets ARP mode, the Services, one election for all VIPs, and each time of the election.
+* - takes the image from the node (`imagePullPolicy: Never`).
+*/
+const kubeVipObjects = (image) => {
+	const env = {
+		vip_arp: "true",
+		vip_interface: "eth0",
+		port: "6443",
+		cp_enable: "false",
+		svc_enable: "true",
+		svc_election: "false",
+		svc_leasename: KUBE_VIP_LEASE.name,
+		vip_leaderelection: "true",
+		vip_leaseduration: whole$1(KUBE_VIP_ELECTION.lease),
+		vip_renewdeadline: whole$1(KUBE_VIP_ELECTION.renew),
+		vip_retryperiod: whole$1(KUBE_VIP_ELECTION.retry),
+		vip_arpRate: String(Math.round(valueIn(KUBE_VIP_ARP_RATE, "ms")))
+	};
+	const namespace = KUBE_VIP_LEASE.namespace;
+	return [
+		{
+			apiVersion: "v1",
+			kind: "ServiceAccount",
+			metadata: {
+				name: KUBE_VIP,
+				namespace
+			}
+		},
+		{
+			apiVersion: "rbac.authorization.k8s.io/v1",
+			kind: "ClusterRole",
+			metadata: { name: "system:kube-vip-role" },
+			rules: [
+				{
+					apiGroups: [""],
+					resources: ["services/status"],
+					verbs: ["update"]
+				},
+				{
+					apiGroups: [""],
+					resources: ["services", "endpoints"],
+					verbs: [
+						"list",
+						"get",
+						"watch",
+						"update"
+					]
+				},
+				{
+					apiGroups: [""],
+					resources: ["nodes"],
+					verbs: [
+						"list",
+						"get",
+						"watch",
+						"update",
+						"patch"
+					]
+				},
+				{
+					apiGroups: ["coordination.k8s.io"],
+					resources: ["leases"],
+					verbs: [
+						"list",
+						"get",
+						"watch",
+						"update",
+						"create"
+					]
+				},
+				{
+					apiGroups: ["discovery.k8s.io"],
+					resources: ["endpointslices"],
+					verbs: [
+						"list",
+						"get",
+						"watch",
+						"update"
+					]
+				},
+				{
+					apiGroups: [""],
+					resources: ["pods"],
+					verbs: ["list"]
+				},
+				{
+					apiGroups: ["networking.k8s.io"],
+					resources: ["servicecidrs"],
+					verbs: [
+						"list",
+						"get",
+						"watch"
+					]
+				}
+			]
+		},
+		{
+			apiVersion: "rbac.authorization.k8s.io/v1",
+			kind: "ClusterRoleBinding",
+			metadata: { name: "system:kube-vip-binding" },
+			roleRef: {
+				apiGroup: "rbac.authorization.k8s.io",
+				kind: "ClusterRole",
+				name: "system:kube-vip-role"
+			},
+			subjects: [{
+				kind: "ServiceAccount",
+				name: KUBE_VIP,
+				namespace
+			}]
+		},
+		{
+			apiVersion: "apps/v1",
+			kind: "DaemonSet",
+			metadata: {
+				name: KUBE_VIP,
+				namespace,
+				labels: APP
+			},
+			spec: {
+				selector: { matchLabels: APP },
+				template: {
+					metadata: { labels: APP },
+					spec: {
+						serviceAccountName: KUBE_VIP,
+						hostNetwork: true,
+						nodeSelector: { [VIP_LABEL]: "1" },
+						containers: [{
+							name: KUBE_VIP,
+							image,
+							imagePullPolicy: "Never",
+							args: ["manager"],
+							env: [...Object.entries(env).map(([name, value]) => ({
+								name,
+								value
+							})), {
+								name: "vip_nodename",
+								valueFrom: { fieldRef: { fieldPath: "spec.nodeName" } }
+							}],
+							securityContext: { capabilities: { add: ["NET_ADMIN", "NET_RAW"] } }
+						}]
+					}
+				}
+			}
+		}
+	];
+};
+/** This function gives the Lease of the election, or null when it is not there yet. */
+const vipLeaseOf = async (cluster) => {
+	const text = await cluster.kubectl([
+		"-n",
+		KUBE_VIP_LEASE.namespace,
+		"get",
+		"lease",
+		KUBE_VIP_LEASE.name,
+		"-o",
+		"json",
+		"--ignore-not-found"
+	]);
+	if (text.trim() === "") return null;
+	const spec = JSON.parse(text).spec ?? {};
+	const text0 = (key) => typeof spec[key] === "string" && spec[key] !== "" ? spec[key] : null;
+	const number0 = (key) => typeof spec[key] === "number" ? spec[key] : null;
+	return {
+		holder: text0("holderIdentity"),
+		acquireTime: text0("acquireTime"),
+		renewTime: text0("renewTime"),
+		leaseDurationSeconds: number0("leaseDurationSeconds"),
+		leaseTransitions: number0("leaseTransitions")
+	};
+};
+/** This function reads the Docker network of a cluster: its IPv4 subnet, and the MAC address of each container. */
+const vipNetworkOf = async (network) => {
+	const [info] = JSON.parse(await command("docker", [
+		"network",
+		"inspect",
+		network
+	]));
+	const subnet = (info?.IPAM?.Config ?? []).map((config) => config.Subnet ?? "").find((each) => /^\d+\.\d+\.\d+\.\d+\/\d+$/.test(each));
+	if (subnet === void 0) throw new Error(`the Docker network ${network} has no IPv4 subnet`);
+	const macs = {};
+	for (const container of Object.values(info?.Containers ?? {})) if (container.Name !== void 0 && container.MacAddress !== void 0) macs[container.Name] = container.MacAddress.toLowerCase();
+	return {
+		subnet,
+		macs
+	};
+};
+/**
+* This function gives the VIP of a subnet: the sixth address from its end. Docker gives the address
+* of each container from the start of the subnet, thus no container takes it.
+*/
+const vipOf = (subnet) => {
+	const [first, last] = cidrRange(subnet);
+	const address = last - 5;
+	if (address <= first + 1) throw new Error(`the subnet ${subnet} is too small for a VIP`);
+	return [
+		24,
+		16,
+		8,
+		0
+	].map((shift) => Math.floor(address / 2 ** shift) % 256).join(".");
+};
+const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/** This function gives the pods of kube-vip that are not deleted. */
+const kubeVipPods = async (cluster) => {
+	return JSON.parse(await cluster.kubectl([
+		"-n",
+		KUBE_VIP_LEASE.namespace,
+		"get",
+		"pods",
+		"-l",
+		`app.kubernetes.io/name=${KUBE_VIP}`,
+		"-o",
+		"json"
+	])).items.filter((pod) => pod.metadata.deletionTimestamp === void 0).map((pod) => {
+		const statuses = pod.status?.containerStatuses ?? [];
+		return {
+			name: pod.metadata.name,
+			node: pod.spec.nodeName ?? "",
+			ready: statuses.length > 0 && statuses.every((status) => status.ready),
+			restarts: statuses.reduce((sum, status) => sum + (status.restartCount ?? 0), 0)
+		};
+	});
+};
+/** This function gives the nodes with a ready pod of kube-vip that is not deleted. */
+const kubeVipReady = async (cluster) => (await kubeVipPods(cluster)).filter((pod) => pod.ready && pod.node !== "").map((pod) => pod.node).sort();
+/** This function waits until a condition gives null, and gives null, or the last reason after `within` ms. */
+const until = async (within, check) => {
+	const end = Date.now() + within;
+	let why = await check();
+	while (why !== null && Date.now() < end) {
+		await sleep$1(500);
+		why = await check();
+	}
+	return why;
+};
+const label = (cluster, node, on) => cluster.kubectl([
+	"label",
+	"node",
+	node,
+	on ? `${VIP_LABEL}=1` : `${VIP_LABEL}-`,
+	...on ? ["--overwrite"] : []
+]);
+/**
+* This function makes `holder` the holder of the VIP, with a ready pod of kube-vip on each node of
+* `nodes`. It gives null, or why it could not in `within` ms. If another node holds the lease, the
+* function takes the label from each other node. The DaemonSet then deletes their pods. The leader
+* gives back its lease at its stop (`ReleaseOnCancel`). The function gives each label back after
+* `holder` takes the lease.
+*
+* A leader that lost its lease restarted its container. The kubelet gives a container a longer
+* back-off at each restart, thus the function first deletes each pod of kube-vip with a restart.
+* The DaemonSet then makes a new pod with no back-off.
+*/
+const holdVip = async (cluster, holder, nodes, within = 18e4) => {
+	const others = nodes.filter((node) => node !== holder);
+	for (const pod of await kubeVipPods(cluster)) if (pod.restarts > 0) await cluster.kubectl([
+		"-n",
+		KUBE_VIP_LEASE.namespace,
+		"delete",
+		"pod",
+		pod.name,
+		"--wait=true",
+		"--timeout=120s"
+	]);
+	await label(cluster, holder, true);
+	const own = await until(within, async () => (await kubeVipReady(cluster)).includes(holder) ? null : `kube-vip on ${holder} is not ready`);
+	if (own !== null) return own;
+	const held = async () => {
+		const lease = await vipLeaseOf(cluster);
+		return lease?.holder === holder ? null : `the lease ${KUBE_VIP_LEASE.name} is held by ${lease?.holder ?? "no node"}, not ${holder}`;
+	};
+	if (await held() !== null) {
+		for (const node of others) await label(cluster, node, false);
+		const moved = await until(within, held);
+		if (moved !== null) return moved;
+	}
+	for (const node of others) await label(cluster, node, true);
+	return until(within, async () => {
+		const ready = await kubeVipReady(cluster);
+		const missing = nodes.filter((node) => !ready.includes(node));
+		return missing.length === 0 ? null : `kube-vip is not ready on ${missing.join(", ")}`;
+	});
+};
+/**
+* This function installs kube-vip in a cluster, with its image on each node (`cluster.load`), and
+* makes `holder` the holder of the VIP. A failed install gives the pods of kube-vip and the tail of
+* their logs.
+*/
+const installKubeVip = async (cluster, image, holder, nodes) => {
+	try {
+		await label(cluster, holder, true);
+		await cluster.apply(kubeVipObjects(image));
+		const why = await holdVip(cluster, holder, nodes);
+		if (why !== null) throw new Error(why);
+	} catch (error) {
+		const state = async (args) => (await cluster.kubectl([
+			"-n",
+			KUBE_VIP_LEASE.namespace,
+			...args
+		]).catch((failure) => String(failure))).slice(-4e3);
+		const pods = await state([
+			"get",
+			"pods",
+			"-l",
+			`app.kubernetes.io/name=${KUBE_VIP}`,
+			"-o",
+			"wide"
+		]);
+		const logs = await state([
+			"logs",
+			"-l",
+			`app.kubernetes.io/name=${KUBE_VIP}`,
+			"--tail=40",
+			"--prefix"
+		]);
+		throw new Error(`the install of kube-vip failed: ${String(error)}\nthe pods of kube-vip:\n${pods}\nthe logs:\n${logs}`);
+	}
+};
+/**
+* This function waits until the Service of a VIP has the VIP in its status: kube-vip took it. It
+* gives null, or why not in `within` ms.
+*/
+const vipServed = (cluster, namespace, service, vip, within = 12e4) => until(within, async () => {
+	const ingress = (await cluster.kubectl([
+		"-n",
+		namespace,
+		"get",
+		"service",
+		service,
+		"-o",
+		"jsonpath={.status.loadBalancer.ingress[*].ip}"
+	])).trim();
+	return ingress.split(/\s+/).includes(vip) ? null : `the Service ${service} has the ingress ${ingress || "none"}, not ${vip}`;
+});
+//#endregion
 //#region src/lds.ts
 /**
 * This filter of RE2 selects the two counters of the listener manager of Envoy that an update of LDS
@@ -2335,6 +2742,11 @@ const openKindLab = async (cluster, image) => {
 	const opened = [];
 	let watcher = null;
 	let prober = null;
+	let outsiders = [];
+	const stopOutside = async () => {
+		for (const container of outsiders) await container.stop({ timeout: 0 }).catch(() => void 0);
+		outsiders = [];
+	};
 	try {
 		for (const node of cluster.nodes) {
 			const agent = await new Shared(image).withPidMode(`container:${node}`).withPrivilegedMode().withNetworkMode("none").withEnvironment({ NODE_NAME: node }).withCommand([
@@ -2401,7 +2813,8 @@ const openKindLab = async (cluster, image) => {
 			return [
 				...opened,
 				...watcher === null ? [] : [watcher],
-				...prober === null ? [] : [prober]
+				...prober === null ? [] : [prober],
+				...outsiders
 			].map((container) => container.getName().replace(/^\//, ""));
 		},
 		take() {
@@ -2448,6 +2861,21 @@ const openKindLab = async (cluster, image) => {
 			})).start();
 			await Promise.race([ready, new Promise((_, reject) => setTimeout(() => reject(/* @__PURE__ */ new Error("no begin line from the prober in 60 s")), 6e4))]);
 		},
+		async outside(spec) {
+			await stopOutside();
+			if (spec === null) return;
+			for (let k = 0; k < spec.count; k += 1) {
+				const client = await new GenericContainer(image).withNetworkMode(cluster.network).withEnvironment({
+					TARGET: spec.target,
+					EVERY_MS: String(valueIn(spec.every, "ms")),
+					TIMEOUT_MS: String(valueIn(spec.timeout, "ms")),
+					POD_NAME: `${OUTSIDE}-${k}`,
+					NODE_NAME: OUTSIDE,
+					ARP_WATCH: spec.arp
+				}).withCommand(["node", "client.mjs"]).withLogConsumer(consume((line) => push(line))).start();
+				outsiders.push(client);
+			}
+		},
 		async stopProbe() {
 			if (prober !== null) await prober.stop({ timeout: 0 }).catch(() => void 0);
 			prober = null;
@@ -2634,6 +3062,7 @@ const openKindLab = async (cluster, image) => {
 		},
 		async close() {
 			await lab.stopProbe();
+			await stopOutside();
 			if (watcher !== null) await watcher.stop({ timeout: 0 }).catch(() => void 0);
 			watcher = null;
 			for (const container of opened.reverse()) await container.stop({ timeout: 0 }).catch(() => void 0);
@@ -2853,14 +3282,53 @@ const preStopOf = (preStop, grace) => {
 		`sleep ${preStop}`
 	] } } } };
 };
+/** This affinity keeps a pod in the zone that a zone loss keeps. */
+const keptZone = { nodeAffinity: { requiredDuringSchedulingIgnoredDuringExecution: { nodeSelectorTerms: [{ matchExpressions: [{
+	key: ZONE_LABEL,
+	operator: "NotIn",
+	values: [ZONE_LOST]
+}] }] } } };
+/**
+* This function gives the Service of a VIP (`kubevip.ts`). It is of type LoadBalancer, with the
+* class of kube-vip and the address of the VIP in its annotation. The policy `Cluster` lets each node
+* send a request to a replica on another node. The holder writes the address to the status of the
+* Service. Then kube-proxy has the rules of the address on each node.
+*/
+const vipService = (namespace, vip, ports) => ({
+	apiVersion: "v1",
+	kind: "Service",
+	metadata: {
+		name: "lab",
+		namespace,
+		annotations: { [VIP_ANNOTATION]: vip }
+	},
+	spec: {
+		type: "LoadBalancer",
+		loadBalancerClass: LB_CLASS,
+		externalTrafficPolicy: "Cluster",
+		selector: { app: "lab" },
+		ports
+	}
+});
 /**
 * These are the objects of the server: a Service of type ClusterIP, and the Deployment with the
 * settings of the cell. The image is the lab image for the service, or the image of grpc-go.
+*
+* With a VIP (`KindCell.vip`), the Service is of type LoadBalancer, with the address `vip` and the
+* class of kube-vip (`vipService`). No replica is in the lost zone. Thus a zone loss takes away
+* only the holder of the VIP.
 */
-const serverObjects = (cell, namespace, image, version = 1) => {
+const serverObjects = (cell, namespace, image, version = 1, vip = null) => {
 	const preStop = whole(cell.pod.preStop, "the preStop sleep");
 	const grace = whole(cell.pod.grace, "the grace period");
-	return [{
+	if (cell.vip !== void 0 && vip === null) throw new Error(`${cell.name}: a cell with a VIP needs the address of the VIP`);
+	const ports = [{
+		name: stackOf(cell) === "grpc" ? "grpc" : "http",
+		port: 80,
+		targetPort: PORT
+	}];
+	const zoneLoss = cell.trigger.kind === "zone-loss";
+	return [vip === null ? {
 		apiVersion: "v1",
 		kind: "Service",
 		metadata: {
@@ -2869,13 +3337,9 @@ const serverObjects = (cell, namespace, image, version = 1) => {
 		},
 		spec: {
 			selector: { app: "lab" },
-			ports: [{
-				name: stackOf(cell) === "grpc" ? "grpc" : "http",
-				port: 80,
-				targetPort: PORT
-			}]
+			ports
 		}
-	}, {
+	} : vipService(namespace, vip, ports), {
 		apiVersion: "apps/v1",
 		kind: "Deployment",
 		metadata: {
@@ -2908,12 +3372,13 @@ const serverObjects = (cell, namespace, image, version = 1) => {
 						topologyKey: "kubernetes.io/hostname",
 						whenUnsatisfiable: "ScheduleAnyway",
 						labelSelector: { matchLabels: { app: "lab" } }
-					}, ...cell.trigger.kind === "zone-loss" ? [{
+					}, ...zoneLoss && cell.vip === void 0 ? [{
 						maxSkew: 1,
 						topologyKey: ZONE_LABEL,
 						whenUnsatisfiable: "DoNotSchedule",
 						labelSelector: { matchLabels: { app: "lab" } }
 					}] : []],
+					...zoneLoss && cell.vip !== void 0 ? { affinity: keptZone } : {},
 					containers: [{
 						name: "app",
 						image,
@@ -2955,11 +3420,7 @@ const clientObjects = (cell, namespace, image, target) => [{
 					operator: "Exists",
 					effect: "NoSchedule"
 				}],
-				...cell.trigger.kind === "zone-loss" ? { affinity: { nodeAffinity: { requiredDuringSchedulingIgnoredDuringExecution: { nodeSelectorTerms: [{ matchExpressions: [{
-					key: ZONE_LABEL,
-					operator: "NotIn",
-					values: [ZONE_LOST]
-				}] }] } } } } : {},
+				...cell.trigger.kind === "zone-loss" ? { affinity: keptZone } : {},
 				containers: [{
 					name: CLIENT,
 					image,
@@ -3288,7 +3749,7 @@ const hasRedis = (cell) => cell.declaration !== null && cell.declaration.redis !
 * This function gives the key of the cluster of a cell: its kube-proxy mode, its node count, and the
 * flavor of its mesh. A zone loss has clusters of its own, with a zone on each node.
 */
-const clusterKey = (cell) => cell.mesh !== void 0 ? `${cell.mode}-${cell.workers}-${cell.mesh.flavor}` : zoned(cell) ? `${cell.mode}-${cell.workers}-zones` : `${cell.mode}-${cell.workers}`;
+const clusterKey = (cell) => (cell.mesh !== void 0 ? `${cell.mode}-${cell.workers}-${cell.mesh.flavor}` : zoned(cell) ? `${cell.mode}-${cell.workers}-zones` : `${cell.mode}-${cell.workers}`) + (cell.vip === void 0 ? "" : "-vip");
 /** This function tells if a cell needs a cluster with zones: a zone loss. */
 const zoned = (cell) => cell.trigger.kind === "zone-loss";
 /**
@@ -3353,8 +3814,12 @@ const unsteady = (ctx) => {
 		for (const node of ctx.cluster.nodes) if ((tracker.hosts.get(node) ?? []).join() !== expected) return `the client sidecar of ${node} has ${(tracker.hosts.get(node) ?? []).join(" ") || "no host"}`;
 	}
 	const nodes = new Set([...tracker.clients.values()].map((client) => client.node));
-	const clientNodes = ctx.cluster.nodes.length - (cell.trigger.kind === "zone-loss" ? 1 : 0);
+	const clientNodes = ctx.cluster.nodes.length - (cell.trigger.kind === "zone-loss" ? 1 : 0) + (cell.vip === void 0 ? 0 : 1);
 	if (nodes.size !== clientNodes) return `clients answer on ${nodes.size} nodes, not ${clientNodes}`;
+	if (cell.vip !== void 0) {
+		const outside = [...tracker.clients.values()].filter((client) => client.node === OUTSIDE).length;
+		if (outside !== cell.vip.clients) return `${outside} clients outside answer, not ${cell.vip.clients}`;
+	}
 	for (const [pod, client] of tracker.clients) {
 		if (tracker.last - client.ok > 1e3) return `client ${pod} has no answer in the last second`;
 		if (tracker.last - client.failed < 2e3) return `client ${pod} failed in the last 2 s`;
@@ -3457,7 +3922,7 @@ const quiet = async (ctx, ours, within = QUIET_WITHIN) => {
 * clients. A client of gRPC starts after the server is ready: each failed connect of a channel makes
 * the backoff of grpc-js longer, up to 120 s.
 */
-const deploy = async (lab, cluster, cell, namespace, images) => {
+const deploy = async (lab, cluster, cell, namespace, images, vip) => {
 	await cluster.apply([namespaceObject(cell, namespace)]);
 	await lab.watch([namespace], { ztunnel: followsZtunnel(cell) });
 	if (hasRedis(cell)) {
@@ -3473,7 +3938,7 @@ const deploy = async (lab, cluster, cell, namespace, images) => {
 	}
 	const grpc = stackOf(cell) === "grpc";
 	if (grpc && images.grpc === void 0) throw new Error(`${cell.name}: the cluster has no image of grpc-go`);
-	await cluster.apply(serverObjects(cell, namespace, grpc ? images.grpc : lab.image));
+	await cluster.apply(serverObjects(cell, namespace, grpc ? images.grpc : lab.image, 1, cell.vip === void 0 ? null : vip));
 	const clusterIp = (await cluster.kubectl([
 		"-n",
 		namespace,
@@ -3502,6 +3967,18 @@ const deploy = async (lab, cluster, cell, namespace, images) => {
 		`daemonset/${CLIENT}`,
 		"--timeout=300s"
 	]);
+	if (cell.vip !== void 0) {
+		if (vip === null) throw new Error(`${cell.name}: the cluster has no VIP`);
+		const served = await vipServed(cluster, namespace, "lab", vip);
+		if (served !== null) throw new Error(`${cell.name}: ${served}`);
+		await lab.outside({
+			target: `${vip}:80`,
+			every: cell.workload.every,
+			timeout: cell.workload.timeout,
+			count: cell.vip.clients,
+			arp: vip
+		});
+	}
 	await lab.rules({
 		mode: cluster.scenario.dataplane,
 		ns: namespace,
@@ -3938,6 +4415,39 @@ const doStep = async (ctx, step, replicas, background) => {
 };
 /** This function gives the node of the lost zone: the last worker (`zoneOf`). */
 const lostNodeOf = (cluster) => cluster.nodes.at(-1);
+/** This function gives the workers of a cluster: each node but the control plane, the first (`nodeNames`). */
+const workersOf = (cluster) => cluster.nodes.slice(1);
+/**
+* This function adds a line `vip` to the lines. The line has the Lease of the VIP and the MAC address
+* of each node on the Docker network. It also has the wall clock of the runner with its CLOCK_MONOTONIC. The times of the
+* Lease are on the wall clock, and each stamp of the lab is on CLOCK_MONOTONIC. The containers of
+* the runner share one kernel, thus the pair converts one into the other.
+*/
+const recordVip = async (ctx) => {
+	const wall = Date.now();
+	const t = Number(process.hrtime.bigint()) / 1e6;
+	const lease = await vipLeaseOf(ctx.cluster);
+	const { macs } = await vipNetworkOf(ctx.cluster.network);
+	const log = await ctx.cluster.kubectl([
+		"-n",
+		KUBE_VIP_LEASE.namespace,
+		"logs",
+		"-l",
+		"app.kubernetes.io/name=kube-vip",
+		"--tail=30",
+		"--prefix",
+		"--timestamps"
+	]).catch((error) => String(error));
+	ctx.lab.record({
+		ev: "vip",
+		t,
+		wall,
+		vip: ctx.vip,
+		lease,
+		macs,
+		log: log.slice(-6e3)
+	});
+};
 /** This function puts the replicas of a node first, and keeps the sequence of the others. */
 const lostFirst = (replicas, node) => [...replicas.filter((replica) => replica.node === node), ...replicas.filter((replica) => replica.node !== node)];
 /** This function tells if a run is over: the trigger is done, and the cell is at rest again. */
@@ -3969,7 +4479,6 @@ const stoppedOf = (cell, replicas, target, lines) => deletes(cell) ? replicas.fi
 const runOnce = async (ctx, env, index, warmup, ours) => {
 	const { lab, cell, cluster, namespace } = ctx;
 	const plan = planOf(cell);
-	const atRest = await steady(ctx, 3e5);
 	const empty = (why, start) => ({
 		env,
 		index,
@@ -3988,6 +4497,7 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
 		lines: lab.take(),
 		dropped: {}
 	});
+	const atRest = await steady(ctx, 3e5);
 	if (atRest !== null) return empty(`the cell was not at rest: ${atRest}`, await lab.now());
 	await probeThePods(ctx);
 	await pollListeners(ctx);
@@ -3999,11 +4509,16 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
 	const zoneLoss = cell.trigger.kind === "zone-loss";
 	const lost = zoneLoss ? await nodeSteady(cluster, lostNodeOf(cluster), 3e5) : null;
 	if (lost !== null) return empty(`the node of the lost zone was not at rest: ${lost}`, await lab.now());
+	if (cell.vip !== void 0) {
+		const held = await holdVip(cluster, lostNodeOf(cluster), workersOf(cluster));
+		if (held !== null) return empty(`the VIP is not on ${lostNodeOf(cluster)}: ${held}`, await lab.now());
+		await sleep(valueIn(KUBE_VIP_ARP_RATE, "ms") + valueIn(cell.workload.timeout, "ms") + 1e3);
+	}
 	if (await steady(ctx, 3e5) !== null) return empty("the cell was not at rest after the wait for a quiet VM", await lab.now());
 	const replicas = zoneLoss ? lostFirst(replicasOf(ctx), lostNodeOf(cluster)) : replicasOf(ctx);
-	if (zoneLoss && replicas[0]?.node !== lostNodeOf(cluster)) return empty(`no replica is on the node of the lost zone, ${lostNodeOf(cluster)}`, await lab.now());
+	if (zoneLoss && cell.vip === void 0 && replicas[0]?.node !== lostNodeOf(cluster)) return empty(`no replica is on the node of the lost zone, ${lostNodeOf(cluster)}`, await lab.now());
 	const restarts = new Map(ctx.tracker.servers().map((pod) => [pod.name, pod.cs.reduce((sum, status) => sum + status.rc, 0)]));
-	const target = hasRedis(cell) && [
+	const target = cell.vip !== void 0 || hasRedis(cell) && [
 		"rst",
 		"blackhole",
 		"freeze-process",
@@ -4103,6 +4618,7 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
 	}
 	const end = await lab.now();
 	const timedOut = Date.now() >= limit;
+	if (cell.vip !== void 0) await recordVip(ctx).catch((error) => notes.push(`the record of the VIP failed: ${String(error)}`));
 	for (const step of plan.restore) try {
 		await doStep(ctx, step, replicas, background);
 	} catch (error) {
@@ -4220,6 +4736,7 @@ const runKindCells = async (cells, options) => {
 		operator: await localPin(CILIUM_IMAGES.operator),
 		envoy: await localPin(CILIUM_IMAGES.envoy)
 	} : void 0;
+	const kubeVip = cells.some((cell) => cell.vip !== void 0) ? await localPin(KUBE_VIP_IMAGE) : void 0;
 	const images = {
 		lab: image,
 		redis: redisImage,
@@ -4265,6 +4782,15 @@ const runKindCells = async (cells, options) => {
 					await installMesh(cluster, mesh.flavor, images.mesh);
 					options.log(`  Istio ${mesh.flavor} installed in ${Math.round((Date.now() - began) / 100) / 10} s`);
 				}
+				let vip = null;
+				if (group[0].vip !== void 0) {
+					if (kubeVip === void 0) throw new Error(`${group[0].name}: the run has no image of kube-vip`);
+					await cluster.load(kubeVip);
+					const began = Date.now();
+					await installKubeVip(cluster, kubeVip, lostNodeOf(cluster), workersOf(cluster));
+					vip = vipOf((await vipNetworkOf(cluster.network)).subnet);
+					options.log(`  kube-vip installed in ${Math.round((Date.now() - began) / 100) / 10} s, the VIP ${vip}`);
+				}
 				lab = await openKindLab(cluster, image);
 				const { GenericContainer } = await import("testcontainers");
 				const Shared = class extends GenericContainer {
@@ -4285,7 +4811,7 @@ const runKindCells = async (cells, options) => {
 					const ztunnelEnv = cell.mesh?.ztunnel;
 					try {
 						if (ztunnelEnv !== void 0) await setZtunnelEnv(cluster, ztunnelEnv, false);
-						const clusterIp = await deploy(lab, cluster, cell, namespace, images);
+						const clusterIp = await deploy(lab, cluster, cell, namespace, images, vip);
 						for (const name of [...lab.own(), loader.getName().replace(/^\//, "")]) ours.add(name);
 						const ctx = {
 							lab,
@@ -4294,6 +4820,7 @@ const runKindCells = async (cells, options) => {
 							namespace,
 							tracker,
 							clusterIp,
+							vip: cell.vip === void 0 ? null : vip,
 							loads,
 							loader,
 							idle: options.idle ?? .6,
@@ -4322,7 +4849,8 @@ const runKindCells = async (cells, options) => {
 								redis: redisImage,
 								...stackOf(cell) === "grpc" ? { grpc: grpcImage } : {},
 								...cell.mesh === void 0 ? {} : Object.fromEntries(MESH_IMAGES[cell.mesh.flavor].map((name) => [`istio-${name}`, ISTIO_IMAGES[name]])),
-								...s.cni === "cilium" ? Object.fromEntries(Object.entries(CILIUM_IMAGES).map(([name, ref]) => [`cilium-${name}`, ref])) : {}
+								...s.cni === "cilium" ? Object.fromEntries(Object.entries(CILIUM_IMAGES).map(([name, ref]) => [`cilium-${name}`, ref])) : {},
+								...cell.vip === void 0 ? {} : { "kube-vip": KUBE_VIP_IMAGE }
 							}
 						});
 						entry.loads.push(...loads);
@@ -4336,6 +4864,7 @@ const runKindCells = async (cells, options) => {
 						}));
 					} finally {
 						off();
+						await lab.outside(null).catch(() => void 0);
 						await lab.rules(null).catch(() => void 0);
 						await lab.eds(null).catch(() => void 0);
 						await lab.lds(null).catch(() => void 0);
@@ -4395,8 +4924,8 @@ const estimate = (cell, environments = 2) => {
 	const observed = after === null ? 0 : hold + valueIn(after, "s");
 	const closes = cell.prediction.kind === "ends" && cell.prediction.band !== null ? valueIn(cell.prediction.band.hi, "s") + 3 : 0;
 	const quiet = cell.trigger.quiet === true ? hold + 10 : 0;
-	const each = 25 + Math.max(observed, closes, quiet, trigger[cell.trigger.kind] ?? (cell.template === "T18" ? hold + 35 : 15 + valueIn(cell.pod.preStop, "s")));
-	return (cell.mesh === void 0 ? 60 : 180) * environments + (cell.runs + environments) * each;
+	const each = 25 + (cell.vip === void 0 ? 0 : 30) + Math.max(observed, closes, quiet, trigger[cell.trigger.kind] ?? (cell.template === "T18" ? hold + 35 : 15 + valueIn(cell.pod.preStop, "s")));
+	return ((cell.mesh === void 0 ? 60 : 180) + (cell.vip === void 0 ? 0 : 60)) * environments + (cell.runs + environments) * each;
 };
 /**
 * This function gives an estimate of the time of a cell: its setup, then each run and each
@@ -4427,6 +4956,6 @@ const shardOf = (cells, index, count) => {
 	return out;
 };
 //#endregion
-export { ACCESS_LOG_FORMAT, CHANNEL_USE, CILIUM_CHART, CILIUM_CONFIG_KEYS, CILIUM_IMAGES, CILIUM_VERSION, CILIUM_WORKLOADS, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DATAPLANES, DNS_PORT, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, HELM, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, MIDDLEBOX, NETEM_DRAIN, NETNS_DEFAULTS, NODE_IMAGES, NODE_LIFECYCLE, O04, O09, O10, O16, POD_LINK, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, SPREAD_LEAD, STAGES, SYSCTL, TCP_STATE, WATCHDOG, ZONE_KEPT, ZONE_LABEL, ZONE_LOST, ZTUNNEL_PODS, backendsAt, backendsOf, backlogEnd, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, certificateArgs, checkRun, cidrOverlap, cidrRange, ciliumBackendsOf, ciliumMode, ciliumValues, clientObjects, clusterFilter, compose, createCluster, cyclesOf, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dnsCommands, drainBand, dropOf, emptyOf, ensureChart, ensureTool, estimatedTime, factsOf, failuresOf, faultSpan, followsZtunnel, goDuration, hostsOf, installCilium, installMesh, intakeAfter, intakeOf, isKubeProxyMode, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, labSource, latchBand, leafCertificates, learnWindow, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, middleboxCommands, namesService, namespaceObject, namespaceOf, nodeLossBand, nodeNames, nodeReturnBand, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, overshootBand, pausedWindow, periodOf, placementOf, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, sessionsAt, setZtunnelEnv, sha256, shardOf, sharedGate, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stopOf, stoppingOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, writeResult, zoneOf, ztunnelCertificates, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
+export { ACCESS_LOG_FORMAT, CHANNEL_USE, CILIUM_CHART, CILIUM_CONFIG_KEYS, CILIUM_IMAGES, CILIUM_VERSION, CILIUM_WORKLOADS, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DATAPLANES, DNS_PORT, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, HELM, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, KUBE_VIP, KUBE_VIP_ARP_RATE, KUBE_VIP_ELECTION, KUBE_VIP_IMAGE, KUBE_VIP_LEASE, KUBE_VIP_VERSION, LAST_SLACK, LB_CLASS, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, MIDDLEBOX, NETEM_DRAIN, NETNS_DEFAULTS, NODE_IMAGES, NODE_LIFECYCLE, O04, O09, O10, O16, OUTSIDE, POD_LINK, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, SPREAD_LEAD, STAGES, SYSCTL, TCP_STATE, VIP_ANNOTATION, VIP_LABEL, WATCHDOG, ZONE_KEPT, ZONE_LABEL, ZONE_LOST, ZTUNNEL_PODS, backendsAt, backendsOf, backlogEnd, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, certificateArgs, checkRun, cidrOverlap, cidrRange, ciliumBackendsOf, ciliumMode, ciliumValues, clientObjects, clusterFilter, compose, createCluster, cyclesOf, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dnsCommands, drainBand, dropOf, emptyOf, ensureChart, ensureTool, estimatedTime, factsOf, failuresOf, faultSpan, followsZtunnel, goDuration, holdVip, hostsOf, installCilium, installKubeVip, installMesh, intakeAfter, intakeOf, isKubeProxyMode, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeVipObjects, kubeVipPods, kubeVipReady, kubeletBand, labImage, labSource, latchBand, leafCertificates, learnWindow, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, middleboxCommands, namesService, namespaceObject, namespaceOf, nodeLossBand, nodeNames, nodeReturnBand, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, overshootBand, pausedWindow, periodOf, placementOf, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, sessionsAt, setZtunnelEnv, sha256, shardOf, sharedGate, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stopOf, stoppingOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, vipLeaseOf, vipNetworkOf, vipOf, vipServed, vipService, writeResult, zoneOf, ztunnelCertificates, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
 
 //# sourceMappingURL=index.js.map