@mark1russell7/kind 0.6.0 → 0.8.1

package/build/index.js

@@ -1,4 +1,4 @@
-import { $ as slicesOf, A as medianInterval, B as intakeOf, C as propagationOf, D as unitOf, E as reusesReset, F as byNode, G as recordsOf, H as outcomesOf, I as deletionOf, J as rulesOf, K as requestsOf, L as dropOf, M as quantileOf, N as backendsAt, O as updatesOf, P as byClients, Q as sliceEmptyOf, R as emptyOf, S as observationEnd, T as referenceOf, U as podsAt, V as kernelCounts, W as podsOf, X as signalsOf, Y as signalOf, Z as sliceDropOf, _ as checkRun, a as calibrationOf, at as LOG_LABEL, b as judgeKind, c as predictionText, ct as SERVER, d as FORMAT_V1, dt as secondsOf, et as socketsOf, f as judge, ft as showBand, g as LAST_SLACK, h as KIND_NOISE, i as readRecord, it as HEALTH_PORT, j as middle, k as RELATIVE, l as timeSpread, lt as bandOf, m as upgradeCell, n as kindCellRecordSchema, nt as transitionOf, o as cellsTable, ot as PORT, p as upgrade, q as restartsAt, r as kindCellRecordV1Schema, rt as CLIENT, s as medianText, st as REDIS, t as STAGES, tt as stoppingOf, u as FORMAT, ut as seconds, v as deletionsOf, w as propagationSource, x as observationAfter, y as factsOf, z as failuresOf } from "./judge-DNCI5vvt.js";
+import { $ as signalOf, A as medianInterval, B as emptyOf, C as propagationOf, D as unitOf, E as reusesReset, F as backendsAt, G as outcomesOf, H as intakeOf, I as byClients, J as recordsOf, K as podsAt, L as byNode, M as quantileOf, N as HBONE_PORT, O as updatesOf, P as TCP_STATE, Q as rulesOf, R as deletionOf, S as observationEnd, T as referenceOf, U as isTunnelState, V as failuresOf, W as kernelCounts, X as restartOf, Y as requestsOf, Z as restartsAt, _ as checkRun, _t as seconds, a as calibrationOf, at as stoppingOf, b as judgeKind, c as predictionText, ct as ztunnelPodsOf, d as FORMAT_V1, dt as HEALTH_PORT, et as signalsOf, f as judge, ft as LOG_LABEL, g as LAST_SLACK, gt as bandOf, h as KIND_NOISE, ht as SERVER, i as readRecord, it as socketsOf, j as middle, k as RELATIVE, l as timeSpread, lt as ztunnelReplaced, m as upgradeCell, mt as REDIS, n as kindCellRecordSchema, nt as sliceEmptyOf, o as cellsTable, ot as transitionOf, p as upgrade, pt as PORT, q as podsOf, r as kindCellRecordV1Schema, rt as slicesOf, s as medianText, st as tunnelsOf, t as STAGES, tt as sliceDropOf, u as FORMAT, ut as CLIENT, v as deletionsOf, vt as secondsOf, w as propagationSource, x as observationAfter, y as factsOf, yt as showBand, z as dropOf } from "./judge-Cgvkh4JX.js";
 import { valueIn } from "@mark1russell7/quantity";
 import { execFile, execFileSync } from "node:child_process";
 import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
@@ -637,6 +637,8 @@ const DELETION_COST = "controller.kubernetes.io/pod-deletion-cost";
 const POLICY = "lab-deny";
 /** This service account is the account of the clients: their pods name no other account. */
 const CLIENT_ACCOUNT = "default";
+/** This selector gives the ztunnel pods in istio-system: the label `app` of the chart of ztunnel. */
+const ZTUNNEL_PODS = "app=ztunnel";
 /**
 * This function gives the AuthorizationPolicy of the trigger `authorization-policy`. It denies each
 * request to the server pods from the principal of the service account `account`. Istio gives a
@@ -656,6 +658,186 @@ const policyObject = (account) => ({
 		rules: [{ from: [{ source: { principals: [`cluster.local/ns/{namespace}/sa/${account}`] } }] }]
 	}
 });
+/** These are the names of the sysctls that the steps of a plan set. Each one is a sysctl of a network namespace. */
+const SYSCTL = { retries: "net.ipv4.tcp_retries2" };
+/**
+* These are the values of those sysctls in a new network namespace of Linux v6.17. A restore puts
+* them back. `tcp_retries2` is TCP_RETR2 (include/net/tcp.h, line 105, and net/ipv4/tcp_ipv4.c,
+* line 3604).
+*/
+const NETNS_DEFAULTS = { retries: "15" };
+/**
+* These are the names of the middlebox in the filter table of a replica. One chain takes the
+* segments from the peers, and one chain takes the segments to them. The table of xt_recent holds
+* the time of the last segment of each peer.
+*/
+const MIDDLEBOX = {
+	input: "LAB-MBX-IN",
+	output: "LAB-MBX-OUT",
+	table: "LAB-MBX"
+};
+/** This time is between the stop of the calls and the fault of a quiet black hole. The last answers and their ACKs, at most 200 ms late, come in it. */
+const QUIET_LEAD = {
+	value: 1,
+	unit: "s"
+};
+/** This time is between the close of the old channel of each client and the fault of a gap with `close`. The stream of the old channel ends in it. */
+const CLOSE_LEAD = {
+	value: 2,
+	unit: "s"
+};
+/** This time is how long each client calls on its new channel at the fault of a gap with `close`, before it closes it. */
+const CHANNEL_USE = {
+	value: 5,
+	unit: "s"
+};
+const IPV4 = /^(25[0-5]|2[0-4]\d|1?\d?\d)(\.(25[0-5]|2[0-4]\d|1?\d?\d)){3}$/;
+/**
+* This function gives the command of the injector that sets one sysctl of its network namespace.
+* Only a sysctl of the namespace (`net.`) and a whole number reach it.
+*/
+const sysctlCommand = (key, value) => {
+	if (!/^net\.[a-z0-9_.]+$/.test(key)) throw new Error(`not a sysctl of a network namespace: ${key}`);
+	if (!/^\d+$/.test(value)) throw new Error(`not a whole number: ${value}`);
+	return [
+		"sysctl",
+		"-w",
+		`${key}=${value}`
+	];
+};
+/**
+* This function gives the commands of the injector that put a stateful middlebox in its network
+* namespace, with each peer, on the local port `port`. With `idle` null, they remove it.
+*
+* The middlebox forgets a flow of the port after `idle` with no segment. Then it drops each later
+* segment of it: the silent-drop form of the router of the harness. Conntrack stays as it is. The
+* local leg from ztunnel to the app has no PING, and istio-cni needs its conntrack entry. The
+* connmark of that entry sends each answer of the app back to ztunnel (cni/pkg/iptables/iptables.go
+* of Istio 1.31.3, lines 172–178, 278–284 and 336–347).
+*
+* The match recent keeps a table in each network namespace (xt_recent.c of Linux v6.17). Its key is
+* the address of the peer: the source of a segment from it, or the destination of a segment to it
+* (lines 251–257). The chain of each direction has these rules, in sequence:
+* - In the input chain only, a SYN sets the time of its peer and passes: a new flow.
+* - A segment of a peer with a time in the last `idle` seconds passes, and it sets the time again
+*   (lines 298–321).
+* - A segment of a peer with an older time is dropped, and the time stays old. Thus the flow stays forgotten.
+* - A segment of a peer that the table does not hold sets its time and passes (lines 283–290). That
+*   is the flow that was open before the set.
+*
+* Each rule that passes a segment goes back to INPUT or OUTPUT (RETURN), thus the other rules of the
+* namespace still act on it. The key is an address, not a flow. Thus a new SYN of a peer also refreshes an
+* old flow of the peer. A cell of the middlebox opens no new flow in its gap.
+*/
+const middleboxCommands = (peers, idle, port) => {
+	if (peers.length === 0) throw new Error("a middlebox needs a peer");
+	for (const peer of peers) if (!IPV4.test(peer)) throw new Error(`not an address: ${peer}`);
+	if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`not a port: ${port}`);
+	const op = idle === null ? "-D" : "-I";
+	const jumps = peers.flatMap((peer) => [[
+		"iptables",
+		op,
+		"INPUT",
+		"-s",
+		`${peer}/32`,
+		"-p",
+		"tcp",
+		"--dport",
+		String(port),
+		"-j",
+		MIDDLEBOX.input
+	], [
+		"iptables",
+		op,
+		"OUTPUT",
+		"-d",
+		`${peer}/32`,
+		"-p",
+		"tcp",
+		"--sport",
+		String(port),
+		"-j",
+		MIDDLEBOX.output
+	]]);
+	const chains = [MIDDLEBOX.input, MIDDLEBOX.output];
+	if (idle === null) return [
+		...chains.map((chain) => [
+			"iptables",
+			"-F",
+			chain
+		]),
+		...jumps,
+		...chains.map((chain) => [
+			"iptables",
+			"-X",
+			chain
+		])
+	];
+	const time = valueIn(idle, "s");
+	if (!Number.isInteger(time) || time < 1) throw new Error(`the idle timeout of the middlebox is a whole number of seconds, not ${time}`);
+	const rules = (chain, side) => {
+		const recent = [
+			"-m",
+			"recent",
+			"--name",
+			MIDDLEBOX.table,
+			side
+		];
+		return [
+			...chain === MIDDLEBOX.input ? [[
+				"iptables",
+				"-A",
+				chain,
+				"-p",
+				"tcp",
+				"--syn",
+				...recent,
+				"--set",
+				"-j",
+				"RETURN"
+			]] : [],
+			[
+				"iptables",
+				"-A",
+				chain,
+				...recent,
+				"--update",
+				"--seconds",
+				String(time),
+				"-j",
+				"RETURN"
+			],
+			[
+				"iptables",
+				"-A",
+				chain,
+				...recent,
+				"--rcheck",
+				"-j",
+				"DROP"
+			],
+			[
+				"iptables",
+				"-A",
+				chain,
+				...recent,
+				"--set",
+				"-j",
+				"RETURN"
+			]
+		];
+	};
+	return [
+		...chains.map((chain) => [
+			"iptables",
+			"-N",
+			chain
+		]),
+		...rules(MIDDLEBOX.input, "--rsource"),
+		...rules(MIDDLEBOX.output, "--rdest"),
+		...jumps
+	];
+};
 /**
 * This function gives the target of a reset or a black hole of a cell. In T07 and T18 it is Redis,
 * with the replicas as peers. In T15 the fault is on the mesh leg between the client pods and the
@@ -825,20 +1007,49 @@ const planOf = (cell) => {
 			}],
 			restore: []
 		};
-		case "blackhole": return {
-			inject: [{
-				do: "inject",
-				target: linkOf(cell),
-				action: "blackhole"
-			}],
-			hold: needs(trigger.hold, "a hold"),
-			heal: [{
-				do: "inject",
-				target: linkOf(cell),
-				action: "clear"
-			}],
-			restore: []
-		};
+		case "blackhole": {
+			const quiet = trigger.quiet === true;
+			const retries = trigger.retries;
+			if (retries !== void 0 && !(Number.isInteger(retries) && retries >= 1 && retries <= 255)) throw new Error(`tcp_retries2 is a count from 1 to 255, not ${retries}`);
+			const start = quiet ? [{
+				do: "signal",
+				target: "clients",
+				signal: "SIGUSR2"
+			}] : [];
+			const prepare = [...retries === void 0 ? [] : [{
+				do: "sysctl",
+				target: "clients",
+				key: SYSCTL.retries,
+				value: String(retries)
+			}], ...quiet ? [{
+				do: "signal",
+				target: "clients",
+				signal: "SIGUSR1"
+			}, {
+				do: "wait",
+				time: QUIET_LEAD
+			}] : []];
+			return {
+				...prepare.length === 0 ? {} : { prepare },
+				inject: [{
+					do: "inject",
+					target: linkOf(cell),
+					action: "blackhole"
+				}],
+				hold: needs(trigger.hold, "a hold"),
+				heal: [{
+					do: "inject",
+					target: linkOf(cell),
+					action: "clear"
+				}, ...start],
+				restore: [...retries === void 0 ? [] : [{
+					do: "sysctl",
+					target: "clients",
+					key: SYSCTL.retries,
+					value: NETNS_DEFAULTS.retries
+				}], ...start]
+			};
+		}
 		case "peer-gone-without-close": return {
 			inject: [{
 				do: "inject",
@@ -950,22 +1161,85 @@ const planOf = (cell) => {
 				]
 			}]
 		};
-		case "idle": return {
-			inject: [{
-				do: "signal",
-				target: "clients",
-				signal: "SIGUSR1"
-			}],
-			hold: needs(trigger.hold, "a hold, the time of the gap"),
-			heal: [{
+		case "idle": {
+			const hold = needs(trigger.hold, "a hold, the time of the gap");
+			const start = {
 				do: "signal",
 				target: "clients",
 				signal: "SIGUSR2"
+			};
+			if (trigger.close === true) {
+				const close = {
+					do: "signal",
+					target: "clients",
+					signal: "SIGHUP"
+				};
+				return {
+					prepare: [close, {
+						do: "wait",
+						time: CLOSE_LEAD
+					}],
+					inject: [
+						start,
+						{
+							do: "wait",
+							time: CHANNEL_USE
+						},
+						close
+					],
+					hold,
+					heal: [start],
+					restore: [start]
+				};
+			}
+			const middlebox = trigger.middlebox;
+			return {
+				...middlebox === void 0 ? {} : { prepare: [{
+					do: "middlebox",
+					target: "servers",
+					idle: middlebox,
+					port: HBONE_PORT
+				}] },
+				inject: [{
+					do: "signal",
+					target: "clients",
+					signal: "SIGUSR1"
+				}],
+				hold,
+				heal: [start],
+				restore: [start, ...middlebox === void 0 ? [] : [{
+					do: "middlebox",
+					target: "servers",
+					idle: null,
+					port: HBONE_PORT
+				}]]
+			};
+		}
+		case "ztunnel-restart": return {
+			inject: [{
+				do: "kubectl",
+				args: [
+					"-n",
+					"istio-system",
+					"delete",
+					"pod",
+					"-l",
+					ZTUNNEL_PODS,
+					"--wait=false"
+				]
 			}],
+			hold: null,
+			heal: [],
 			restore: [{
-				do: "signal",
-				target: "clients",
-				signal: "SIGUSR2"
+				do: "kubectl",
+				args: [
+					"-n",
+					"istio-system",
+					"rollout",
+					"status",
+					"daemonset/ztunnel",
+					"--timeout=300s"
+				]
 			}]
 		};
 	}
@@ -1296,8 +1570,55 @@ const LDS_EVERY = {
 	value: 50,
 	unit: "ms"
 };
-/** This function tells if the watcher follows the log of each ztunnel for a cell: a cell in ambient mode. */
-const followsZtunnel = (cell) => cell.mesh?.mode === "ambient";
+/**
+* This function tells if the watcher follows each ztunnel pod and its log for a cell: a cell in the
+* flavor `ambient`. A cell of sidecar pods in that flavor needs the lines of the ztunnel pods too.
+* They show the restart of ztunnel, which its pods do not see.
+*/
+const followsZtunnel = (cell) => cell.mesh?.flavor === "ambient";
+/**
+* This function gives the arguments of kubectl that set the environment of ztunnel for a cell, or
+* remove it (`undo`). The variables go in the sequence of their names.
+*/
+const ztunnelEnvArgs = (env, undo) => [
+	"-n",
+	"istio-system",
+	"set",
+	"env",
+	"daemonset/ztunnel",
+	...Object.keys(env).sort().map((key) => undo ? `${key}-` : `${key}=${env[key]}`)
+];
+/**
+* This function sets the environment of ztunnel for a cell, or removes it (`undo`). Then it waits for
+* the rollout of the DaemonSet, and until no old ztunnel pod is left. An old pod drains for its
+* deadline after the rollout replaces it. Thus no run starts beside a ztunnel pod that is going.
+*/
+const setZtunnelEnv = async (cluster, env, undo) => {
+	await cluster.kubectl(ztunnelEnvArgs(env, undo));
+	await cluster.kubectl([
+		"-n",
+		"istio-system",
+		"rollout",
+		"status",
+		"daemonset/ztunnel",
+		"--timeout=300s"
+	]);
+	const until = Date.now() + 12e4;
+	for (;;) {
+		if (JSON.parse(await cluster.kubectl([
+			"-n",
+			"istio-system",
+			"get",
+			"pods",
+			"-l",
+			"app=ztunnel",
+			"-o",
+			"json"
+		])).items.every((item) => item.metadata.deletionTimestamp === void 0)) return;
+		if (Date.now() > until) throw new Error("an old ztunnel pod is still going 120 s after the rollout");
+		await new Promise((resolve) => setTimeout(resolve, 2e3));
+	}
+};
 /** This function tells if the lab polls the LDS updates of the server sidecars of a cell: a policy in a sidecar mesh. */
 const pollsListeners = (cell) => cell.mesh?.mode === "sidecar" && cell.trigger.kind === "authorization-policy";
 //#endregion
@@ -1337,7 +1658,11 @@ const deletedOf = (text) => {
 };
 //#endregion
 //#region src/lab.ts
-/** These kinds of kernel record stay in the raw record of a run. The lab counts the other kinds and drops them. */
+/**
+* These kinds of kernel record stay in the raw record of a run. The lab also keeps each change of the
+* state of a socket of the HBONE port (`isTunnelState`): the tunnels of ztunnel. It counts the other
+* records and drops them.
+*/
 const KEPT_KERNEL = /* @__PURE__ */ new Set([
 	"rst_tx",
 	"rst_rx",
@@ -1347,6 +1672,8 @@ const KEPT_KERNEL = /* @__PURE__ */ new Set([
 	"begin",
 	"fail"
 ]);
+/** This function tells if the lab keeps a kernel record of the prober. */
+const keptKernel = (record) => KEPT_KERNEL.has(String(record["ev"])) || isTunnelState(record);
 /** This container class can share the process namespace of another container. */
 const shared = async () => {
 	const { GenericContainer } = await import("testcontainers");
@@ -1504,7 +1831,7 @@ const openKindLab = async (cluster, image) => {
 				const ev = String(record["ev"]);
 				if (ev === "begin") begun(true, "");
 				if (ev === "fail") begun(false, String(record["why"]));
-				if (!KEPT_KERNEL.has(ev) && ev !== "clock") {
+				if (!keptKernel(record) && ev !== "clock") {
 					dropped[ev] = (dropped[ev] ?? 0) + 1;
 					return;
 				}
@@ -1636,6 +1963,33 @@ const openKindLab = async (cluster, image) => {
 			});
 			return text;
 		},
+		async netns(node, pid, commands) {
+			for (const word of commands.flat()) if (!/^[A-Za-z0-9_.,:=/+-]+$/.test(word)) throw new Error(`not a plain word of a command: ${word}`);
+			if (commands.length === 0) throw new Error("no command");
+			const t = await lab.now();
+			const text = await output(sidecarOf(node).injector, [
+				"nsenter",
+				"-t",
+				String(pid),
+				"-n",
+				"sh",
+				"-c",
+				commands.map((argv) => argv.join(" ")).join(" && ")
+			]);
+			push({
+				ev: "act",
+				t,
+				t1: await lab.now(),
+				node,
+				spec: {
+					op: "netns",
+					pid,
+					commands
+				},
+				out: text.slice(0, 2e3)
+			});
+			return text;
+		},
 		async now() {
 			return (await lab.act(firstNode, { op: "now" })).t0;
 		},
@@ -2642,6 +2996,46 @@ const signalClients = async (ctx, signal) => {
 		t1: Math.max(...stamps.map((stamp) => stamp.t1))
 	};
 };
+/**
+* This function gives the network namespace of each pod of a target, as a node and a PID in it. The
+* pods are each client pod (`clients`), or each replica (`servers`).
+*/
+const namespacesOf = async (ctx, target, replicas) => {
+	if (target === "servers") return Promise.all(replicas.map(async (replica) => ({
+		node: replica.node,
+		pod: replica.pod,
+		pid: await ctx.lab.pidOf(replica.node, ctx.namespace, replica.pod, "app")
+	})));
+	if (target !== "clients") throw new Error(`a sysctl or a middlebox takes the target clients or servers, not ${target}`);
+	const clients = [...ctx.tracker.pods.values()].filter((pod) => pod.labels["app"] === "client" && !pod.del && pod.node !== null);
+	if (clients.length === 0) throw new Error("the step needs clients");
+	return Promise.all(clients.map(async (pod) => ({
+		node: pod.node,
+		pod: pod.name,
+		pid: await ctx.lab.pidOf(pod.node, ctx.namespace, pod.name, CLIENT)
+	})));
+};
+/**
+* This function puts a stateful middlebox in the network namespace of each replica, or removes it.
+* Each client pod is a peer (`middleboxCommands`), and the nodes go at the same time. After the set,
+* the lab reads the filter table: it shows the chains of the middlebox. Before the removal, the lab
+* reads the filter table again: the counters show which segments the middlebox dropped.
+*/
+const middleboxServers = async (ctx, idle, port, replicas) => {
+	const peers = [...ctx.tracker.pods.values()].filter((pod) => pod.labels["app"] === "client" && !pod.del && pod.ip !== null).map((pod) => pod.ip).sort();
+	const commands = middleboxCommands(peers, idle, port);
+	const t0 = await ctx.lab.now();
+	const each = await namespacesOf(ctx, "servers", replicas);
+	await Promise.all(each.map(async ({ node, pid }) => {
+		if (idle === null) await ctx.lab.tables(node, pid, "filter");
+		await ctx.lab.netns(node, pid, commands);
+		if (idle !== null) await ctx.lab.tables(node, pid, "filter");
+	}));
+	return {
+		t0,
+		t1: await ctx.lab.now()
+	};
+};
 /** This function does one step of a fault plan. It gives the stamp of its act, or null for an act without a stamp. */
 const doStep = async (ctx, step, replicas, background) => {
 	const { lab, cluster, namespace } = ctx;
@@ -2773,6 +3167,18 @@ const doStep = async (ctx, step, replicas, background) => {
 				t1: await lab.now()
 			};
 		}
+		case "sysctl": {
+			const command = sysctlCommand(step.key, step.value);
+			const t0 = await lab.now();
+			await Promise.all((await namespacesOf(ctx, step.target, replicas)).map(({ node, pid }) => lab.netns(node, pid, [command])));
+			return {
+				t0,
+				t1: await lab.now()
+			};
+		}
+		case "middlebox":
+			if (step.target !== "servers") throw new Error(`a middlebox takes the target servers, not ${step.target}`);
+			return middleboxServers(ctx, step.idle, step.port, replicas);
 	}
 };
 /** This function tells if a run is over: the trigger is done, and the cell is at rest again. */
@@ -2791,6 +3197,7 @@ const over = (ctx, run, lines) => {
 	].includes(kind) && (ctx.cell.template === "T07" || ctx.cell.template === "T15")) return kind === "scale-in" || kind === "node-drain" ? run.replicas.some((replica) => !ctx.tracker.pods.has(replica.pod)) : !ctx.tracker.pods.has(run.target.pod);
 	if (kind === "rolling-update" || kind === "rollout") return run.replicas.every((replica) => !ctx.tracker.pods.has(replica.pod));
 	if (kind === "app-stall") return lines.some((line) => line.ev === "stall" && line["phase"] === "end");
+	if (kind === "ztunnel-restart") return ztunnelReplaced(lines, ctx.cluster.nodes);
 	return true;
 };
 /** This function gives the replica that the trigger stopped. For a scale-in or a drain, the controller chooses it. */
@@ -2848,6 +3255,14 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
 	const notes = [];
 	const forgotten = Object.entries(cleared).filter(([, count]) => count > 0);
 	if (forgotten.length > 0) notes.push(`before the run, the lab deleted SYN_SENT entries of the ClusterIP: ${forgotten.map(([node, count]) => `${count} on ${node}`).join(", ")}`);
+	for (const step of plan.prepare ?? []) try {
+		await doStep(ctx, step, replicas, background);
+	} catch (error) {
+		const why = `the step ${step.do} before the fault failed: ${String(error).slice(0, 400)}`;
+		ctx.log(`  ${why}`);
+		for (const each of plan.restore) await doStep(ctx, each, replicas, background).catch((failure) => ctx.log(`  the restore ${each.do} failed: ${String(failure)}`));
+		return empty(why, start);
+	}
 	for (const step of plan.inject) {
 		const stamp = await doStep(ctx, step, replicas, background);
 		if (stamp !== null && fault === null) fault = stamp.t0;
@@ -3050,7 +3465,9 @@ const runKindCells = async (cells, options) => {
 					const tracker = new Tracker();
 					const off = lab.listen(tracker.see);
 					const loads = [];
+					const ztunnelEnv = cell.mesh?.ztunnel;
 					try {
+						if (ztunnelEnv !== void 0) await setZtunnelEnv(cluster, ztunnelEnv, false);
 						const clusterIp = await deploy(lab, cluster, cell, namespace, images);
 						for (const name of [...lab.own(), loader.getName().replace(/^\//, "")]) ours.add(name);
 						const ctx = {
@@ -3112,6 +3529,7 @@ const runKindCells = async (cells, options) => {
 							"--wait=true",
 							"--timeout=300s"
 						]).catch((error) => options.log(`  ${String(error)}`));
+						if (ztunnelEnv !== void 0) await setZtunnelEnv(cluster, ztunnelEnv, true).catch((error) => options.log(`  the undo of the environment of ztunnel failed: ${String(error)}`));
 					}
 				}
 			} finally {
@@ -3149,12 +3567,14 @@ const estimate = (cell, environments = 2) => {
 		"cpu-throttle": hold + 40,
 		rollout: hold + 50,
 		"authorization-policy": hold + 20,
-		idle: hold + 5
+		idle: hold + 5,
+		"ztunnel-restart": 40
 	};
 	const after = observationAfter(cell);
 	const observed = after === null ? 0 : hold + valueIn(after, "s");
 	const closes = cell.prediction.kind === "ends" && cell.prediction.band !== null ? valueIn(cell.prediction.band.hi, "s") + 3 : 0;
-	const each = 25 + Math.max(observed, closes, trigger[cell.trigger.kind] ?? (cell.template === "T18" ? hold + 35 : 15 + valueIn(cell.pod.preStop, "s")));
+	const quiet = cell.trigger.quiet === true ? hold + 10 : 0;
+	const each = 25 + Math.max(observed, closes, quiet, trigger[cell.trigger.kind] ?? (cell.template === "T18" ? hold + 35 : 15 + valueIn(cell.pod.preStop, "s")));
 	return (cell.mesh === void 0 ? 60 : 180) * environments + (cell.runs + environments) * each;
 };
 /**
@@ -3186,6 +3606,6 @@ const shardOf = (cells, index, count) => {
 	return out;
 };
 //#endregion
-export { ACCESS_LOG_FORMAT, CLIENT, CLIENT_ACCOUNT, CONTROLLER, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HEALTH_PORT, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, NODE_IMAGES, O04, O09, O10, O16, POLICY, PORT, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, STAGES, WATCHDOG, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, checkRun, cidrOverlap, cidrRange, clientObjects, clusterFilter, compose, createCluster, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, followsZtunnel, goDuration, hostsOf, installMesh, intakeAfter, intakeOf, istioOperator, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, latchBand, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, namespaceObject, namespaceOf, nodeNames, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stoppingOf, tarEntry, timeSpread, toolPath, transitionOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, writeResult };
+export { ACCESS_LOG_FORMAT, CHANNEL_USE, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, MIDDLEBOX, NETNS_DEFAULTS, NODE_IMAGES, O04, O09, O10, O16, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, STAGES, SYSCTL, TCP_STATE, WATCHDOG, ZTUNNEL_PODS, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, checkRun, cidrOverlap, cidrRange, clientObjects, clusterFilter, compose, createCluster, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, followsZtunnel, goDuration, hostsOf, installMesh, intakeAfter, intakeOf, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, latchBand, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, middleboxCommands, namespaceObject, namespaceOf, nodeNames, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, setZtunnelEnv, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stoppingOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, writeResult, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
 
 //# sourceMappingURL=index.js.map