@mark1russell7/kind 0.11.9 → 0.11.11

package/build/index.d.ts

@@ -1,4 +1,4 @@
-import { $ as unitOf, $t as secondsOf, A as TriggerV1, An as cidrOverlap, At as restartsAt, B as RunCheck, Bt as stoppingOf, C as KindCellResult, Cn as Workload, Ct as kernelCounts, D as PredictionV1, Dn as KubeProxyMode, Dt as recordsOf, E as LoadSample, En as KindScenario, Et as podsOf, F as KIND_NOISE, Fn as nodeNames, Ft as sliceEmptyOf, G as factsOf, Gt as CLIENT, H as checkRun, Ht as tunnelsOf, I as KindDecision, In as scenario, It as sliceReadyAt, J as observationEnd, Jt as PORT, K as judgeKind, Kt as HEALTH_PORT, L as KindVerdict, Ln as scenarioProblems, Lt as sliceReadyOf, M as upgrade, Mn as goDuration, Mt as signalOf, N as upgradeCell, Nn as kindConfig, Nt as signalsOf, O as RunCheckV1, On as NODE_IMAGES, Ot as requestsOf, P as Deleted, Pn as kubeProxyPatch, Pt as sliceDropOf, Q as reusesReset, Qt as seconds, R as LAST_SLACK, Rt as slicesOf, S as KindCellRecordV1, Sn as WindowPrediction, St as isTunnelState, T as KindVerdictV1, Tn as KindConfig, Tt as podsAt, U as cyclesOf, Ut as ztunnelPodsOf, V as RunFacts, Vt as transitionOf, W as deletionsOf, Wt as ztunnelReplaced, X as propagationSource, Xt as SERVER, Y as propagationOf, Yt as REDIS, Z as referenceOf, Zt as bandOf, _ as ClusterManifest, _n as ResetReuse, _t as deletionOf, a as STAGES, an as Flap, at as PropagationSource, b as FORMAT_V1, bn as Trigger, bt as failuresOf, c as readRecord, cn as KubeletProbe, ct as RulesLine, d as calibrationOf, dn as MeshMode, dt as TCP_STATE, en as showBand, et as updatesOf, f as cellsTable, fn as PodSettings, ft as TunnelLine, g as BandV1, gn as Replica, gt as byNode, h as timeSpread, hn as Reference, ht as byClients, i as quantileOf, in as EdgeProtocol, it as PodLine, j as judge, jn as cidrRange, jt as rulesOf, k as RunFactsV1, kn as NodeImage, kt as restartOf, l as Judged, ln as Mesh, lt as SliceLine, m as predictionText, mn as RedisDeclaration, mt as backendsAt, n as medianInterval, nn as Cycle, nt as HBONE_PORT, o as kindCellRecordSchema, on as KindCell, ot as RecordLine, p as medianText, pn as Prediction, pt as ZtunnelPodLine, q as observationAfter, qt as LOG_LABEL, r as middle, rn as Declaration, rt as Line, s as kindCellRecordV1Schema, sn as KindRun, st as Request, t as RELATIVE, tn as Band, tt as Endpoint, u as TimeSpread, un as MeshFlavor, ut as SocketLine, v as EnvironmentManifest, vn as Stack, vt as dropOf, w as KindCellV1, wn as KUBE_PROXY_MODES, wt as outcomesOf, x as KindCellRecord, xn as TriggerKind, xt as intakeOf, y as FORMAT, yn as Template, yt as emptyOf, z as Propagated, zt as socketsOf } from "./judge-BBHSeu4i.js";
+import { $ as unitOf, $t as secondsOf, A as TriggerV1, An as NodeImage, At as restartsAt, B as RunCheck, Bn as scenario, Bt as stoppingOf, C as KindCellResult, Cn as WindowPrediction, Ct as kernelCounts, D as PredictionV1, Dn as KindScenario, Dt as recordsOf, E as LoadSample, En as KindConfig, Et as podsOf, F as KIND_NOISE, Fn as cidrRange, Ft as sliceEmptyOf, G as factsOf, Gt as CLIENT, H as checkRun, Hn as zoneOf, Ht as tunnelsOf, I as KindDecision, In as goDuration, It as sliceReadyAt, J as observationEnd, Jt as PORT, K as judgeKind, Kt as HEALTH_PORT, L as KindVerdict, Ln as kindConfig, Lt as sliceReadyOf, M as upgrade, Mn as ZONE_LABEL, Mt as signalOf, N as upgradeCell, Nn as ZONE_LOST, Nt as signalsOf, O as RunCheckV1, On as KubeProxyMode, Ot as requestsOf, P as Deleted, Pn as cidrOverlap, Pt as sliceDropOf, Q as reusesReset, Qt as seconds, R as LAST_SLACK, Rn as kubeProxyPatch, Rt as slicesOf, S as KindCellRecordV1, Sn as TriggerKind, St as isTunnelState, T as KindVerdictV1, Tn as KUBE_PROXY_MODES, Tt as podsAt, U as cyclesOf, Ut as ztunnelPodsOf, V as RunFacts, Vn as scenarioProblems, Vt as transitionOf, W as deletionsOf, Wt as ztunnelReplaced, X as propagationSource, Xt as SERVER, Y as propagationOf, Yt as REDIS, Z as referenceOf, Zt as bandOf, _ as ClusterManifest, _n as Replica, _t as deletionOf, a as STAGES, an as Flap, at as PropagationSource, b as FORMAT_V1, bn as Template, bt as failuresOf, c as readRecord, cn as KubeletProbe, ct as RulesLine, d as calibrationOf, dn as MeshMode, dt as TCP_STATE, en as showBand, et as updatesOf, f as cellsTable, fn as NonePrediction, ft as TunnelLine, g as BandV1, gn as Reference, gt as byNode, h as timeSpread, hn as RedisDeclaration, ht as byClients, i as quantileOf, in as EdgeProtocol, it as PodLine, j as judge, jn as ZONE_KEPT, jt as rulesOf, k as RunFactsV1, kn as NODE_IMAGES, kt as restartOf, l as Judged, ln as Mesh, lt as SliceLine, m as predictionText, mn as Prediction, mt as backendsAt, n as medianInterval, nn as Cycle, nt as HBONE_PORT, o as kindCellRecordSchema, on as KindCell, ot as RecordLine, p as medianText, pn as PodSettings, pt as ZtunnelPodLine, q as observationAfter, qt as LOG_LABEL, r as middle, rn as Declaration, rt as Line, s as kindCellRecordV1Schema, sn as KindRun, st as Request, t as RELATIVE, tn as Band, tt as Endpoint, u as TimeSpread, un as MeshFlavor, ut as SocketLine, v as EnvironmentManifest, vn as ResetReuse, vt as dropOf, w as KindCellV1, wn as Workload, wt as outcomesOf, x as KindCellRecord, xn as Trigger, xt as intakeOf, y as FORMAT, yn as Stack, yt as emptyOf, z as Propagated, zn as nodeNames, zt as socketsOf } from "./judge-0ku_4dsJ.js";
 import { Quantity } from "@mark1russell7/quantity";
 import { CheckFn, Finding, HealthServer, Poll, Scenario, ServiceBuilder, ServiceDescription, Stage, StopWay, Timing, TransportImpl, Verdict } from "@mark1russell7/service";
 import { BuildImage } from "@mark1russell7/harness";
@@ -169,6 +169,13 @@ type Step = {
   readonly port: number;
 } | {
   readonly do: "certificates";
+} | {
+  readonly do: "zone";
+  readonly lost: boolean;
+} | {
+  readonly do: "netem";
+  readonly target: Target;
+  readonly delay: Quantity | null;
 };
 /**
  * A fault plan is the steps of a run:
@@ -275,6 +282,14 @@ export declare const planOf: (cell: KindCell) => FaultPlan;
  * then the rest and the hold of each flap. A trigger with no hold gives zero.
  */
 export declare const faultSpan: (trigger: Pick<KindCell["trigger"], "hold" | "flaps">) => Quantity;
+/** This device is the link of a pod: the end of its veth pair in its network namespace. */
+export declare const POD_LINK = "eth0";
+/**
+ * This time is how long the heal of a delay waits from the delay of zero to the removal of the
+ * qdisc. Each packet in the queue leaves in it, because each one waits the delay at the most. A delay
+ * of a cell must be shorter than it.
+ */
+export declare const NETEM_DRAIN: Quantity;
 /** This function tells if a trigger stops a pod of the service with its grace: a deletion, an eviction, a scale-in or a rollout. */
 export declare const deletes: (cell: KindCell) => boolean;
 //#endregion
@@ -627,6 +642,64 @@ export declare const restartBand: (latch: Band, liveness: KubeletProbe, preStop:
  * replica from the rules. Each probe in the stall fails at its timeout, because the loop does not answer it.
  */
 export declare const stallRemovalBand: (readiness: KubeletProbe, propagation: Propagation) => Band;
+/**
+ * A node lifecycle is the set of timers that removes the pods of a lost node. The same timers give
+ * them back after the heal of the node. The lab sets none of them, thus each one is a default of Kubernetes
+ * v1.37.0 (file and line in that tag).
+ */
+interface NodeLifecycle {
+  /**
+   * This time is `NodeMonitorGracePeriod`. The node lifecycle controller marks a node Unknown when
+   * it saw no renewal of its lease for this time. Then it marks each pod of the node not ready.
+   */
+  readonly grace: Quantity;
+  /** This time is `NodeMonitorPeriod`: the controller checks each node once in it. */
+  readonly monitor: Quantity;
+  /** This time is the renewal interval of the lease of the kubelet. */
+  readonly renew: Quantity;
+  /** This share is the jitter of each renewal interval: a renewal comes `renew` to `renew × (1 + jitter)` after the one before it. */
+  readonly jitter: number;
+  /** This time is the margin of the writes of the controller: the status of the node, then each pod of the node. */
+  readonly writes: Quantity;
+  /** This time is the PING timeout of the HTTP/2 health check of client-go. A kubelet that gets no PING ACK in it connects again. */
+  readonly pingTimeout: Quantity;
+  /** This time is the longest first backoff of a reflector of client-go before it watches again. */
+  readonly rewatch: Quantity;
+  /** This time is the sync period of the status manager of the kubelet: it puts back a pod status that the controller changed. */
+  readonly statusSync: Quantity;
+}
+/**
+ * These are the defaults of Kubernetes v1.37.0 for a lost node:
+ * - `grace` 50 s and `monitor` 5 s: pkg/controller/nodelifecycle/config/v1alpha1/defaults.go, lines
+ *   46 and 52. The controller checks the nodes on a sliding period (node_lifecycle_controller.go,
+ *   lines 506–510). It sets the probe time when it sees a new lease (lines 952–960), and it marks the
+ *   node Unknown after `grace` (line 962). Then it marks each pod of the node not ready (lines 768–773).
+ * - `renew` 10 s: `NodeLeaseDurationSeconds` 40 (pkg/kubelet/apis/config/v1beta1/defaults.go, line 150)
+ *   times the fraction 0.25 (pkg/kubelet/kubelet.go, lines 237 and 1156).
+ * - `jitter` 0.04: the lease controller renews on a sliding jitter of 0.04
+ *   (staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go, line 153).
+ * - `pingTimeout` 15 s: the PING timeout of the HTTP/2 health check, after a read idle timeout of 30 s
+ *   (staging/src/k8s.io/apimachinery/pkg/util/net/http.go, lines 131–192).
+ * - `rewatch` 1.6 s: a first backoff of 800 ms with a jitter of 1.0
+ *   (staging/src/k8s.io/client-go/tools/cache/reflector.go, lines 62–67).
+ * - `statusSync` 10 s: the sync period of the status manager (pkg/kubelet/status/status_manager.go, line 215).
+ * - `writes` 0.25 s is a margin of the lab, not a default.
+ */
+export declare const NODE_LIFECYCLE: NodeLifecycle;
+/**
+ * This function gives the time from the loss of a node to the drop of its pods from the rules of
+ * the other nodes. The last renewal of the lease comes up to `renew × (1 + jitter)` before the loss.
+ * The controller sees it up to one `monitor` later, and marks the node `grace` after that. Then the
+ * writes and the propagation follow. The band does not add the low bound of the propagation.
+ */
+export declare const nodeLossBand: (lifecycle: NodeLifecycle, propagation: Propagation) => Band;
+/**
+ * This function gives the time from the heal of a lost node until its pods are ready again in the
+ * EndpointSlice. The kubelet connects again in one PING timeout at the most: its old connection
+ * waits behind the backoff of TCP. Its reflectors watch again after their first backoff. Then its
+ * status manager puts back the pod status in one sync period at the most.
+ */
+export declare const nodeReturnBand: (lifecycle: NodeLifecycle) => Band;
 /** This function gives the cap of a stage in a declaration, or zero when the declaration does not name the stage. */
 export declare const capOf: (declaration: Declaration, stage: Stage) => Quantity;
 /**
@@ -706,6 +779,7 @@ type Verdicts = Readonly<Record<string, Verdict>>;
  * - In T07 a pause of kube-proxy comes before a deletion. In T18 the pause is on Redis.
  * - The crash of the lab is SIGKILL from the node. Like a crash, it has no preStop hook and no drain.
  * - The other faults are on a link, on Redis or on the process. The kubelet can restart a pod after them.
+ *   A zone loss freezes the pods of its node and deletes none of them in its hold.
  */
 export declare const stopOf: (cell: KindCell) => StopWay | null;
 /**
@@ -812,5 +886,5 @@ export declare const toolPath: (tool: Tool, key?: string, cache?: string) => str
  */
 export declare const ensureTool: (tool: Tool, key?: string, cache?: string) => Promise<string>;
 //#endregion
-export { type Band, type BandV1, CLIENT, type Cluster, type ClusterHosts, type ClusterManifest, type Credentials, type Cycle, DRAIN_RULES, type Declaration, type Deleted, type DeletionWindow, type EdgeProtocol, type EdsSpec, Endpoint, type EnvironmentManifest, FORMAT, FORMAT_V1, type FaultPlan, type Flap, HBONE_PORT, HEALTH_PORT, type Judged, KIND_NOISE, KUBE_PROXY_MODES, type KindCell, type KindCellRecord, type KindCellRecordV1, type KindCellResult, type KindCellV1, type KindConfig, type KindDecision, type KindLab, type KindRun, type KindScenario, type KindVerdict, type KindVerdictV1, type KubeProxyMode, type KubeletProbe, LAST_SLACK, LOG_LABEL, type LdsSpec, type LeafCertificate, type LearnWindow, Line, type ListenerCounts, type LoadSample, type Mesh, type MeshFlavor, type MeshImages, type MeshMode, NODE_IMAGES, type NodeImage, O04, O09, O10, O16, PORT, type Parts, PodLine, type PodSettings, type Prediction, type PredictionV1, type Propagated, type Propagation, PropagationSource, REDIS, RELATIVE, RecordLine, type RedisDeclaration, type Reference, type Replica, Request, type ResetReuse, RulesLine, type RunCheck, type RunCheckV1, type RunFacts, type RunFactsV1, type RunOptions, SERVER, STAGES, type ServiceRule, SliceLine, SocketLine, type Stack, type Stamp, type Step, TCP_STATE, type Target, type Template, type TimeSpread, type Tool, type Trigger, type TriggerKind, type TriggerV1, TunnelLine, type Verdicts, type WindowPrediction, type Workload, ZtunnelPodLine, backendsAt, bandOf, byClients, byNode, calibrationOf, cellsTable, checkRun, cidrOverlap, cidrRange, cyclesOf, deletionOf, deletionsOf, dropOf, emptyOf, factsOf, failuresOf, goDuration, intakeOf, isTunnelState, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, medianInterval, medianText, middle, nodeNames, observationAfter, observationEnd, outcomesOf, podsAt, podsOf, predictionText, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, referenceOf, requestsOf, restartOf, restartsAt, reusesReset, rulesOf, scenario, scenarioProblems, seconds, secondsOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stoppingOf, timeSpread, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, ztunnelPodsOf, ztunnelReplaced };
+export { type Band, type BandV1, CLIENT, type Cluster, type ClusterHosts, type ClusterManifest, type Credentials, type Cycle, DRAIN_RULES, type Declaration, type Deleted, type DeletionWindow, type EdgeProtocol, type EdsSpec, Endpoint, type EnvironmentManifest, FORMAT, FORMAT_V1, type FaultPlan, type Flap, HBONE_PORT, HEALTH_PORT, type Judged, KIND_NOISE, KUBE_PROXY_MODES, type KindCell, type KindCellRecord, type KindCellRecordV1, type KindCellResult, type KindCellV1, type KindConfig, type KindDecision, type KindLab, type KindRun, type KindScenario, type KindVerdict, type KindVerdictV1, type KubeProxyMode, type KubeletProbe, LAST_SLACK, LOG_LABEL, type LdsSpec, type LeafCertificate, type LearnWindow, Line, type ListenerCounts, type LoadSample, type Mesh, type MeshFlavor, type MeshImages, type MeshMode, NODE_IMAGES, type NodeImage, type NodeLifecycle, type NonePrediction, O04, O09, O10, O16, PORT, type Parts, PodLine, type PodSettings, type Prediction, type PredictionV1, type Propagated, type Propagation, PropagationSource, REDIS, RELATIVE, RecordLine, type RedisDeclaration, type Reference, type Replica, Request, type ResetReuse, RulesLine, type RunCheck, type RunCheckV1, type RunFacts, type RunFactsV1, type RunOptions, SERVER, STAGES, type ServiceRule, SliceLine, SocketLine, type Stack, type Stamp, type Step, TCP_STATE, type Target, type Template, type TimeSpread, type Tool, type Trigger, type TriggerKind, type TriggerV1, TunnelLine, type Verdicts, type WindowPrediction, type Workload, ZONE_KEPT, ZONE_LABEL, ZONE_LOST, ZtunnelPodLine, backendsAt, bandOf, byClients, byNode, calibrationOf, cellsTable, checkRun, cidrOverlap, cidrRange, cyclesOf, deletionOf, deletionsOf, dropOf, emptyOf, factsOf, failuresOf, goDuration, intakeOf, isTunnelState, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, medianInterval, medianText, middle, nodeNames, observationAfter, observationEnd, outcomesOf, podsAt, podsOf, predictionText, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, referenceOf, requestsOf, restartOf, restartsAt, reusesReset, rulesOf, scenario, scenarioProblems, seconds, secondsOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stoppingOf, timeSpread, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, zoneOf, ztunnelPodsOf, ztunnelReplaced };
 //# sourceMappingURL=index.d.ts.map

package/build/index.js

@@ -1,4 +1,4 @@
-import { $ as rulesOf, A as RELATIVE, B as dropOf, C as observationEnd, D as reusesReset, E as referenceOf, F as TCP_STATE, G as kernelCounts, H as failuresOf, I as backendsAt, J as podsOf, K as outcomesOf, L as byClients, M as middle, N as quantileOf, O as unitOf, P as HBONE_PORT, Q as restartsAt, R as byNode, S as observationAfter, St as showBand, T as propagationSource, U as intakeOf, V as emptyOf, W as isTunnelState, X as requestsOf, Y as recordsOf, Z as restartOf, _ as checkRun, _t as REDIS, a as calibrationOf, at as sliceReadyOf, b as factsOf, bt as seconds, c as predictionText, ct as stoppingOf, d as FORMAT_V1, dt as ztunnelPodsOf, et as signalOf, f as judge, ft as ztunnelReplaced, g as LAST_SLACK, gt as PORT, h as KIND_NOISE, ht as LOG_LABEL, i as readRecord, it as sliceReadyAt, j as medianInterval, k as updatesOf, l as timeSpread, lt as transitionOf, m as upgradeCell, mt as HEALTH_PORT, n as kindCellRecordSchema, nt as sliceDropOf, o as cellsTable, ot as slicesOf, p as upgrade, pt as CLIENT, q as podsAt, r as kindCellRecordV1Schema, rt as sliceEmptyOf, s as medianText, st as socketsOf, t as STAGES, tt as signalsOf, u as FORMAT, ut as tunnelsOf, v as cyclesOf, vt as SERVER, w as propagationOf, x as judgeKind, xt as secondsOf, y as deletionsOf, yt as bandOf, z as deletionOf } from "./judge-ZoLw3AXV.js";
+import { $ as rulesOf, A as RELATIVE, B as dropOf, C as observationEnd, D as reusesReset, E as referenceOf, F as TCP_STATE, G as kernelCounts, H as failuresOf, I as backendsAt, J as podsOf, K as outcomesOf, L as byClients, M as middle, N as quantileOf, O as unitOf, P as HBONE_PORT, Q as restartsAt, R as byNode, S as observationAfter, St as showBand, T as propagationSource, U as intakeOf, V as emptyOf, W as isTunnelState, X as requestsOf, Y as recordsOf, Z as restartOf, _ as checkRun, _t as REDIS, a as calibrationOf, at as sliceReadyOf, b as factsOf, bt as seconds, c as predictionText, ct as stoppingOf, d as FORMAT_V1, dt as ztunnelPodsOf, et as signalOf, f as judge, ft as ztunnelReplaced, g as LAST_SLACK, gt as PORT, h as KIND_NOISE, ht as LOG_LABEL, i as readRecord, it as sliceReadyAt, j as medianInterval, k as updatesOf, l as timeSpread, lt as transitionOf, m as upgradeCell, mt as HEALTH_PORT, n as kindCellRecordSchema, nt as sliceDropOf, o as cellsTable, ot as slicesOf, p as upgrade, pt as CLIENT, q as podsAt, r as kindCellRecordV1Schema, rt as sliceEmptyOf, s as medianText, st as socketsOf, t as STAGES, tt as signalsOf, u as FORMAT, ut as tunnelsOf, v as cyclesOf, vt as SERVER, w as propagationOf, x as judgeKind, xt as secondsOf, y as deletionsOf, yt as bandOf, z as deletionOf } from "./judge-CLBiWALR.js";
 import { valueIn } from "@mark1russell7/quantity";
 import { execFile, execFileSync } from "node:child_process";
 import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
@@ -92,8 +92,16 @@ const NODE_IMAGES = {
 		image: "kindest/node:v1.36.4@sha256:099e049362a1526b2db71494e1947aae99bd16290d7c895f2b7ea312e3cbfaed"
 	}
 };
+/** This label gives the zone of a node, as Kubernetes names it. */
+const ZONE_LABEL = "topology.kubernetes.io/zone";
+/** This zone holds the control plane and each worker but the last. A zone loss keeps it. */
+const ZONE_KEPT = "zone-a";
+/** This zone holds only the last worker. A zone loss loses it. */
+const ZONE_LOST = "zone-b";
+/** This function gives the zone of the node at the position `at` of `nodeNames`, in a cluster of `count` nodes. */
+const zoneOf = (at, count) => at === count - 1 && at > 0 ? ZONE_LOST : ZONE_KEPT;
 /** This function gives a scenario with the defaults of Kubernetes for kube-proxy, as explicit values. */
-const scenario = (name, mode, workers, kubernetes = "1.37") => ({
+const scenario = (name, mode, workers, kubernetes = "1.37", zoned = false) => ({
 	name,
 	kubernetes,
 	workers,
@@ -110,7 +118,8 @@ const scenario = (name, mode, workers, kubernetes = "1.37") => ({
 	},
 	cni: "kindnet",
 	podSubnet: "10.244.0.0/16",
-	serviceSubnet: "10.96.0.0/16"
+	serviceSubnet: "10.96.0.0/16",
+	...zoned ? { zoned } : {}
 });
 const NAME = /^[a-z0-9]([a-z0-9-]{0,30}[a-z0-9])?$/;
 /** This function gives the problems of a scenario: a name that kind refuses, no image for the version, or a count out of range. */
@@ -155,10 +164,13 @@ const kindConfig = (s) => {
 			podSubnet: s.podSubnet,
 			serviceSubnet: s.serviceSubnet
 		},
-		nodes: nodeNames(s).map((node, at) => ({
+		nodes: nodeNames(s).map((node, at, all) => ({
 			role: at === 0 ? "control-plane" : "worker",
 			image,
-			labels: { "lab.systems/node": node.slice(s.name.length + 1) }
+			labels: {
+				"lab.systems/node": node.slice(s.name.length + 1),
+				...s.zoned === true ? { [ZONE_LABEL]: zoneOf(at, all.length) } : {}
+			}
 		})),
 		kubeadmConfigPatches: [kubeProxyPatch(s)]
 	};
@@ -882,6 +894,17 @@ const faultSpan = (trigger) => {
 		unit: "s"
 	};
 };
+/** This device is the link of a pod: the end of its veth pair in its network namespace. */
+const POD_LINK = "eth0";
+/**
+* This time is how long the heal of a delay waits from the delay of zero to the removal of the
+* qdisc. Each packet in the queue leaves in it, because each one waits the delay at the most. A delay
+* of a cell must be shorter than it.
+*/
+const NETEM_DRAIN = {
+	value: 1,
+	unit: "s"
+};
 /** This function gives the steps of the fault plan of a cell, with no flaps. */
 const stepsOf = (cell) => {
 	const ns = ["-n", "{namespace}"];
@@ -1334,6 +1357,62 @@ const stepsOf = (cell) => {
 				on: false
 			}]
 		};
+		case "zone-loss": return {
+			prepare: [{
+				do: "signal",
+				target: "clients",
+				signal: "SIGHUP"
+			}, {
+				do: "wait",
+				time: SPREAD_LEAD
+			}],
+			inject: [{
+				do: "zone",
+				lost: true
+			}],
+			hold: needs(trigger.hold, "a hold, the time that the zone is lost"),
+			heal: [{
+				do: "zone",
+				lost: false
+			}],
+			restore: [{
+				do: "zone",
+				lost: false
+			}]
+		};
+		case "latency": {
+			const delay = needs(trigger.delay, "a delay, the time that each packet of Redis waits");
+			if (valueIn(delay, "ms") <= 0 || valueIn(delay, "ms") >= valueIn(NETEM_DRAIN, "ms")) throw new Error(`the delay of latency is above zero and below ${valueIn(NETEM_DRAIN, "s")} s`);
+			const back = [
+				{
+					do: "netem",
+					target: "redis",
+					delay: {
+						value: 0,
+						unit: "s"
+					}
+				},
+				{
+					do: "wait",
+					time: NETEM_DRAIN
+				},
+				{
+					do: "netem",
+					target: "redis",
+					delay: null
+				}
+			];
+			return {
+				inject: [{
+					do: "netem",
+					target: "redis",
+					delay
+				}],
+				hold: needs(trigger.hold, "a hold, the time that Redis is slow"),
+				heal: back,
+				restore: back
+			};
+		}
 	}
 };
 /** This function tells if a trigger stops a pod of the service with its grace: a deletion, an eviction, a scale-in or a rollout. */
@@ -2450,7 +2529,12 @@ const serverObjects = (cell, namespace, image, version = 1) => {
 						topologyKey: "kubernetes.io/hostname",
 						whenUnsatisfiable: "ScheduleAnyway",
 						labelSelector: { matchLabels: { app: "lab" } }
-					}],
+					}, ...cell.trigger.kind === "zone-loss" ? [{
+						maxSkew: 1,
+						topologyKey: ZONE_LABEL,
+						whenUnsatisfiable: "DoNotSchedule",
+						labelSelector: { matchLabels: { app: "lab" } }
+					}] : []],
 					containers: [{
 						name: "app",
 						image,
@@ -2492,6 +2576,11 @@ const clientObjects = (cell, namespace, image, target) => [{
 					operator: "Exists",
 					effect: "NoSchedule"
 				}],
+				...cell.trigger.kind === "zone-loss" ? { affinity: { nodeAffinity: { requiredDuringSchedulingIgnoredDuringExecution: { nodeSelectorTerms: [{ matchExpressions: [{
+					key: ZONE_LABEL,
+					operator: "NotIn",
+					values: [ZONE_LOST]
+				}] }] } } } } : {},
 				containers: [{
 					name: CLIENT,
 					image,
@@ -2604,6 +2693,50 @@ const stallRemovalBand = (readiness, propagation) => {
 	const spread = secondsOf(propagation);
 	return band(kubelet.lo + spread.lo, kubelet.hi + spread.hi);
 };
+/**
+* These are the defaults of Kubernetes v1.37.0 for a lost node:
+* - `grace` 50 s and `monitor` 5 s: pkg/controller/nodelifecycle/config/v1alpha1/defaults.go, lines
+*   46 and 52. The controller checks the nodes on a sliding period (node_lifecycle_controller.go,
+*   lines 506–510). It sets the probe time when it sees a new lease (lines 952–960), and it marks the
+*   node Unknown after `grace` (line 962). Then it marks each pod of the node not ready (lines 768–773).
+* - `renew` 10 s: `NodeLeaseDurationSeconds` 40 (pkg/kubelet/apis/config/v1beta1/defaults.go, line 150)
+*   times the fraction 0.25 (pkg/kubelet/kubelet.go, lines 237 and 1156).
+* - `jitter` 0.04: the lease controller renews on a sliding jitter of 0.04
+*   (staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go, line 153).
+* - `pingTimeout` 15 s: the PING timeout of the HTTP/2 health check, after a read idle timeout of 30 s
+*   (staging/src/k8s.io/apimachinery/pkg/util/net/http.go, lines 131–192).
+* - `rewatch` 1.6 s: a first backoff of 800 ms with a jitter of 1.0
+*   (staging/src/k8s.io/client-go/tools/cache/reflector.go, lines 62–67).
+* - `statusSync` 10 s: the sync period of the status manager (pkg/kubelet/status/status_manager.go, line 215).
+* - `writes` 0.25 s is a margin of the lab, not a default.
+*/
+const NODE_LIFECYCLE = {
+	grace: seconds(50),
+	monitor: seconds(5),
+	renew: seconds(10),
+	jitter: .04,
+	writes: seconds(.25),
+	pingTimeout: seconds(15),
+	rewatch: seconds(1.6),
+	statusSync: seconds(10)
+};
+/**
+* This function gives the time from the loss of a node to the drop of its pods from the rules of
+* the other nodes. The last renewal of the lease comes up to `renew × (1 + jitter)` before the loss.
+* The controller sees it up to one `monitor` later, and marks the node `grace` after that. Then the
+* writes and the propagation follow. The band does not add the low bound of the propagation.
+*/
+const nodeLossBand = (lifecycle, propagation) => {
+	const last = s$1(lifecycle.renew) * (1 + lifecycle.jitter);
+	return band(s$1(lifecycle.grace) - last, s$1(lifecycle.grace) + s$1(lifecycle.monitor) + s$1(lifecycle.writes) + s$1(propagation.hi));
+};
+/**
+* This function gives the time from the heal of a lost node until its pods are ready again in the
+* EndpointSlice. The kubelet connects again in one PING timeout at the most: its old connection
+* waits behind the backoff of TCP. Its reflectors watch again after their first backoff. Then its
+* status manager puts back the pod status in one sync period at the most.
+*/
+const nodeReturnBand = (lifecycle) => band(0, s$1(lifecycle.pingTimeout) + s$1(lifecycle.rewatch) + s$1(lifecycle.statusSync) + s$1(lifecycle.writes));
 /** This function gives the cap of a stage in a declaration, or zero when the declaration does not name the stage. */
 const capOf = (declaration, stage) => {
 	const found = declaration.phases.find(([name]) => name === stage);
@@ -2709,6 +2842,7 @@ const scenarioOf = (cell, propagation) => ({
 * - In T07 a pause of kube-proxy comes before a deletion. In T18 the pause is on Redis.
 * - The crash of the lab is SIGKILL from the node. Like a crash, it has no preStop hook and no drain.
 * - The other faults are on a link, on Redis or on the process. The kubelet can restart a pod after them.
+*   A zone loss freezes the pods of its node and deletes none of them in its hold.
 */
 const stopOf = (cell) => {
 	switch (cell.trigger.kind) {
@@ -2729,7 +2863,9 @@ const stopOf = (cell) => {
 		case "idle":
 		case "ztunnel-restart":
 		case "certificate-expiry":
-		case "cannot-serve": return null;
+		case "cannot-serve":
+		case "zone-loss":
+		case "latency": return null;
 	}
 };
 /**
@@ -2768,8 +2904,13 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const slug = (name) => name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
 /** This function tells if a cell has Redis: only a declaration of the service can have it. */
 const hasRedis = (cell) => cell.declaration !== null && cell.declaration.redis !== null;
-/** This function gives the key of the cluster of a cell: its kube-proxy mode, its node count, and the flavor of its mesh. */
-const clusterKey = (cell) => cell.mesh === void 0 ? `${cell.mode}-${cell.workers}` : `${cell.mode}-${cell.workers}-${cell.mesh.flavor}`;
+/**
+* This function gives the key of the cluster of a cell: its kube-proxy mode, its node count, and the
+* flavor of its mesh. A zone loss has clusters of its own, with a zone on each node.
+*/
+const clusterKey = (cell) => cell.mesh !== void 0 ? `${cell.mode}-${cell.workers}-${cell.mesh.flavor}` : zoned(cell) ? `${cell.mode}-${cell.workers}-zones` : `${cell.mode}-${cell.workers}`;
+/** This function tells if a cell needs a cluster with zones: a zone loss. */
+const zoned = (cell) => cell.trigger.kind === "zone-loss";
 /**
 * This tracker keeps the latest state of the cell from the lines: each pod, each slice and each
 * client. It also has, for each node, the backends of its rules and the hosts of its client sidecar.
@@ -2832,7 +2973,8 @@ const unsteady = (ctx) => {
 		for (const node of ctx.cluster.nodes) if ((tracker.hosts.get(node) ?? []).join() !== expected) return `the client sidecar of ${node} has ${(tracker.hosts.get(node) ?? []).join(" ") || "no host"}`;
 	}
 	const nodes = new Set([...tracker.clients.values()].map((client) => client.node));
-	if (nodes.size !== ctx.cluster.nodes.length) return `clients answer on ${nodes.size} nodes`;
+	const clientNodes = ctx.cluster.nodes.length - (cell.trigger.kind === "zone-loss" ? 1 : 0);
+	if (nodes.size !== clientNodes) return `clients answer on ${nodes.size} nodes, not ${clientNodes}`;
 	for (const [pod, client] of tracker.clients) {
 		if (tracker.last - client.ok > 1e3) return `client ${pod} has no answer in the last second`;
 		if (tracker.last - client.failed < 2e3) return `client ${pod} failed in the last 2 s`;
@@ -2849,6 +2991,37 @@ const steady = async (ctx, within) => {
 	}
 	return why;
 };
+/** These taints of the node lifecycle controller mark a node that it does not see as ready. */
+const NODE_LOSS_TAINTS = ["node.kubernetes.io/unreachable", "node.kubernetes.io/not-ready"];
+/**
+* This function tells why a node is not at rest for the node lifecycle controller, or gives null. A
+* ready pod is not sufficient after a zone loss. The controller can still see the node as lost, and
+* then it does not mark the pods again at the next loss. The controller removes its taints only after
+* it sees the node ready.
+*/
+const nodeUnsteady = async (cluster, name) => {
+	const node = JSON.parse(await cluster.kubectl([
+		"get",
+		"node",
+		name,
+		"-o",
+		"json"
+	]));
+	const ready = node.status?.conditions?.find((condition) => condition.type === "Ready")?.status;
+	if (ready !== "True") return `the node ${name} has the Ready condition ${ready ?? "none"}`;
+	const taints = (node.spec?.taints ?? []).map((taint) => taint.key).filter((key) => NODE_LOSS_TAINTS.includes(key));
+	return taints.length === 0 ? null : `the node ${name} has the taints ${taints.join(", ")}`;
+};
+/** This function waits until a node is at rest for the node lifecycle controller. It gives null, or the reason when the node is not at rest in time. */
+const nodeSteady = async (cluster, name, within) => {
+	const end = Date.now() + within;
+	let why = await nodeUnsteady(cluster, name);
+	while (why !== null && Date.now() < end) {
+		await sleep(1e3);
+		why = await nodeUnsteady(cluster, name);
+	}
+	return why;
+};
 /** This function samples the load of the VM for 4 s, and names the lab containers that are not of this run. */
 const sample = async (ctx, ours) => {
 	const second = (await ctx.loader.exec([
@@ -3316,8 +3489,27 @@ const doStep = async (ctx, step, replicas, background) => {
 		case "middlebox":
 			if (step.target !== "servers") throw new Error(`a middlebox takes the target servers, not ${step.target}`);
 			return middleboxServers(ctx, step.idle, step.port, replicas);
+		case "zone": return lab.act(lostNodeOf(cluster), {
+			op: "zone",
+			lost: step.lost
+		});
+		case "netem": {
+			if (step.target !== "redis") throw new Error(`a netem takes the target redis, not ${step.target}`);
+			const at = await where(step.target);
+			const spec = step.delay === null ? ["clear-netem", POD_LINK] : [
+				"netem",
+				POD_LINK,
+				"delay",
+				`${Math.round(valueIn(step.delay, "ms"))}ms`
+			];
+			return lab.inject(at.node, at.pid, spec);
+		}
 	}
 };
+/** This function gives the node of the lost zone: the last worker (`zoneOf`). */
+const lostNodeOf = (cluster) => cluster.nodes.at(-1);
+/** This function puts the replicas of a node first, and keeps the sequence of the others. */
+const lostFirst = (replicas, node) => [...replicas.filter((replica) => replica.node === node), ...replicas.filter((replica) => replica.node !== node)];
 /** This function tells if a run is over: the trigger is done, and the cell is at rest again. */
 const over = (ctx, run, lines) => {
 	if (unsteady(ctx) !== null) return false;
@@ -3370,17 +3562,22 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
 	await probeThePods(ctx);
 	await pollListeners(ctx);
 	await quiet(ctx, ours);
+	const zoneLoss = cell.trigger.kind === "zone-loss";
+	const lost = zoneLoss ? await nodeSteady(cluster, lostNodeOf(cluster), 3e5) : null;
+	if (lost !== null) return empty(`the node of the lost zone was not at rest: ${lost}`, await lab.now());
 	if (await steady(ctx, 3e5) !== null) return empty("the cell was not at rest after the wait for a quiet VM", await lab.now());
-	const replicas = replicasOf(ctx);
+	const replicas = zoneLoss ? lostFirst(replicasOf(ctx), lostNodeOf(cluster)) : replicasOf(ctx);
+	if (zoneLoss && replicas[0]?.node !== lostNodeOf(cluster)) return empty(`no replica is on the node of the lost zone, ${lostNodeOf(cluster)}`, await lab.now());
 	const restarts = new Map(ctx.tracker.servers().map((pod) => [pod.name, pod.cs.reduce((sum, status) => sum + status.rc, 0)]));
 	const target = hasRedis(cell) && [
 		"rst",
 		"blackhole",
 		"freeze-process",
 		"peer-gone-without-close",
-		"rollout"
+		"rollout",
+		"latency"
 	].includes(cell.trigger.kind) ? null : replicas[0];
-	const paused = cell.template === "T07" && cell.trigger.kind === "freeze-process" ? cluster.nodes[1] ?? cluster.nodes[0] : null;
+	const paused = cell.template === "T07" && cell.trigger.kind === "freeze-process" ? cluster.nodes[1] ?? cluster.nodes[0] : zoneLoss ? lostNodeOf(cluster) : null;
 	const cleared = await lab.forgetHandshakes(ctx.clusterIp, 80);
 	lab.take();
 	for (const key of Object.keys(lab.dropped)) delete lab.dropped[key];
@@ -3595,7 +3792,7 @@ const runKindCells = async (cells, options) => {
 	for (const [key, group] of groups) {
 		const partial = /* @__PURE__ */ new Map();
 		for (let env = 0; env < environments; env += 1) {
-			const s = scenario(`lab-${key}-${env}`, group[0].mode, group[0].workers);
+			const s = scenario(`lab-${key}-${env}`, group[0].mode, group[0].workers, void 0, zoned(group[0]));
 			const dir = join(options.work ?? join(tmpdir(), "systems-kind-work"), s.name);
 			const mesh = group[0].mesh;
 			options.log(`cluster ${s.name}: ${s.kubeProxy.mode}, ${s.workers} workers${mesh === void 0 ? "" : `, Istio ${mesh.flavor}`}`);
@@ -3739,7 +3936,8 @@ const estimate = (cell, environments = 2) => {
 		idle: hold + 5,
 		"ztunnel-restart": 40,
 		"certificate-expiry": hold + 20,
-		"cannot-serve": hold + 20
+		"cannot-serve": hold + 20,
+		"zone-loss": hold + 45
 	};
 	const after = observationAfter(cell);
 	const observed = after === null ? 0 : hold + valueIn(after, "s");
@@ -3777,6 +3975,6 @@ const shardOf = (cells, index, count) => {
 	return out;
 };
 //#endregion
-export { ACCESS_LOG_FORMAT, CHANNEL_USE, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, MIDDLEBOX, NETNS_DEFAULTS, NODE_IMAGES, O04, O09, O10, O16, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, SPREAD_LEAD, STAGES, SYSCTL, TCP_STATE, WATCHDOG, ZTUNNEL_PODS, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, certificateArgs, checkRun, cidrOverlap, cidrRange, clientObjects, clusterFilter, compose, createCluster, cyclesOf, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, faultSpan, followsZtunnel, goDuration, hostsOf, installMesh, intakeAfter, intakeOf, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, labSource, latchBand, leafCertificates, learnWindow, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, middleboxCommands, namespaceObject, namespaceOf, nodeNames, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, setZtunnelEnv, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stopOf, stoppingOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, writeResult, ztunnelCertificates, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
+export { ACCESS_LOG_FORMAT, CHANNEL_USE, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, MIDDLEBOX, NETEM_DRAIN, NETNS_DEFAULTS, NODE_IMAGES, NODE_LIFECYCLE, O04, O09, O10, O16, POD_LINK, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, SPREAD_LEAD, STAGES, SYSCTL, TCP_STATE, WATCHDOG, ZONE_KEPT, ZONE_LABEL, ZONE_LOST, ZTUNNEL_PODS, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, certificateArgs, checkRun, cidrOverlap, cidrRange, clientObjects, clusterFilter, compose, createCluster, cyclesOf, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, faultSpan, followsZtunnel, goDuration, hostsOf, installMesh, intakeAfter, intakeOf, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, labSource, latchBand, leafCertificates, learnWindow, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, middleboxCommands, namespaceObject, namespaceOf, nodeLossBand, nodeNames, nodeReturnBand, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, setZtunnelEnv, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stopOf, stoppingOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, writeResult, zoneOf, ztunnelCertificates, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
 
 //# sourceMappingURL=index.js.map