@mark1russell7/kind 0.11.8 → 0.11.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/build/index.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { $ as updatesOf, $t as Declaration, A as TriggerV1, An as nodeNames, At as rulesOf, B as RunCheck, Bt as ztunnelPodsOf, C as KindCellResult, Cn as NODE_IMAGES, Ct as outcomesOf, D as PredictionV1, Dn as goDuration, Dt as requestsOf, E as LoadSample, En as cidrRange, Et as recordsOf, F as KIND_NOISE, Ft as slicesOf, G as judgeKind, Gt as PORT, H as checkRun, Ht as CLIENT, I as KindDecision, It as socketsOf, J as propagationOf, Jt as bandOf, K as observationAfter, Kt as REDIS, L as KindVerdict, Lt as stoppingOf, M as upgrade, Mn as scenarioProblems, Mt as signalsOf, N as upgradeCell, Nt as sliceDropOf, O as RunCheckV1, On as kindConfig, Ot as restartOf, P as Deleted, Pt as sliceEmptyOf, Q as unitOf, Qt as Band, R as LAST_SLACK, Rt as transitionOf, S as KindCellRecordV1, Sn as KubeProxyMode, St as kernelCounts, T as KindVerdictV1, Tn as cidrOverlap, Tt as podsOf, U as deletionsOf, Ut as HEALTH_PORT, V as RunFacts, Vt as ztunnelReplaced, W as factsOf, Wt as LOG_LABEL, X as referenceOf, Xt as secondsOf, Y as propagationSource, Yt as seconds, Z as reusesReset, Zt as showBand, _ as ClusterManifest, _n as WindowPrediction, _t as dropOf, a as STAGES, an as MeshFlavor, at as RecordLine, b as FORMAT_V1, bn as KindConfig, bt as intakeOf, c as readRecord, cn as Prediction, ct as SliceLine, d as calibrationOf, dn as Replica, dt as TunnelLine, en as EdgeProtocol, et as Endpoint, f as cellsTable, fn as ResetReuse, ft as ZtunnelPodLine, g as BandV1, gn as TriggerKind, gt as deletionOf, h as timeSpread, hn as Trigger, ht as byNode, i as quantileOf, in as Mesh, it as PropagationSource, j as judge, jn as scenario, jt as signalOf, k as RunFactsV1, kn as kubeProxyPatch, kt as restartsAt, l as Judged, ln as RedisDeclaration, lt as SocketLine, m as predictionText, mn as Template, mt as byClients, n as medianInterval, nn as KindRun, nt as Line, o as kindCellRecordSchema, on as MeshMode, ot as Request, p as medianText, pn as Stack, pt as backendsAt, q as observationEnd, qt as SERVER, r as middle, rn as KubeletProbe, rt as PodLine, s as kindCellRecordV1Schema, sn as PodSettings, st as RulesLine, t as RELATIVE, tn as KindCell, tt as HBONE_PORT, u as TimeSpread, un as Reference, ut as TCP_STATE, v as EnvironmentManifest, vn as Workload, vt as emptyOf, w as KindCellV1, wn as NodeImage, wt as podsAt, x as KindCellRecord, xn as KindScenario, xt as isTunnelState, y as FORMAT, yn as KUBE_PROXY_MODES, yt as failuresOf, z as Propagated, zt as tunnelsOf } from "./judge-DnB9wEwC.js";
1
+ import { $ as unitOf, $t as secondsOf, A as TriggerV1, An as ZONE_KEPT, At as restartsAt, B as RunCheck, Bn as scenarioProblems, Bt as stoppingOf, C as KindCellResult, Cn as Workload, Ct as kernelCounts, D as PredictionV1, Dn as KubeProxyMode, Dt as recordsOf, E as LoadSample, En as KindScenario, Et as podsOf, F as KIND_NOISE, Fn as goDuration, Ft as sliceEmptyOf, G as factsOf, Gt as CLIENT, H as checkRun, Ht as tunnelsOf, I as KindDecision, In as kindConfig, It as sliceReadyAt, J as observationEnd, Jt as PORT, K as judgeKind, Kt as HEALTH_PORT, L as KindVerdict, Ln as kubeProxyPatch, Lt as sliceReadyOf, M as upgrade, Mn as ZONE_LOST, Mt as signalOf, N as upgradeCell, Nn as cidrOverlap, Nt as signalsOf, O as RunCheckV1, On as NODE_IMAGES, Ot as requestsOf, P as Deleted, Pn as cidrRange, Pt as sliceDropOf, Q as reusesReset, Qt as seconds, R as LAST_SLACK, Rn as nodeNames, Rt as slicesOf, S as KindCellRecordV1, Sn as WindowPrediction, St as isTunnelState, T as KindVerdictV1, Tn as KindConfig, Tt as podsAt, U as cyclesOf, Ut as ztunnelPodsOf, V as RunFacts, Vn as zoneOf, Vt as transitionOf, W as deletionsOf, Wt as ztunnelReplaced, X as propagationSource, Xt as SERVER, Y as propagationOf, Yt as REDIS, Z as referenceOf, Zt as bandOf, _ as ClusterManifest, _n as ResetReuse, _t as deletionOf, a as STAGES, an as Flap, at as PropagationSource, b as FORMAT_V1, bn as Trigger, bt as failuresOf, c as readRecord, cn as KubeletProbe, ct as RulesLine, d as calibrationOf, dn as MeshMode, dt as TCP_STATE, en as showBand, et as updatesOf, f as cellsTable, fn as PodSettings, ft as TunnelLine, g as BandV1, gn as Replica, gt as byNode, h as timeSpread, hn as Reference, ht as byClients, i as quantileOf, in as EdgeProtocol, it as PodLine, j as judge, jn as ZONE_LABEL, jt as rulesOf, k as RunFactsV1, kn as NodeImage, kt as restartOf, l as Judged, ln as Mesh, lt as SliceLine, m as predictionText, mn as RedisDeclaration, mt as backendsAt, n as medianInterval, nn as Cycle, nt as HBONE_PORT, o as kindCellRecordSchema, on as KindCell, ot as RecordLine, p as medianText, pn as Prediction, pt as ZtunnelPodLine, q as observationAfter, qt as LOG_LABEL, r as middle, rn as Declaration, rt as Line, s as kindCellRecordV1Schema, sn as KindRun, st as Request, t as RELATIVE, tn as Band, tt as Endpoint, u as TimeSpread, un as MeshFlavor, ut as SocketLine, v as EnvironmentManifest, vn as Stack, vt as dropOf, w as KindCellV1, wn as KUBE_PROXY_MODES, wt as outcomesOf, x as KindCellRecord, xn as TriggerKind, xt as intakeOf, y as FORMAT, yn as Template, yt as emptyOf, z as Propagated, zn as scenario, zt as socketsOf } from "./judge-DLxpBamt.js";
2
2
  import { Quantity } from "@mark1russell7/quantity";
3
3
  import { CheckFn, Finding, HealthServer, Poll, Scenario, ServiceBuilder, ServiceDescription, Stage, StopWay, Timing, TransportImpl, Verdict } from "@mark1russell7/service";
4
4
  import { BuildImage } from "@mark1russell7/harness";
@@ -169,6 +169,9 @@ type Step = {
169
169
  readonly port: number;
170
170
  } | {
171
171
  readonly do: "certificates";
172
+ } | {
173
+ readonly do: "zone";
174
+ readonly lost: boolean;
172
175
  };
173
176
  /**
174
177
  * A fault plan is the steps of a run:
@@ -177,6 +180,7 @@ type Step = {
177
180
  * - `inject` starts the trigger. The stamp of its first act is the `fault` reference.
178
181
  * - `heal` ends the fault, `hold` after the fault. A trigger that ends by itself has no heal.
179
182
  * - `restore` brings the cell back to its steady state after the run.
183
+ * - `flaps` are the faults after the first one. A plan without them has one fault.
180
184
  */
181
185
  interface FaultPlan {
182
186
  readonly prepare?: readonly Step[] | undefined;
@@ -184,6 +188,8 @@ interface FaultPlan {
184
188
  readonly hold: Quantity | null;
185
189
  readonly heal: readonly Step[];
186
190
  readonly restore: readonly Step[];
191
+ /** These flaps are the faults after the first one (`Trigger.flaps`). Each one does the inject and the heal again. */
192
+ readonly flaps?: readonly Flap[] | undefined;
187
193
  }
188
194
  /** This name is the name of the AuthorizationPolicy of the trigger `authorization-policy`. */
189
195
  export declare const POLICY = "lab-deny";
@@ -261,8 +267,17 @@ export declare const sysctlCommand: (key: string, value: string) => string[];
261
267
  * old flow of the peer. A cell of the middlebox opens no new flow in its gap.
262
268
  */
263
269
  export declare const middleboxCommands: (peers: readonly string[], idle: Quantity | null, port: number) => string[][];
264
- /** This function gives the fault plan of a cell. */
270
+ /**
271
+ * This function gives the fault plan of a cell. A trigger with flaps (`Trigger.flaps`) gives them to
272
+ * its plan: the runner then does the inject and the heal again for each flap. A plan with no hold or
273
+ * no heal cannot flap.
274
+ */
265
275
  export declare const planOf: (cell: KindCell) => FaultPlan;
276
+ /**
277
+ * This function gives the time from the first fault of a trigger to its last heal. It is the hold,
278
+ * then the rest and the hold of each flap. A trigger with no hold gives zero.
279
+ */
280
+ export declare const faultSpan: (trigger: Pick<KindCell["trigger"], "hold" | "flaps">) => Quantity;
266
281
  /** This function tells if a trigger stops a pod of the service with its grace: a deletion, an eviction, a scale-in or a rollout. */
267
282
  export declare const deletes: (cell: KindCell) => boolean;
268
283
  //#endregion
@@ -615,6 +630,64 @@ export declare const restartBand: (latch: Band, liveness: KubeletProbe, preStop:
615
630
  * replica from the rules. Each probe in the stall fails at its timeout, because the loop does not answer it.
616
631
  */
617
632
  export declare const stallRemovalBand: (readiness: KubeletProbe, propagation: Propagation) => Band;
633
+ /**
634
+ * A node lifecycle is the set of timers that removes the pods of a lost node. The same timers give
635
+ * them back after the heal of the node. The lab sets none of them, thus each one is a default of Kubernetes
636
+ * v1.37.0 (file and line in that tag).
637
+ */
638
+ interface NodeLifecycle {
639
+ /**
640
+ * This time is `NodeMonitorGracePeriod`. The node lifecycle controller marks a node Unknown when
641
+ * it saw no renewal of its lease for this time. Then it marks each pod of the node not ready.
642
+ */
643
+ readonly grace: Quantity;
644
+ /** This time is `NodeMonitorPeriod`: the controller checks each node once in it. */
645
+ readonly monitor: Quantity;
646
+ /** This time is the renewal interval of the lease of the kubelet. */
647
+ readonly renew: Quantity;
648
+ /** This share is the jitter of each renewal interval: a renewal comes `renew` to `renew × (1 + jitter)` after the one before it. */
649
+ readonly jitter: number;
650
+ /** This time is the margin of the writes of the controller: the status of the node, then each pod of the node. */
651
+ readonly writes: Quantity;
652
+ /** This time is the PING timeout of the HTTP/2 health check of client-go. A kubelet that gets no PING ACK in it connects again. */
653
+ readonly pingTimeout: Quantity;
654
+ /** This time is the longest first backoff of a reflector of client-go before it watches again. */
655
+ readonly rewatch: Quantity;
656
+ /** This time is the sync period of the status manager of the kubelet: it puts back a pod status that the controller changed. */
657
+ readonly statusSync: Quantity;
658
+ }
659
+ /**
660
+ * These are the defaults of Kubernetes v1.37.0 for a lost node:
661
+ * - `grace` 50 s and `monitor` 5 s: pkg/controller/nodelifecycle/config/v1alpha1/defaults.go, lines
662
+ * 46 and 52. The controller checks the nodes on a sliding period (node_lifecycle_controller.go,
663
+ * lines 506–510). It sets the probe time when it sees a new lease (lines 952–960), and it marks the
664
+ * node Unknown after `grace` (line 962). Then it marks each pod of the node not ready (lines 768–773).
665
+ * - `renew` 10 s: `NodeLeaseDurationSeconds` 40 (pkg/kubelet/apis/config/v1beta1/defaults.go, line 150)
666
+ * times the fraction 0.25 (pkg/kubelet/kubelet.go, lines 237 and 1156).
667
+ * - `jitter` 0.04: the lease controller renews on a sliding jitter of 0.04
668
+ * (staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go, line 153).
669
+ * - `pingTimeout` 15 s: the PING timeout of the HTTP/2 health check, after a read idle timeout of 30 s
670
+ * (staging/src/k8s.io/apimachinery/pkg/util/net/http.go, lines 131–192).
671
+ * - `rewatch` 1.6 s: a first backoff of 800 ms with a jitter of 1.0
672
+ * (staging/src/k8s.io/client-go/tools/cache/reflector.go, lines 62–67).
673
+ * - `statusSync` 10 s: the sync period of the status manager (pkg/kubelet/status/status_manager.go, line 215).
674
+ * - `writes` 0.25 s is a margin of the lab, not a default.
675
+ */
676
+ export declare const NODE_LIFECYCLE: NodeLifecycle;
677
+ /**
678
+ * This function gives the time from the loss of a node to the drop of its pods from the rules of
679
+ * the other nodes. The last renewal of the lease comes up to `renew × (1 + jitter)` before the loss.
680
+ * The controller sees it up to one `monitor` later, and marks the node `grace` after that. Then the
681
+ * writes and the propagation follow. The band does not add the low bound of the propagation.
682
+ */
683
+ export declare const nodeLossBand: (lifecycle: NodeLifecycle, propagation: Propagation) => Band;
684
+ /**
685
+ * This function gives the time from the heal of a lost node until its pods are ready again in the
686
+ * EndpointSlice. The kubelet connects again in one PING timeout at the most: its old connection
687
+ * waits behind the backoff of TCP. Its reflectors watch again after their first backoff. Then its
688
+ * status manager puts back the pod status in one sync period at the most.
689
+ */
690
+ export declare const nodeReturnBand: (lifecycle: NodeLifecycle) => Band;
618
691
  /** This function gives the cap of a stage in a declaration, or zero when the declaration does not name the stage. */
619
692
  export declare const capOf: (declaration: Declaration, stage: Stage) => Quantity;
620
693
  /**
@@ -694,6 +767,7 @@ type Verdicts = Readonly<Record<string, Verdict>>;
694
767
  * - In T07 a pause of kube-proxy comes before a deletion. In T18 the pause is on Redis.
695
768
  * - The crash of the lab is SIGKILL from the node. Like a crash, it has no preStop hook and no drain.
696
769
  * - The other faults are on a link, on Redis or on the process. The kubelet can restart a pod after them.
770
+ * A zone loss freezes the pods of its node and deletes none of them in its hold.
697
771
  */
698
772
  export declare const stopOf: (cell: KindCell) => StopWay | null;
699
773
  /**
@@ -800,5 +874,5 @@ export declare const toolPath: (tool: Tool, key?: string, cache?: string) => str
800
874
  */
801
875
  export declare const ensureTool: (tool: Tool, key?: string, cache?: string) => Promise<string>;
802
876
  //#endregion
803
- export { type Band, type BandV1, CLIENT, type Cluster, type ClusterHosts, type ClusterManifest, type Credentials, DRAIN_RULES, type Declaration, type Deleted, type DeletionWindow, type EdgeProtocol, type EdsSpec, Endpoint, type EnvironmentManifest, FORMAT, FORMAT_V1, type FaultPlan, HBONE_PORT, HEALTH_PORT, type Judged, KIND_NOISE, KUBE_PROXY_MODES, type KindCell, type KindCellRecord, type KindCellRecordV1, type KindCellResult, type KindCellV1, type KindConfig, type KindDecision, type KindLab, type KindRun, type KindScenario, type KindVerdict, type KindVerdictV1, type KubeProxyMode, type KubeletProbe, LAST_SLACK, LOG_LABEL, type LdsSpec, type LeafCertificate, type LearnWindow, Line, type ListenerCounts, type LoadSample, type Mesh, type MeshFlavor, type MeshImages, type MeshMode, NODE_IMAGES, type NodeImage, O04, O09, O10, O16, PORT, type Parts, PodLine, type PodSettings, type Prediction, type PredictionV1, type Propagated, type Propagation, PropagationSource, REDIS, RELATIVE, RecordLine, type RedisDeclaration, type Reference, type Replica, Request, type ResetReuse, RulesLine, type RunCheck, type RunCheckV1, type RunFacts, type RunFactsV1, type RunOptions, SERVER, STAGES, type ServiceRule, SliceLine, SocketLine, type Stack, type Stamp, type Step, TCP_STATE, type Target, type Template, type TimeSpread, type Tool, type Trigger, type TriggerKind, type TriggerV1, TunnelLine, type Verdicts, type WindowPrediction, type Workload, ZtunnelPodLine, backendsAt, bandOf, byClients, byNode, calibrationOf, cellsTable, checkRun, cidrOverlap, cidrRange, deletionOf, deletionsOf, dropOf, emptyOf, factsOf, failuresOf, goDuration, intakeOf, isTunnelState, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, medianInterval, medianText, middle, nodeNames, observationAfter, observationEnd, outcomesOf, podsAt, podsOf, predictionText, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, referenceOf, requestsOf, restartOf, restartsAt, reusesReset, rulesOf, scenario, scenarioProblems, seconds, secondsOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, socketsOf, stoppingOf, timeSpread, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, ztunnelPodsOf, ztunnelReplaced };
877
+ export { type Band, type BandV1, CLIENT, type Cluster, type ClusterHosts, type ClusterManifest, type Credentials, type Cycle, DRAIN_RULES, type Declaration, type Deleted, type DeletionWindow, type EdgeProtocol, type EdsSpec, Endpoint, type EnvironmentManifest, FORMAT, FORMAT_V1, type FaultPlan, type Flap, HBONE_PORT, HEALTH_PORT, type Judged, KIND_NOISE, KUBE_PROXY_MODES, type KindCell, type KindCellRecord, type KindCellRecordV1, type KindCellResult, type KindCellV1, type KindConfig, type KindDecision, type KindLab, type KindRun, type KindScenario, type KindVerdict, type KindVerdictV1, type KubeProxyMode, type KubeletProbe, LAST_SLACK, LOG_LABEL, type LdsSpec, type LeafCertificate, type LearnWindow, Line, type ListenerCounts, type LoadSample, type Mesh, type MeshFlavor, type MeshImages, type MeshMode, NODE_IMAGES, type NodeImage, type NodeLifecycle, O04, O09, O10, O16, PORT, type Parts, PodLine, type PodSettings, type Prediction, type PredictionV1, type Propagated, type Propagation, PropagationSource, REDIS, RELATIVE, RecordLine, type RedisDeclaration, type Reference, type Replica, Request, type ResetReuse, RulesLine, type RunCheck, type RunCheckV1, type RunFacts, type RunFactsV1, type RunOptions, SERVER, STAGES, type ServiceRule, SliceLine, SocketLine, type Stack, type Stamp, type Step, TCP_STATE, type Target, type Template, type TimeSpread, type Tool, type Trigger, type TriggerKind, type TriggerV1, TunnelLine, type Verdicts, type WindowPrediction, type Workload, ZONE_KEPT, ZONE_LABEL, ZONE_LOST, ZtunnelPodLine, backendsAt, bandOf, byClients, byNode, calibrationOf, cellsTable, checkRun, cidrOverlap, cidrRange, cyclesOf, deletionOf, deletionsOf, dropOf, emptyOf, factsOf, failuresOf, goDuration, intakeOf, isTunnelState, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, medianInterval, medianText, middle, nodeNames, observationAfter, observationEnd, outcomesOf, podsAt, podsOf, predictionText, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, referenceOf, requestsOf, restartOf, restartsAt, reusesReset, rulesOf, scenario, scenarioProblems, seconds, secondsOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stoppingOf, timeSpread, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, zoneOf, ztunnelPodsOf, ztunnelReplaced };
804
878
  //# sourceMappingURL=index.d.ts.map
package/build/index.js CHANGED
@@ -1,4 +1,4 @@
1
- import { $ as signalOf, A as medianInterval, B as emptyOf, C as propagationOf, D as unitOf, E as reusesReset, F as backendsAt, G as outcomesOf, H as intakeOf, I as byClients, J as recordsOf, K as podsAt, L as byNode, M as quantileOf, N as HBONE_PORT, O as updatesOf, P as TCP_STATE, Q as rulesOf, R as deletionOf, S as observationEnd, T as referenceOf, U as isTunnelState, V as failuresOf, W as kernelCounts, X as restartOf, Y as requestsOf, Z as restartsAt, _ as checkRun, _t as seconds, a as calibrationOf, at as stoppingOf, b as judgeKind, c as predictionText, ct as ztunnelPodsOf, d as FORMAT_V1, dt as HEALTH_PORT, et as signalsOf, f as judge, ft as LOG_LABEL, g as LAST_SLACK, gt as bandOf, h as KIND_NOISE, ht as SERVER, i as readRecord, it as socketsOf, j as middle, k as RELATIVE, l as timeSpread, lt as ztunnelReplaced, m as upgradeCell, mt as REDIS, n as kindCellRecordSchema, nt as sliceEmptyOf, o as cellsTable, ot as transitionOf, p as upgrade, pt as PORT, q as podsOf, r as kindCellRecordV1Schema, rt as slicesOf, s as medianText, st as tunnelsOf, t as STAGES, tt as sliceDropOf, u as FORMAT, ut as CLIENT, v as deletionsOf, vt as secondsOf, w as propagationSource, x as observationAfter, y as factsOf, yt as showBand, z as dropOf } from "./judge-Cs91dOGO.js";
1
+ import { $ as rulesOf, A as RELATIVE, B as dropOf, C as observationEnd, D as reusesReset, E as referenceOf, F as TCP_STATE, G as kernelCounts, H as failuresOf, I as backendsAt, J as podsOf, K as outcomesOf, L as byClients, M as middle, N as quantileOf, O as unitOf, P as HBONE_PORT, Q as restartsAt, R as byNode, S as observationAfter, St as showBand, T as propagationSource, U as intakeOf, V as emptyOf, W as isTunnelState, X as requestsOf, Y as recordsOf, Z as restartOf, _ as checkRun, _t as REDIS, a as calibrationOf, at as sliceReadyOf, b as factsOf, bt as seconds, c as predictionText, ct as stoppingOf, d as FORMAT_V1, dt as ztunnelPodsOf, et as signalOf, f as judge, ft as ztunnelReplaced, g as LAST_SLACK, gt as PORT, h as KIND_NOISE, ht as LOG_LABEL, i as readRecord, it as sliceReadyAt, j as medianInterval, k as updatesOf, l as timeSpread, lt as transitionOf, m as upgradeCell, mt as HEALTH_PORT, n as kindCellRecordSchema, nt as sliceDropOf, o as cellsTable, ot as slicesOf, p as upgrade, pt as CLIENT, q as podsAt, r as kindCellRecordV1Schema, rt as sliceEmptyOf, s as medianText, st as socketsOf, t as STAGES, tt as signalsOf, u as FORMAT, ut as tunnelsOf, v as cyclesOf, vt as SERVER, w as propagationOf, x as judgeKind, xt as secondsOf, y as deletionsOf, yt as bandOf, z as deletionOf } from "./judge-BIXZEX__.js";
2
2
  import { valueIn } from "@mark1russell7/quantity";
3
3
  import { execFile, execFileSync } from "node:child_process";
4
4
  import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
@@ -92,8 +92,16 @@ const NODE_IMAGES = {
92
92
  image: "kindest/node:v1.36.4@sha256:099e049362a1526b2db71494e1947aae99bd16290d7c895f2b7ea312e3cbfaed"
93
93
  }
94
94
  };
95
+ /** This label gives the zone of a node, as Kubernetes names it. */
96
+ const ZONE_LABEL = "topology.kubernetes.io/zone";
97
+ /** This zone holds the control plane and each worker but the last. A zone loss keeps it. */
98
+ const ZONE_KEPT = "zone-a";
99
+ /** This zone holds only the last worker. A zone loss loses it. */
100
+ const ZONE_LOST = "zone-b";
101
+ /** This function gives the zone of the node at the position `at` of `nodeNames`, in a cluster of `count` nodes. */
102
+ const zoneOf = (at, count) => at === count - 1 && at > 0 ? ZONE_LOST : ZONE_KEPT;
95
103
  /** This function gives a scenario with the defaults of Kubernetes for kube-proxy, as explicit values. */
96
- const scenario = (name, mode, workers, kubernetes = "1.37") => ({
104
+ const scenario = (name, mode, workers, kubernetes = "1.37", zoned = false) => ({
97
105
  name,
98
106
  kubernetes,
99
107
  workers,
@@ -110,7 +118,8 @@ const scenario = (name, mode, workers, kubernetes = "1.37") => ({
110
118
  },
111
119
  cni: "kindnet",
112
120
  podSubnet: "10.244.0.0/16",
113
- serviceSubnet: "10.96.0.0/16"
121
+ serviceSubnet: "10.96.0.0/16",
122
+ ...zoned ? { zoned } : {}
114
123
  });
115
124
  const NAME = /^[a-z0-9]([a-z0-9-]{0,30}[a-z0-9])?$/;
116
125
  /** This function gives the problems of a scenario: a name that kind refuses, no image for the version, or a count out of range. */
@@ -155,10 +164,13 @@ const kindConfig = (s) => {
155
164
  podSubnet: s.podSubnet,
156
165
  serviceSubnet: s.serviceSubnet
157
166
  },
158
- nodes: nodeNames(s).map((node, at) => ({
167
+ nodes: nodeNames(s).map((node, at, all) => ({
159
168
  role: at === 0 ? "control-plane" : "worker",
160
169
  image,
161
- labels: { "lab.systems/node": node.slice(s.name.length + 1) }
170
+ labels: {
171
+ "lab.systems/node": node.slice(s.name.length + 1),
172
+ ...s.zoned === true ? { [ZONE_LABEL]: zoneOf(at, all.length) } : {}
173
+ }
162
174
  })),
163
175
  kubeadmConfigPatches: [kubeProxyPatch(s)]
164
176
  };
@@ -854,8 +866,36 @@ const middleboxCommands = (peers, idle, port) => {
854
866
  * is a rule in the target replica only, with the client pods as peers.
855
867
  */
856
868
  const linkOf = (cell) => cell.trigger.partition === true ? "server" : cell.template === "T15" ? "servers" : "redis";
857
- /** This function gives the fault plan of a cell. */
869
+ /**
870
+ * This function gives the fault plan of a cell. A trigger with flaps (`Trigger.flaps`) gives them to
871
+ * its plan: the runner then does the inject and the heal again for each flap. A plan with no hold or
872
+ * no heal cannot flap.
873
+ */
858
874
  const planOf = (cell) => {
875
+ const plan = stepsOf(cell);
876
+ const flaps = cell.trigger.flaps;
877
+ if (flaps === void 0 || flaps.length === 0) return plan;
878
+ if (plan.hold === null || plan.heal.length === 0) throw new Error(`the trigger ${cell.trigger.kind} has no hold and heal, thus it cannot flap`);
879
+ for (const flap of flaps) if (valueIn(flap.rest, "ms") <= 0 || valueIn(flap.hold, "ms") <= 0) throw new Error("each flap has a rest and a hold above zero");
880
+ return {
881
+ ...plan,
882
+ flaps
883
+ };
884
+ };
885
+ /**
886
+ * This function gives the time from the first fault of a trigger to its last heal. It is the hold,
887
+ * then the rest and the hold of each flap. A trigger with no hold gives zero.
888
+ */
889
+ const faultSpan = (trigger) => {
890
+ const hold = trigger.hold === void 0 ? 0 : valueIn(trigger.hold, "s");
891
+ const flaps = (trigger.flaps ?? []).reduce((sum, flap) => sum + valueIn(flap.rest, "s") + valueIn(flap.hold, "s"), 0);
892
+ return {
893
+ value: Math.round((hold + flaps) * 1e6) / 1e6,
894
+ unit: "s"
895
+ };
896
+ };
897
+ /** This function gives the steps of the fault plan of a cell, with no flaps. */
898
+ const stepsOf = (cell) => {
859
899
  const ns = ["-n", "{namespace}"];
860
900
  const trigger = cell.trigger;
861
901
  const replicas = cell.pod.replicas;
@@ -1306,6 +1346,29 @@ const planOf = (cell) => {
1306
1346
  on: false
1307
1347
  }]
1308
1348
  };
1349
+ case "zone-loss": return {
1350
+ prepare: [{
1351
+ do: "signal",
1352
+ target: "clients",
1353
+ signal: "SIGHUP"
1354
+ }, {
1355
+ do: "wait",
1356
+ time: SPREAD_LEAD
1357
+ }],
1358
+ inject: [{
1359
+ do: "zone",
1360
+ lost: true
1361
+ }],
1362
+ hold: needs(trigger.hold, "a hold, the time that the zone is lost"),
1363
+ heal: [{
1364
+ do: "zone",
1365
+ lost: false
1366
+ }],
1367
+ restore: [{
1368
+ do: "zone",
1369
+ lost: false
1370
+ }]
1371
+ };
1309
1372
  }
1310
1373
  };
1311
1374
  /** This function tells if a trigger stops a pod of the service with its grace: a deletion, an eviction, a scale-in or a rollout. */
@@ -2422,7 +2485,12 @@ const serverObjects = (cell, namespace, image, version = 1) => {
2422
2485
  topologyKey: "kubernetes.io/hostname",
2423
2486
  whenUnsatisfiable: "ScheduleAnyway",
2424
2487
  labelSelector: { matchLabels: { app: "lab" } }
2425
- }],
2488
+ }, ...cell.trigger.kind === "zone-loss" ? [{
2489
+ maxSkew: 1,
2490
+ topologyKey: ZONE_LABEL,
2491
+ whenUnsatisfiable: "DoNotSchedule",
2492
+ labelSelector: { matchLabels: { app: "lab" } }
2493
+ }] : []],
2426
2494
  containers: [{
2427
2495
  name: "app",
2428
2496
  image,
@@ -2464,6 +2532,11 @@ const clientObjects = (cell, namespace, image, target) => [{
2464
2532
  operator: "Exists",
2465
2533
  effect: "NoSchedule"
2466
2534
  }],
2535
+ ...cell.trigger.kind === "zone-loss" ? { affinity: { nodeAffinity: { requiredDuringSchedulingIgnoredDuringExecution: { nodeSelectorTerms: [{ matchExpressions: [{
2536
+ key: ZONE_LABEL,
2537
+ operator: "NotIn",
2538
+ values: [ZONE_LOST]
2539
+ }] }] } } } } : {},
2467
2540
  containers: [{
2468
2541
  name: CLIENT,
2469
2542
  image,
@@ -2576,6 +2649,50 @@ const stallRemovalBand = (readiness, propagation) => {
2576
2649
  const spread = secondsOf(propagation);
2577
2650
  return band(kubelet.lo + spread.lo, kubelet.hi + spread.hi);
2578
2651
  };
2652
+ /**
2653
+ * These are the defaults of Kubernetes v1.37.0 for a lost node:
2654
+ * - `grace` 50 s and `monitor` 5 s: pkg/controller/nodelifecycle/config/v1alpha1/defaults.go, lines
2655
+ * 46 and 52. The controller checks the nodes on a sliding period (node_lifecycle_controller.go,
2656
+ * lines 506–510). It sets the probe time when it sees a new lease (lines 952–960), and it marks the
2657
+ * node Unknown after `grace` (line 962). Then it marks each pod of the node not ready (lines 768–773).
2658
+ * - `renew` 10 s: `NodeLeaseDurationSeconds` 40 (pkg/kubelet/apis/config/v1beta1/defaults.go, line 150)
2659
+ * times the fraction 0.25 (pkg/kubelet/kubelet.go, lines 237 and 1156).
2660
+ * - `jitter` 0.04: the lease controller renews on a sliding jitter of 0.04
2661
+ * (staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go, line 153).
2662
+ * - `pingTimeout` 15 s: the PING timeout of the HTTP/2 health check, after a read idle timeout of 30 s
2663
+ * (staging/src/k8s.io/apimachinery/pkg/util/net/http.go, lines 131–192).
2664
+ * - `rewatch` 1.6 s: a first backoff of 800 ms with a jitter of 1.0
2665
+ * (staging/src/k8s.io/client-go/tools/cache/reflector.go, lines 62–67).
2666
+ * - `statusSync` 10 s: the sync period of the status manager (pkg/kubelet/status/status_manager.go, line 215).
2667
+ * - `writes` 0.25 s is a margin of the lab, not a default.
2668
+ */
2669
+ const NODE_LIFECYCLE = {
2670
+ grace: seconds(50),
2671
+ monitor: seconds(5),
2672
+ renew: seconds(10),
2673
+ jitter: .04,
2674
+ writes: seconds(.25),
2675
+ pingTimeout: seconds(15),
2676
+ rewatch: seconds(1.6),
2677
+ statusSync: seconds(10)
2678
+ };
2679
+ /**
2680
+ * This function gives the time from the loss of a node to the drop of its pods from the rules of
2681
+ * the other nodes. The last renewal of the lease comes up to `renew × (1 + jitter)` before the loss.
2682
+ * The controller sees it up to one `monitor` later, and marks the node `grace` after that. Then the
2683
+ * writes and the propagation follow. The band does not add the low bound of the propagation.
2684
+ */
2685
+ const nodeLossBand = (lifecycle, propagation) => {
2686
+ const last = s$1(lifecycle.renew) * (1 + lifecycle.jitter);
2687
+ return band(s$1(lifecycle.grace) - last, s$1(lifecycle.grace) + s$1(lifecycle.monitor) + s$1(lifecycle.writes) + s$1(propagation.hi));
2688
+ };
2689
+ /**
2690
+ * This function gives the time from the heal of a lost node until its pods are ready again in the
2691
+ * EndpointSlice. The kubelet connects again in one PING timeout at the most: its old connection
2692
+ * waits behind the backoff of TCP. Its reflectors watch again after their first backoff. Then its
2693
+ * status manager puts back the pod status in one sync period at the most.
2694
+ */
2695
+ const nodeReturnBand = (lifecycle) => band(0, s$1(lifecycle.pingTimeout) + s$1(lifecycle.rewatch) + s$1(lifecycle.statusSync) + s$1(lifecycle.writes));
2579
2696
  /** This function gives the cap of a stage in a declaration, or zero when the declaration does not name the stage. */
2580
2697
  const capOf = (declaration, stage) => {
2581
2698
  const found = declaration.phases.find(([name]) => name === stage);
@@ -2681,6 +2798,7 @@ const scenarioOf = (cell, propagation) => ({
2681
2798
  * - In T07 a pause of kube-proxy comes before a deletion. In T18 the pause is on Redis.
2682
2799
  * - The crash of the lab is SIGKILL from the node. Like a crash, it has no preStop hook and no drain.
2683
2800
  * - The other faults are on a link, on Redis or on the process. The kubelet can restart a pod after them.
2801
+ * A zone loss freezes the pods of its node and deletes none of them in its hold.
2684
2802
  */
2685
2803
  const stopOf = (cell) => {
2686
2804
  switch (cell.trigger.kind) {
@@ -2701,7 +2819,8 @@ const stopOf = (cell) => {
2701
2819
  case "idle":
2702
2820
  case "ztunnel-restart":
2703
2821
  case "certificate-expiry":
2704
- case "cannot-serve": return null;
2822
+ case "cannot-serve":
2823
+ case "zone-loss": return null;
2705
2824
  }
2706
2825
  };
2707
2826
  /**
@@ -2740,8 +2859,13 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
2740
2859
  const slug = (name) => name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
2741
2860
  /** This function tells if a cell has Redis: only a declaration of the service can have it. */
2742
2861
  const hasRedis = (cell) => cell.declaration !== null && cell.declaration.redis !== null;
2743
- /** This function gives the key of the cluster of a cell: its kube-proxy mode, its node count, and the flavor of its mesh. */
2744
- const clusterKey = (cell) => cell.mesh === void 0 ? `${cell.mode}-${cell.workers}` : `${cell.mode}-${cell.workers}-${cell.mesh.flavor}`;
2862
+ /**
2863
+ * This function gives the key of the cluster of a cell: its kube-proxy mode, its node count, and the
2864
+ * flavor of its mesh. A zone loss has clusters of its own, with a zone on each node.
2865
+ */
2866
+ const clusterKey = (cell) => cell.mesh !== void 0 ? `${cell.mode}-${cell.workers}-${cell.mesh.flavor}` : zoned(cell) ? `${cell.mode}-${cell.workers}-zones` : `${cell.mode}-${cell.workers}`;
2867
+ /** This function tells if a cell needs a cluster with zones: a zone loss. */
2868
+ const zoned = (cell) => cell.trigger.kind === "zone-loss";
2745
2869
  /**
2746
2870
  * This tracker keeps the latest state of the cell from the lines: each pod, each slice and each
2747
2871
  * client. It also has, for each node, the backends of its rules and the hosts of its client sidecar.
@@ -2804,7 +2928,8 @@ const unsteady = (ctx) => {
2804
2928
  for (const node of ctx.cluster.nodes) if ((tracker.hosts.get(node) ?? []).join() !== expected) return `the client sidecar of ${node} has ${(tracker.hosts.get(node) ?? []).join(" ") || "no host"}`;
2805
2929
  }
2806
2930
  const nodes = new Set([...tracker.clients.values()].map((client) => client.node));
2807
- if (nodes.size !== ctx.cluster.nodes.length) return `clients answer on ${nodes.size} nodes`;
2931
+ const clientNodes = ctx.cluster.nodes.length - (cell.trigger.kind === "zone-loss" ? 1 : 0);
2932
+ if (nodes.size !== clientNodes) return `clients answer on ${nodes.size} nodes, not ${clientNodes}`;
2808
2933
  for (const [pod, client] of tracker.clients) {
2809
2934
  if (tracker.last - client.ok > 1e3) return `client ${pod} has no answer in the last second`;
2810
2935
  if (tracker.last - client.failed < 2e3) return `client ${pod} failed in the last 2 s`;
@@ -2821,6 +2946,37 @@ const steady = async (ctx, within) => {
2821
2946
  }
2822
2947
  return why;
2823
2948
  };
2949
+ /** These taints of the node lifecycle controller mark a node that it does not see as ready. */
2950
+ const NODE_LOSS_TAINTS = ["node.kubernetes.io/unreachable", "node.kubernetes.io/not-ready"];
2951
+ /**
2952
+ * This function tells why a node is not at rest for the node lifecycle controller, or gives null. A
2953
+ * ready pod is not sufficient after a zone loss. The controller can still see the node as lost, and
2954
+ * then it does not mark the pods again at the next loss. The controller removes its taints only after
2955
+ * it sees the node ready.
2956
+ */
2957
+ const nodeUnsteady = async (cluster, name) => {
2958
+ const node = JSON.parse(await cluster.kubectl([
2959
+ "get",
2960
+ "node",
2961
+ name,
2962
+ "-o",
2963
+ "json"
2964
+ ]));
2965
+ const ready = node.status?.conditions?.find((condition) => condition.type === "Ready")?.status;
2966
+ if (ready !== "True") return `the node ${name} has the Ready condition ${ready ?? "none"}`;
2967
+ const taints = (node.spec?.taints ?? []).map((taint) => taint.key).filter((key) => NODE_LOSS_TAINTS.includes(key));
2968
+ return taints.length === 0 ? null : `the node ${name} has the taints ${taints.join(", ")}`;
2969
+ };
2970
+ /** This function waits until a node is at rest for the node lifecycle controller. It gives null, or the reason when the node is not at rest in time. */
2971
+ const nodeSteady = async (cluster, name, within) => {
2972
+ const end = Date.now() + within;
2973
+ let why = await nodeUnsteady(cluster, name);
2974
+ while (why !== null && Date.now() < end) {
2975
+ await sleep(1e3);
2976
+ why = await nodeUnsteady(cluster, name);
2977
+ }
2978
+ return why;
2979
+ };
2824
2980
  /** This function samples the load of the VM for 4 s, and names the lab containers that are not of this run. */
2825
2981
  const sample = async (ctx, ours) => {
2826
2982
  const second = (await ctx.loader.exec([
@@ -3288,8 +3444,16 @@ const doStep = async (ctx, step, replicas, background) => {
3288
3444
  case "middlebox":
3289
3445
  if (step.target !== "servers") throw new Error(`a middlebox takes the target servers, not ${step.target}`);
3290
3446
  return middleboxServers(ctx, step.idle, step.port, replicas);
3447
+ case "zone": return lab.act(lostNodeOf(cluster), {
3448
+ op: "zone",
3449
+ lost: step.lost
3450
+ });
3291
3451
  }
3292
3452
  };
3453
+ /** This function gives the node of the lost zone: the last worker (`zoneOf`). */
3454
+ const lostNodeOf = (cluster) => cluster.nodes.at(-1);
3455
+ /** This function puts the replicas of a node first, and keeps the sequence of the others. */
3456
+ const lostFirst = (replicas, node) => [...replicas.filter((replica) => replica.node === node), ...replicas.filter((replica) => replica.node !== node)];
3293
3457
  /** This function tells if a run is over: the trigger is done, and the cell is at rest again. */
3294
3458
  const over = (ctx, run, lines) => {
3295
3459
  if (unsteady(ctx) !== null) return false;
@@ -3342,8 +3506,12 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
3342
3506
  await probeThePods(ctx);
3343
3507
  await pollListeners(ctx);
3344
3508
  await quiet(ctx, ours);
3509
+ const zoneLoss = cell.trigger.kind === "zone-loss";
3510
+ const lost = zoneLoss ? await nodeSteady(cluster, lostNodeOf(cluster), 3e5) : null;
3511
+ if (lost !== null) return empty(`the node of the lost zone was not at rest: ${lost}`, await lab.now());
3345
3512
  if (await steady(ctx, 3e5) !== null) return empty("the cell was not at rest after the wait for a quiet VM", await lab.now());
3346
- const replicas = replicasOf(ctx);
3513
+ const replicas = zoneLoss ? lostFirst(replicasOf(ctx), lostNodeOf(cluster)) : replicasOf(ctx);
3514
+ if (zoneLoss && replicas[0]?.node !== lostNodeOf(cluster)) return empty(`no replica is on the node of the lost zone, ${lostNodeOf(cluster)}`, await lab.now());
3347
3515
  const restarts = new Map(ctx.tracker.servers().map((pod) => [pod.name, pod.cs.reduce((sum, status) => sum + status.rc, 0)]));
3348
3516
  const target = hasRedis(cell) && [
3349
3517
  "rst",
@@ -3352,7 +3520,7 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
3352
3520
  "peer-gone-without-close",
3353
3521
  "rollout"
3354
3522
  ].includes(cell.trigger.kind) ? null : replicas[0];
3355
- const paused = cell.template === "T07" && cell.trigger.kind === "freeze-process" ? cluster.nodes[1] ?? cluster.nodes[0] : null;
3523
+ const paused = cell.template === "T07" && cell.trigger.kind === "freeze-process" ? cluster.nodes[1] ?? cluster.nodes[0] : zoneLoss ? lostNodeOf(cluster) : null;
3356
3524
  const cleared = await lab.forgetHandshakes(ctx.clusterIp, 80);
3357
3525
  lab.take();
3358
3526
  for (const key of Object.keys(lab.dropped)) delete lab.dropped[key];
@@ -3377,16 +3545,46 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
3377
3545
  if (stamp !== null && fault === null) fault = stamp.t0;
3378
3546
  }
3379
3547
  let heal = null;
3380
- if (plan.hold !== null) {
3548
+ const healAfter = async (from, hold) => {
3381
3549
  const now = await lab.now();
3382
- await sleep(Math.max(0, valueIn(plan.hold, "ms") - (now - (fault ?? now))));
3550
+ await sleep(Math.max(0, valueIn(hold, "ms") - (now - (from ?? now))));
3551
+ let stamped = null;
3383
3552
  for (const step of plan.heal) try {
3384
3553
  const stamp = await doStep(ctx, step, replicas, background);
3385
- if (stamp !== null && heal === null) heal = stamp.t0;
3554
+ if (stamp !== null && stamped === null) stamped = stamp.t0;
3386
3555
  } catch (error) {
3387
3556
  notes.push(`the heal ${step.do} failed: ${String(error)}`);
3388
3557
  }
3558
+ return stamped;
3559
+ };
3560
+ const cycles = [];
3561
+ if (plan.hold !== null) {
3562
+ heal = await healAfter(fault, plan.hold);
3563
+ if (fault !== null && heal !== null) cycles.push({
3564
+ fault,
3565
+ heal
3566
+ });
3567
+ for (const flap of plan.flaps ?? []) {
3568
+ const now = await lab.now();
3569
+ await sleep(Math.max(0, valueIn(flap.rest, "ms") - (now - (heal ?? now))));
3570
+ let again = null;
3571
+ for (const step of plan.inject) try {
3572
+ const stamp = await doStep(ctx, step, replicas, background);
3573
+ if (stamp !== null && again === null) again = stamp.t0;
3574
+ } catch (error) {
3575
+ notes.push(`the inject ${step.do} of a flap failed: ${String(error)}`);
3576
+ }
3577
+ const healed = await healAfter(again, flap.hold);
3578
+ if (again !== null && healed !== null) cycles.push({
3579
+ fault: again,
3580
+ heal: healed
3581
+ });
3582
+ heal = healed ?? heal;
3583
+ }
3389
3584
  }
3585
+ const flapped = (plan.flaps ?? []).length > 0;
3586
+ if (flapped && cycles.length !== (plan.flaps ?? []).length + 1) notes.push(`the run has ${cycles.length} cycles with stamps of the ${(plan.flaps ?? []).length + 1} of the trigger`);
3587
+ const flapping = flapped ? { cycles } : {};
3390
3588
  const settled = await Promise.allSettled(background);
3391
3589
  for (const each of settled) if (each.status === "fulfilled" && typeof each.value === "object" && each.value !== null && "t1" in each.value) resume = each.value.t1;
3392
3590
  const limit = Date.now() + 24e4 + valueIn(cell.pod.grace, "ms") + valueIn(cell.pod.preStop, "ms");
@@ -3401,7 +3599,8 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
3401
3599
  acts: {
3402
3600
  fault,
3403
3601
  heal,
3404
- resume
3602
+ resume,
3603
+ ...flapping
3405
3604
  },
3406
3605
  target: stoppedOf(cell, replicas, target, lab.lines),
3407
3606
  replicas,
@@ -3450,7 +3649,8 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
3450
3649
  acts: {
3451
3650
  fault,
3452
3651
  heal,
3453
- resume
3652
+ resume,
3653
+ ...flapping
3454
3654
  },
3455
3655
  target: stopped,
3456
3656
  paused,
@@ -3535,7 +3735,7 @@ const runKindCells = async (cells, options) => {
3535
3735
  for (const [key, group] of groups) {
3536
3736
  const partial = /* @__PURE__ */ new Map();
3537
3737
  for (let env = 0; env < environments; env += 1) {
3538
- const s = scenario(`lab-${key}-${env}`, group[0].mode, group[0].workers);
3738
+ const s = scenario(`lab-${key}-${env}`, group[0].mode, group[0].workers, void 0, zoned(group[0]));
3539
3739
  const dir = join(options.work ?? join(tmpdir(), "systems-kind-work"), s.name);
3540
3740
  const mesh = group[0].mesh;
3541
3741
  options.log(`cluster ${s.name}: ${s.kubeProxy.mode}, ${s.workers} workers${mesh === void 0 ? "" : `, Istio ${mesh.flavor}`}`);
@@ -3668,7 +3868,7 @@ const runKindCells = async (cells, options) => {
3668
3868
  };
3669
3869
  /** This function gives the estimate of a cell in seconds, for the arithmetic of the shards. */
3670
3870
  const estimate = (cell, environments = 2) => {
3671
- const hold = cell.trigger.hold === void 0 ? 0 : valueIn(cell.trigger.hold, "s");
3871
+ const hold = valueIn(faultSpan(cell.trigger), "s");
3672
3872
  const trigger = {
3673
3873
  "liveness-restart": 100,
3674
3874
  crash: 30,
@@ -3679,7 +3879,8 @@ const estimate = (cell, environments = 2) => {
3679
3879
  idle: hold + 5,
3680
3880
  "ztunnel-restart": 40,
3681
3881
  "certificate-expiry": hold + 20,
3682
- "cannot-serve": hold + 20
3882
+ "cannot-serve": hold + 20,
3883
+ "zone-loss": hold + 45
3683
3884
  };
3684
3885
  const after = observationAfter(cell);
3685
3886
  const observed = after === null ? 0 : hold + valueIn(after, "s");
@@ -3717,6 +3918,6 @@ const shardOf = (cells, index, count) => {
3717
3918
  return out;
3718
3919
  };
3719
3920
  //#endregion
3720
- export { ACCESS_LOG_FORMAT, CHANNEL_USE, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, MIDDLEBOX, NETNS_DEFAULTS, NODE_IMAGES, O04, O09, O10, O16, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, SPREAD_LEAD, STAGES, SYSCTL, TCP_STATE, WATCHDOG, ZTUNNEL_PODS, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, certificateArgs, checkRun, cidrOverlap, cidrRange, clientObjects, clusterFilter, compose, createCluster, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, followsZtunnel, goDuration, hostsOf, installMesh, intakeAfter, intakeOf, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, labSource, latchBand, leafCertificates, learnWindow, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, middleboxCommands, namespaceObject, namespaceOf, nodeNames, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, setZtunnelEnv, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stopOf, stoppingOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, writeResult, ztunnelCertificates, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
3921
+ export { ACCESS_LOG_FORMAT, CHANNEL_USE, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, MIDDLEBOX, NETNS_DEFAULTS, NODE_IMAGES, NODE_LIFECYCLE, O04, O09, O10, O16, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, SPREAD_LEAD, STAGES, SYSCTL, TCP_STATE, WATCHDOG, ZONE_KEPT, ZONE_LABEL, ZONE_LOST, ZTUNNEL_PODS, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, certificateArgs, checkRun, cidrOverlap, cidrRange, clientObjects, clusterFilter, compose, createCluster, cyclesOf, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, faultSpan, followsZtunnel, goDuration, hostsOf, installMesh, intakeAfter, intakeOf, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, labSource, latchBand, leafCertificates, learnWindow, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, middleboxCommands, namespaceObject, namespaceOf, nodeLossBand, nodeNames, nodeReturnBand, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, setZtunnelEnv, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stopOf, stoppingOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, writeResult, zoneOf, ztunnelCertificates, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
3721
3922
 
3722
3923
  //# sourceMappingURL=index.js.map