@mark1russell7/kind 0.11.8 → 0.11.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.d.ts +77 -3
- package/build/index.js +223 -22
- package/build/index.js.map +1 -1
- package/build/{judge-Cs91dOGO.js → judge-BIXZEX__.js} +150 -17
- package/build/judge-BIXZEX__.js.map +1 -0
- package/build/{judge-DnB9wEwC.d.ts → judge-DLxpBamt.d.ts} +75 -6
- package/build/judge.d.ts +2 -2
- package/build/judge.js +1 -1
- package/image/agent.mjs +58 -1
- package/package.json +2 -2
- package/build/judge-Cs91dOGO.js.map +0 -1
package/build/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as unitOf, $t as secondsOf, A as TriggerV1, An as ZONE_KEPT, At as restartsAt, B as RunCheck, Bn as scenarioProblems, Bt as stoppingOf, C as KindCellResult, Cn as Workload, Ct as kernelCounts, D as PredictionV1, Dn as KubeProxyMode, Dt as recordsOf, E as LoadSample, En as KindScenario, Et as podsOf, F as KIND_NOISE, Fn as goDuration, Ft as sliceEmptyOf, G as factsOf, Gt as CLIENT, H as checkRun, Ht as tunnelsOf, I as KindDecision, In as kindConfig, It as sliceReadyAt, J as observationEnd, Jt as PORT, K as judgeKind, Kt as HEALTH_PORT, L as KindVerdict, Ln as kubeProxyPatch, Lt as sliceReadyOf, M as upgrade, Mn as ZONE_LOST, Mt as signalOf, N as upgradeCell, Nn as cidrOverlap, Nt as signalsOf, O as RunCheckV1, On as NODE_IMAGES, Ot as requestsOf, P as Deleted, Pn as cidrRange, Pt as sliceDropOf, Q as reusesReset, Qt as seconds, R as LAST_SLACK, Rn as nodeNames, Rt as slicesOf, S as KindCellRecordV1, Sn as WindowPrediction, St as isTunnelState, T as KindVerdictV1, Tn as KindConfig, Tt as podsAt, U as cyclesOf, Ut as ztunnelPodsOf, V as RunFacts, Vn as zoneOf, Vt as transitionOf, W as deletionsOf, Wt as ztunnelReplaced, X as propagationSource, Xt as SERVER, Y as propagationOf, Yt as REDIS, Z as referenceOf, Zt as bandOf, _ as ClusterManifest, _n as ResetReuse, _t as deletionOf, a as STAGES, an as Flap, at as PropagationSource, b as FORMAT_V1, bn as Trigger, bt as failuresOf, c as readRecord, cn as KubeletProbe, ct as RulesLine, d as calibrationOf, dn as MeshMode, dt as TCP_STATE, en as showBand, et as updatesOf, f as cellsTable, fn as PodSettings, ft as TunnelLine, g as BandV1, gn as Replica, gt as byNode, h as timeSpread, hn as Reference, ht as byClients, i as quantileOf, in as EdgeProtocol, it as PodLine, j as judge, jn as ZONE_LABEL, jt as rulesOf, k as RunFactsV1, kn as NodeImage, kt as restartOf, l as Judged, ln as Mesh, lt as SliceLine, m as predictionText, mn as RedisDeclaration, mt as backendsAt, n as medianInterval, nn as Cycle, nt as HBONE_PORT, o as kindCellRecordSchema, on as KindCell, ot as RecordLine, p as medianText, pn as Prediction, pt as ZtunnelPodLine, q as observationAfter, qt as LOG_LABEL, r as middle, rn as Declaration, rt as Line, s as kindCellRecordV1Schema, sn as KindRun, st as Request, t as RELATIVE, tn as Band, tt as Endpoint, u as TimeSpread, un as MeshFlavor, ut as SocketLine, v as EnvironmentManifest, vn as Stack, vt as dropOf, w as KindCellV1, wn as KUBE_PROXY_MODES, wt as outcomesOf, x as KindCellRecord, xn as TriggerKind, xt as intakeOf, y as FORMAT, yn as Template, yt as emptyOf, z as Propagated, zn as scenario, zt as socketsOf } from "./judge-DLxpBamt.js";
|
|
2
2
|
import { Quantity } from "@mark1russell7/quantity";
|
|
3
3
|
import { CheckFn, Finding, HealthServer, Poll, Scenario, ServiceBuilder, ServiceDescription, Stage, StopWay, Timing, TransportImpl, Verdict } from "@mark1russell7/service";
|
|
4
4
|
import { BuildImage } from "@mark1russell7/harness";
|
|
@@ -169,6 +169,9 @@ type Step = {
|
|
|
169
169
|
readonly port: number;
|
|
170
170
|
} | {
|
|
171
171
|
readonly do: "certificates";
|
|
172
|
+
} | {
|
|
173
|
+
readonly do: "zone";
|
|
174
|
+
readonly lost: boolean;
|
|
172
175
|
};
|
|
173
176
|
/**
|
|
174
177
|
* A fault plan is the steps of a run:
|
|
@@ -177,6 +180,7 @@ type Step = {
|
|
|
177
180
|
* - `inject` starts the trigger. The stamp of its first act is the `fault` reference.
|
|
178
181
|
* - `heal` ends the fault, `hold` after the fault. A trigger that ends by itself has no heal.
|
|
179
182
|
* - `restore` brings the cell back to its steady state after the run.
|
|
183
|
+
* - `flaps` are the faults after the first one. A plan without them has one fault.
|
|
180
184
|
*/
|
|
181
185
|
interface FaultPlan {
|
|
182
186
|
readonly prepare?: readonly Step[] | undefined;
|
|
@@ -184,6 +188,8 @@ interface FaultPlan {
|
|
|
184
188
|
readonly hold: Quantity | null;
|
|
185
189
|
readonly heal: readonly Step[];
|
|
186
190
|
readonly restore: readonly Step[];
|
|
191
|
+
/** These flaps are the faults after the first one (`Trigger.flaps`). Each one does the inject and the heal again. */
|
|
192
|
+
readonly flaps?: readonly Flap[] | undefined;
|
|
187
193
|
}
|
|
188
194
|
/** This name is the name of the AuthorizationPolicy of the trigger `authorization-policy`. */
|
|
189
195
|
export declare const POLICY = "lab-deny";
|
|
@@ -261,8 +267,17 @@ export declare const sysctlCommand: (key: string, value: string) => string[];
|
|
|
261
267
|
* old flow of the peer. A cell of the middlebox opens no new flow in its gap.
|
|
262
268
|
*/
|
|
263
269
|
export declare const middleboxCommands: (peers: readonly string[], idle: Quantity | null, port: number) => string[][];
|
|
264
|
-
/**
|
|
270
|
+
/**
|
|
271
|
+
* This function gives the fault plan of a cell. A trigger with flaps (`Trigger.flaps`) gives them to
|
|
272
|
+
* its plan: the runner then does the inject and the heal again for each flap. A plan with no hold or
|
|
273
|
+
* no heal cannot flap.
|
|
274
|
+
*/
|
|
265
275
|
export declare const planOf: (cell: KindCell) => FaultPlan;
|
|
276
|
+
/**
|
|
277
|
+
* This function gives the time from the first fault of a trigger to its last heal. It is the hold,
|
|
278
|
+
* then the rest and the hold of each flap. A trigger with no hold gives zero.
|
|
279
|
+
*/
|
|
280
|
+
export declare const faultSpan: (trigger: Pick<KindCell["trigger"], "hold" | "flaps">) => Quantity;
|
|
266
281
|
/** This function tells if a trigger stops a pod of the service with its grace: a deletion, an eviction, a scale-in or a rollout. */
|
|
267
282
|
export declare const deletes: (cell: KindCell) => boolean;
|
|
268
283
|
//#endregion
|
|
@@ -615,6 +630,64 @@ export declare const restartBand: (latch: Band, liveness: KubeletProbe, preStop:
|
|
|
615
630
|
* replica from the rules. Each probe in the stall fails at its timeout, because the loop does not answer it.
|
|
616
631
|
*/
|
|
617
632
|
export declare const stallRemovalBand: (readiness: KubeletProbe, propagation: Propagation) => Band;
|
|
633
|
+
/**
|
|
634
|
+
* A node lifecycle is the set of timers that removes the pods of a lost node. The same timers give
|
|
635
|
+
* them back after the heal of the node. The lab sets none of them, thus each one is a default of Kubernetes
|
|
636
|
+
* v1.37.0 (file and line in that tag).
|
|
637
|
+
*/
|
|
638
|
+
interface NodeLifecycle {
|
|
639
|
+
/**
|
|
640
|
+
* This time is `NodeMonitorGracePeriod`. The node lifecycle controller marks a node Unknown when
|
|
641
|
+
* it saw no renewal of its lease for this time. Then it marks each pod of the node not ready.
|
|
642
|
+
*/
|
|
643
|
+
readonly grace: Quantity;
|
|
644
|
+
/** This time is `NodeMonitorPeriod`: the controller checks each node once in it. */
|
|
645
|
+
readonly monitor: Quantity;
|
|
646
|
+
/** This time is the renewal interval of the lease of the kubelet. */
|
|
647
|
+
readonly renew: Quantity;
|
|
648
|
+
/** This share is the jitter of each renewal interval: a renewal comes `renew` to `renew × (1 + jitter)` after the one before it. */
|
|
649
|
+
readonly jitter: number;
|
|
650
|
+
/** This time is the margin of the writes of the controller: the status of the node, then each pod of the node. */
|
|
651
|
+
readonly writes: Quantity;
|
|
652
|
+
/** This time is the PING timeout of the HTTP/2 health check of client-go. A kubelet that gets no PING ACK in it connects again. */
|
|
653
|
+
readonly pingTimeout: Quantity;
|
|
654
|
+
/** This time is the longest first backoff of a reflector of client-go before it watches again. */
|
|
655
|
+
readonly rewatch: Quantity;
|
|
656
|
+
/** This time is the sync period of the status manager of the kubelet: it puts back a pod status that the controller changed. */
|
|
657
|
+
readonly statusSync: Quantity;
|
|
658
|
+
}
|
|
659
|
+
/**
|
|
660
|
+
* These are the defaults of Kubernetes v1.37.0 for a lost node:
|
|
661
|
+
* - `grace` 50 s and `monitor` 5 s: pkg/controller/nodelifecycle/config/v1alpha1/defaults.go, lines
|
|
662
|
+
* 46 and 52. The controller checks the nodes on a sliding period (node_lifecycle_controller.go,
|
|
663
|
+
* lines 506–510). It sets the probe time when it sees a new lease (lines 952–960), and it marks the
|
|
664
|
+
* node Unknown after `grace` (line 962). Then it marks each pod of the node not ready (lines 768–773).
|
|
665
|
+
* - `renew` 10 s: `NodeLeaseDurationSeconds` 40 (pkg/kubelet/apis/config/v1beta1/defaults.go, line 150)
|
|
666
|
+
* times the fraction 0.25 (pkg/kubelet/kubelet.go, lines 237 and 1156).
|
|
667
|
+
* - `jitter` 0.04: the lease controller renews on a sliding jitter of 0.04
|
|
668
|
+
* (staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go, line 153).
|
|
669
|
+
* - `pingTimeout` 15 s: the PING timeout of the HTTP/2 health check, after a read idle timeout of 30 s
|
|
670
|
+
* (staging/src/k8s.io/apimachinery/pkg/util/net/http.go, lines 131–192).
|
|
671
|
+
* - `rewatch` 1.6 s: a first backoff of 800 ms with a jitter of 1.0
|
|
672
|
+
* (staging/src/k8s.io/client-go/tools/cache/reflector.go, lines 62–67).
|
|
673
|
+
* - `statusSync` 10 s: the sync period of the status manager (pkg/kubelet/status/status_manager.go, line 215).
|
|
674
|
+
* - `writes` 0.25 s is a margin of the lab, not a default.
|
|
675
|
+
*/
|
|
676
|
+
export declare const NODE_LIFECYCLE: NodeLifecycle;
|
|
677
|
+
/**
|
|
678
|
+
* This function gives the time from the loss of a node to the drop of its pods from the rules of
|
|
679
|
+
* the other nodes. The last renewal of the lease comes up to `renew × (1 + jitter)` before the loss.
|
|
680
|
+
* The controller sees it up to one `monitor` later, and marks the node `grace` after that. Then the
|
|
681
|
+
* writes and the propagation follow. The band does not add the low bound of the propagation.
|
|
682
|
+
*/
|
|
683
|
+
export declare const nodeLossBand: (lifecycle: NodeLifecycle, propagation: Propagation) => Band;
|
|
684
|
+
/**
|
|
685
|
+
* This function gives the time from the heal of a lost node until its pods are ready again in the
|
|
686
|
+
* EndpointSlice. The kubelet connects again in one PING timeout at the most: its old connection
|
|
687
|
+
* waits behind the backoff of TCP. Its reflectors watch again after their first backoff. Then its
|
|
688
|
+
* status manager puts back the pod status in one sync period at the most.
|
|
689
|
+
*/
|
|
690
|
+
export declare const nodeReturnBand: (lifecycle: NodeLifecycle) => Band;
|
|
618
691
|
/** This function gives the cap of a stage in a declaration, or zero when the declaration does not name the stage. */
|
|
619
692
|
export declare const capOf: (declaration: Declaration, stage: Stage) => Quantity;
|
|
620
693
|
/**
|
|
@@ -694,6 +767,7 @@ type Verdicts = Readonly<Record<string, Verdict>>;
|
|
|
694
767
|
* - In T07 a pause of kube-proxy comes before a deletion. In T18 the pause is on Redis.
|
|
695
768
|
* - The crash of the lab is SIGKILL from the node. Like a crash, it has no preStop hook and no drain.
|
|
696
769
|
* - The other faults are on a link, on Redis or on the process. The kubelet can restart a pod after them.
|
|
770
|
+
* A zone loss freezes the pods of its node and deletes none of them in its hold.
|
|
697
771
|
*/
|
|
698
772
|
export declare const stopOf: (cell: KindCell) => StopWay | null;
|
|
699
773
|
/**
|
|
@@ -800,5 +874,5 @@ export declare const toolPath: (tool: Tool, key?: string, cache?: string) => str
|
|
|
800
874
|
*/
|
|
801
875
|
export declare const ensureTool: (tool: Tool, key?: string, cache?: string) => Promise<string>;
|
|
802
876
|
//#endregion
|
|
803
|
-
export { type Band, type BandV1, CLIENT, type Cluster, type ClusterHosts, type ClusterManifest, type Credentials, DRAIN_RULES, type Declaration, type Deleted, type DeletionWindow, type EdgeProtocol, type EdsSpec, Endpoint, type EnvironmentManifest, FORMAT, FORMAT_V1, type FaultPlan, HBONE_PORT, HEALTH_PORT, type Judged, KIND_NOISE, KUBE_PROXY_MODES, type KindCell, type KindCellRecord, type KindCellRecordV1, type KindCellResult, type KindCellV1, type KindConfig, type KindDecision, type KindLab, type KindRun, type KindScenario, type KindVerdict, type KindVerdictV1, type KubeProxyMode, type KubeletProbe, LAST_SLACK, LOG_LABEL, type LdsSpec, type LeafCertificate, type LearnWindow, Line, type ListenerCounts, type LoadSample, type Mesh, type MeshFlavor, type MeshImages, type MeshMode, NODE_IMAGES, type NodeImage, O04, O09, O10, O16, PORT, type Parts, PodLine, type PodSettings, type Prediction, type PredictionV1, type Propagated, type Propagation, PropagationSource, REDIS, RELATIVE, RecordLine, type RedisDeclaration, type Reference, type Replica, Request, type ResetReuse, RulesLine, type RunCheck, type RunCheckV1, type RunFacts, type RunFactsV1, type RunOptions, SERVER, STAGES, type ServiceRule, SliceLine, SocketLine, type Stack, type Stamp, type Step, TCP_STATE, type Target, type Template, type TimeSpread, type Tool, type Trigger, type TriggerKind, type TriggerV1, TunnelLine, type Verdicts, type WindowPrediction, type Workload, ZtunnelPodLine, backendsAt, bandOf, byClients, byNode, calibrationOf, cellsTable, checkRun, cidrOverlap, cidrRange, deletionOf, deletionsOf, dropOf, emptyOf, factsOf, failuresOf, goDuration, intakeOf, isTunnelState, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, medianInterval, medianText, middle, nodeNames, observationAfter, observationEnd, outcomesOf, podsAt, podsOf, predictionText, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, referenceOf, requestsOf, restartOf, restartsAt, reusesReset, rulesOf, scenario, scenarioProblems, seconds, secondsOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, socketsOf, stoppingOf, timeSpread, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, ztunnelPodsOf, ztunnelReplaced };
|
|
877
|
+
export { type Band, type BandV1, CLIENT, type Cluster, type ClusterHosts, type ClusterManifest, type Credentials, type Cycle, DRAIN_RULES, type Declaration, type Deleted, type DeletionWindow, type EdgeProtocol, type EdsSpec, Endpoint, type EnvironmentManifest, FORMAT, FORMAT_V1, type FaultPlan, type Flap, HBONE_PORT, HEALTH_PORT, type Judged, KIND_NOISE, KUBE_PROXY_MODES, type KindCell, type KindCellRecord, type KindCellRecordV1, type KindCellResult, type KindCellV1, type KindConfig, type KindDecision, type KindLab, type KindRun, type KindScenario, type KindVerdict, type KindVerdictV1, type KubeProxyMode, type KubeletProbe, LAST_SLACK, LOG_LABEL, type LdsSpec, type LeafCertificate, type LearnWindow, Line, type ListenerCounts, type LoadSample, type Mesh, type MeshFlavor, type MeshImages, type MeshMode, NODE_IMAGES, type NodeImage, type NodeLifecycle, O04, O09, O10, O16, PORT, type Parts, PodLine, type PodSettings, type Prediction, type PredictionV1, type Propagated, type Propagation, PropagationSource, REDIS, RELATIVE, RecordLine, type RedisDeclaration, type Reference, type Replica, Request, type ResetReuse, RulesLine, type RunCheck, type RunCheckV1, type RunFacts, type RunFactsV1, type RunOptions, SERVER, STAGES, type ServiceRule, SliceLine, SocketLine, type Stack, type Stamp, type Step, TCP_STATE, type Target, type Template, type TimeSpread, type Tool, type Trigger, type TriggerKind, type TriggerV1, TunnelLine, type Verdicts, type WindowPrediction, type Workload, ZONE_KEPT, ZONE_LABEL, ZONE_LOST, ZtunnelPodLine, backendsAt, bandOf, byClients, byNode, calibrationOf, cellsTable, checkRun, cidrOverlap, cidrRange, cyclesOf, deletionOf, deletionsOf, dropOf, emptyOf, factsOf, failuresOf, goDuration, intakeOf, isTunnelState, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, medianInterval, medianText, middle, nodeNames, observationAfter, observationEnd, outcomesOf, podsAt, podsOf, predictionText, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, referenceOf, requestsOf, restartOf, restartsAt, reusesReset, rulesOf, scenario, scenarioProblems, seconds, secondsOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stoppingOf, timeSpread, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, zoneOf, ztunnelPodsOf, ztunnelReplaced };
|
|
804
878
|
//# sourceMappingURL=index.d.ts.map
|
package/build/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as rulesOf, A as RELATIVE, B as dropOf, C as observationEnd, D as reusesReset, E as referenceOf, F as TCP_STATE, G as kernelCounts, H as failuresOf, I as backendsAt, J as podsOf, K as outcomesOf, L as byClients, M as middle, N as quantileOf, O as unitOf, P as HBONE_PORT, Q as restartsAt, R as byNode, S as observationAfter, St as showBand, T as propagationSource, U as intakeOf, V as emptyOf, W as isTunnelState, X as requestsOf, Y as recordsOf, Z as restartOf, _ as checkRun, _t as REDIS, a as calibrationOf, at as sliceReadyOf, b as factsOf, bt as seconds, c as predictionText, ct as stoppingOf, d as FORMAT_V1, dt as ztunnelPodsOf, et as signalOf, f as judge, ft as ztunnelReplaced, g as LAST_SLACK, gt as PORT, h as KIND_NOISE, ht as LOG_LABEL, i as readRecord, it as sliceReadyAt, j as medianInterval, k as updatesOf, l as timeSpread, lt as transitionOf, m as upgradeCell, mt as HEALTH_PORT, n as kindCellRecordSchema, nt as sliceDropOf, o as cellsTable, ot as slicesOf, p as upgrade, pt as CLIENT, q as podsAt, r as kindCellRecordV1Schema, rt as sliceEmptyOf, s as medianText, st as socketsOf, t as STAGES, tt as signalsOf, u as FORMAT, ut as tunnelsOf, v as cyclesOf, vt as SERVER, w as propagationOf, x as judgeKind, xt as secondsOf, y as deletionsOf, yt as bandOf, z as deletionOf } from "./judge-BIXZEX__.js";
|
|
2
2
|
import { valueIn } from "@mark1russell7/quantity";
|
|
3
3
|
import { execFile, execFileSync } from "node:child_process";
|
|
4
4
|
import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
@@ -92,8 +92,16 @@ const NODE_IMAGES = {
|
|
|
92
92
|
image: "kindest/node:v1.36.4@sha256:099e049362a1526b2db71494e1947aae99bd16290d7c895f2b7ea312e3cbfaed"
|
|
93
93
|
}
|
|
94
94
|
};
|
|
95
|
+
/** This label gives the zone of a node, as Kubernetes names it. */
|
|
96
|
+
const ZONE_LABEL = "topology.kubernetes.io/zone";
|
|
97
|
+
/** This zone holds the control plane and each worker but the last. A zone loss keeps it. */
|
|
98
|
+
const ZONE_KEPT = "zone-a";
|
|
99
|
+
/** This zone holds only the last worker. A zone loss loses it. */
|
|
100
|
+
const ZONE_LOST = "zone-b";
|
|
101
|
+
/** This function gives the zone of the node at the position `at` of `nodeNames`, in a cluster of `count` nodes. */
|
|
102
|
+
const zoneOf = (at, count) => at === count - 1 && at > 0 ? ZONE_LOST : ZONE_KEPT;
|
|
95
103
|
/** This function gives a scenario with the defaults of Kubernetes for kube-proxy, as explicit values. */
|
|
96
|
-
const scenario = (name, mode, workers, kubernetes = "1.37") => ({
|
|
104
|
+
const scenario = (name, mode, workers, kubernetes = "1.37", zoned = false) => ({
|
|
97
105
|
name,
|
|
98
106
|
kubernetes,
|
|
99
107
|
workers,
|
|
@@ -110,7 +118,8 @@ const scenario = (name, mode, workers, kubernetes = "1.37") => ({
|
|
|
110
118
|
},
|
|
111
119
|
cni: "kindnet",
|
|
112
120
|
podSubnet: "10.244.0.0/16",
|
|
113
|
-
serviceSubnet: "10.96.0.0/16"
|
|
121
|
+
serviceSubnet: "10.96.0.0/16",
|
|
122
|
+
...zoned ? { zoned } : {}
|
|
114
123
|
});
|
|
115
124
|
const NAME = /^[a-z0-9]([a-z0-9-]{0,30}[a-z0-9])?$/;
|
|
116
125
|
/** This function gives the problems of a scenario: a name that kind refuses, no image for the version, or a count out of range. */
|
|
@@ -155,10 +164,13 @@ const kindConfig = (s) => {
|
|
|
155
164
|
podSubnet: s.podSubnet,
|
|
156
165
|
serviceSubnet: s.serviceSubnet
|
|
157
166
|
},
|
|
158
|
-
nodes: nodeNames(s).map((node, at) => ({
|
|
167
|
+
nodes: nodeNames(s).map((node, at, all) => ({
|
|
159
168
|
role: at === 0 ? "control-plane" : "worker",
|
|
160
169
|
image,
|
|
161
|
-
labels: {
|
|
170
|
+
labels: {
|
|
171
|
+
"lab.systems/node": node.slice(s.name.length + 1),
|
|
172
|
+
...s.zoned === true ? { [ZONE_LABEL]: zoneOf(at, all.length) } : {}
|
|
173
|
+
}
|
|
162
174
|
})),
|
|
163
175
|
kubeadmConfigPatches: [kubeProxyPatch(s)]
|
|
164
176
|
};
|
|
@@ -854,8 +866,36 @@ const middleboxCommands = (peers, idle, port) => {
|
|
|
854
866
|
* is a rule in the target replica only, with the client pods as peers.
|
|
855
867
|
*/
|
|
856
868
|
const linkOf = (cell) => cell.trigger.partition === true ? "server" : cell.template === "T15" ? "servers" : "redis";
|
|
857
|
-
/**
|
|
869
|
+
/**
|
|
870
|
+
* This function gives the fault plan of a cell. A trigger with flaps (`Trigger.flaps`) gives them to
|
|
871
|
+
* its plan: the runner then does the inject and the heal again for each flap. A plan with no hold or
|
|
872
|
+
* no heal cannot flap.
|
|
873
|
+
*/
|
|
858
874
|
const planOf = (cell) => {
|
|
875
|
+
const plan = stepsOf(cell);
|
|
876
|
+
const flaps = cell.trigger.flaps;
|
|
877
|
+
if (flaps === void 0 || flaps.length === 0) return plan;
|
|
878
|
+
if (plan.hold === null || plan.heal.length === 0) throw new Error(`the trigger ${cell.trigger.kind} has no hold and heal, thus it cannot flap`);
|
|
879
|
+
for (const flap of flaps) if (valueIn(flap.rest, "ms") <= 0 || valueIn(flap.hold, "ms") <= 0) throw new Error("each flap has a rest and a hold above zero");
|
|
880
|
+
return {
|
|
881
|
+
...plan,
|
|
882
|
+
flaps
|
|
883
|
+
};
|
|
884
|
+
};
|
|
885
|
+
/**
|
|
886
|
+
* This function gives the time from the first fault of a trigger to its last heal. It is the hold,
|
|
887
|
+
* then the rest and the hold of each flap. A trigger with no hold gives zero.
|
|
888
|
+
*/
|
|
889
|
+
const faultSpan = (trigger) => {
|
|
890
|
+
const hold = trigger.hold === void 0 ? 0 : valueIn(trigger.hold, "s");
|
|
891
|
+
const flaps = (trigger.flaps ?? []).reduce((sum, flap) => sum + valueIn(flap.rest, "s") + valueIn(flap.hold, "s"), 0);
|
|
892
|
+
return {
|
|
893
|
+
value: Math.round((hold + flaps) * 1e6) / 1e6,
|
|
894
|
+
unit: "s"
|
|
895
|
+
};
|
|
896
|
+
};
|
|
897
|
+
/** This function gives the steps of the fault plan of a cell, with no flaps. */
|
|
898
|
+
const stepsOf = (cell) => {
|
|
859
899
|
const ns = ["-n", "{namespace}"];
|
|
860
900
|
const trigger = cell.trigger;
|
|
861
901
|
const replicas = cell.pod.replicas;
|
|
@@ -1306,6 +1346,29 @@ const planOf = (cell) => {
|
|
|
1306
1346
|
on: false
|
|
1307
1347
|
}]
|
|
1308
1348
|
};
|
|
1349
|
+
case "zone-loss": return {
|
|
1350
|
+
prepare: [{
|
|
1351
|
+
do: "signal",
|
|
1352
|
+
target: "clients",
|
|
1353
|
+
signal: "SIGHUP"
|
|
1354
|
+
}, {
|
|
1355
|
+
do: "wait",
|
|
1356
|
+
time: SPREAD_LEAD
|
|
1357
|
+
}],
|
|
1358
|
+
inject: [{
|
|
1359
|
+
do: "zone",
|
|
1360
|
+
lost: true
|
|
1361
|
+
}],
|
|
1362
|
+
hold: needs(trigger.hold, "a hold, the time that the zone is lost"),
|
|
1363
|
+
heal: [{
|
|
1364
|
+
do: "zone",
|
|
1365
|
+
lost: false
|
|
1366
|
+
}],
|
|
1367
|
+
restore: [{
|
|
1368
|
+
do: "zone",
|
|
1369
|
+
lost: false
|
|
1370
|
+
}]
|
|
1371
|
+
};
|
|
1309
1372
|
}
|
|
1310
1373
|
};
|
|
1311
1374
|
/** This function tells if a trigger stops a pod of the service with its grace: a deletion, an eviction, a scale-in or a rollout. */
|
|
@@ -2422,7 +2485,12 @@ const serverObjects = (cell, namespace, image, version = 1) => {
|
|
|
2422
2485
|
topologyKey: "kubernetes.io/hostname",
|
|
2423
2486
|
whenUnsatisfiable: "ScheduleAnyway",
|
|
2424
2487
|
labelSelector: { matchLabels: { app: "lab" } }
|
|
2425
|
-
}
|
|
2488
|
+
}, ...cell.trigger.kind === "zone-loss" ? [{
|
|
2489
|
+
maxSkew: 1,
|
|
2490
|
+
topologyKey: ZONE_LABEL,
|
|
2491
|
+
whenUnsatisfiable: "DoNotSchedule",
|
|
2492
|
+
labelSelector: { matchLabels: { app: "lab" } }
|
|
2493
|
+
}] : []],
|
|
2426
2494
|
containers: [{
|
|
2427
2495
|
name: "app",
|
|
2428
2496
|
image,
|
|
@@ -2464,6 +2532,11 @@ const clientObjects = (cell, namespace, image, target) => [{
|
|
|
2464
2532
|
operator: "Exists",
|
|
2465
2533
|
effect: "NoSchedule"
|
|
2466
2534
|
}],
|
|
2535
|
+
...cell.trigger.kind === "zone-loss" ? { affinity: { nodeAffinity: { requiredDuringSchedulingIgnoredDuringExecution: { nodeSelectorTerms: [{ matchExpressions: [{
|
|
2536
|
+
key: ZONE_LABEL,
|
|
2537
|
+
operator: "NotIn",
|
|
2538
|
+
values: [ZONE_LOST]
|
|
2539
|
+
}] }] } } } } : {},
|
|
2467
2540
|
containers: [{
|
|
2468
2541
|
name: CLIENT,
|
|
2469
2542
|
image,
|
|
@@ -2576,6 +2649,50 @@ const stallRemovalBand = (readiness, propagation) => {
|
|
|
2576
2649
|
const spread = secondsOf(propagation);
|
|
2577
2650
|
return band(kubelet.lo + spread.lo, kubelet.hi + spread.hi);
|
|
2578
2651
|
};
|
|
2652
|
+
/**
|
|
2653
|
+
* These are the defaults of Kubernetes v1.37.0 for a lost node:
|
|
2654
|
+
* - `grace` 50 s and `monitor` 5 s: pkg/controller/nodelifecycle/config/v1alpha1/defaults.go, lines
|
|
2655
|
+
* 46 and 52. The controller checks the nodes on a sliding period (node_lifecycle_controller.go,
|
|
2656
|
+
* lines 506–510). It sets the probe time when it sees a new lease (lines 952–960), and it marks the
|
|
2657
|
+
* node Unknown after `grace` (line 962). Then it marks each pod of the node not ready (lines 768–773).
|
|
2658
|
+
* - `renew` 10 s: `NodeLeaseDurationSeconds` 40 (pkg/kubelet/apis/config/v1beta1/defaults.go, line 150)
|
|
2659
|
+
* times the fraction 0.25 (pkg/kubelet/kubelet.go, lines 237 and 1156).
|
|
2660
|
+
* - `jitter` 0.04: the lease controller renews on a sliding jitter of 0.04
|
|
2661
|
+
* (staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go, line 153).
|
|
2662
|
+
* - `pingTimeout` 15 s: the PING timeout of the HTTP/2 health check, after a read idle timeout of 30 s
|
|
2663
|
+
* (staging/src/k8s.io/apimachinery/pkg/util/net/http.go, lines 131–192).
|
|
2664
|
+
* - `rewatch` 1.6 s: a first backoff of 800 ms with a jitter of 1.0
|
|
2665
|
+
* (staging/src/k8s.io/client-go/tools/cache/reflector.go, lines 62–67).
|
|
2666
|
+
* - `statusSync` 10 s: the sync period of the status manager (pkg/kubelet/status/status_manager.go, line 215).
|
|
2667
|
+
* - `writes` 0.25 s is a margin of the lab, not a default.
|
|
2668
|
+
*/
|
|
2669
|
+
const NODE_LIFECYCLE = {
|
|
2670
|
+
grace: seconds(50),
|
|
2671
|
+
monitor: seconds(5),
|
|
2672
|
+
renew: seconds(10),
|
|
2673
|
+
jitter: .04,
|
|
2674
|
+
writes: seconds(.25),
|
|
2675
|
+
pingTimeout: seconds(15),
|
|
2676
|
+
rewatch: seconds(1.6),
|
|
2677
|
+
statusSync: seconds(10)
|
|
2678
|
+
};
|
|
2679
|
+
/**
|
|
2680
|
+
* This function gives the time from the loss of a node to the drop of its pods from the rules of
|
|
2681
|
+
* the other nodes. The last renewal of the lease comes up to `renew × (1 + jitter)` before the loss.
|
|
2682
|
+
* The controller sees it up to one `monitor` later, and marks the node `grace` after that. Then the
|
|
2683
|
+
* writes and the propagation follow. The band does not add the low bound of the propagation.
|
|
2684
|
+
*/
|
|
2685
|
+
const nodeLossBand = (lifecycle, propagation) => {
|
|
2686
|
+
const last = s$1(lifecycle.renew) * (1 + lifecycle.jitter);
|
|
2687
|
+
return band(s$1(lifecycle.grace) - last, s$1(lifecycle.grace) + s$1(lifecycle.monitor) + s$1(lifecycle.writes) + s$1(propagation.hi));
|
|
2688
|
+
};
|
|
2689
|
+
/**
|
|
2690
|
+
* This function gives the time from the heal of a lost node until its pods are ready again in the
|
|
2691
|
+
* EndpointSlice. The kubelet connects again in one PING timeout at the most: its old connection
|
|
2692
|
+
* waits behind the backoff of TCP. Its reflectors watch again after their first backoff. Then its
|
|
2693
|
+
* status manager puts back the pod status in one sync period at the most.
|
|
2694
|
+
*/
|
|
2695
|
+
const nodeReturnBand = (lifecycle) => band(0, s$1(lifecycle.pingTimeout) + s$1(lifecycle.rewatch) + s$1(lifecycle.statusSync) + s$1(lifecycle.writes));
|
|
2579
2696
|
/** This function gives the cap of a stage in a declaration, or zero when the declaration does not name the stage. */
|
|
2580
2697
|
const capOf = (declaration, stage) => {
|
|
2581
2698
|
const found = declaration.phases.find(([name]) => name === stage);
|
|
@@ -2681,6 +2798,7 @@ const scenarioOf = (cell, propagation) => ({
|
|
|
2681
2798
|
* - In T07 a pause of kube-proxy comes before a deletion. In T18 the pause is on Redis.
|
|
2682
2799
|
* - The crash of the lab is SIGKILL from the node. Like a crash, it has no preStop hook and no drain.
|
|
2683
2800
|
* - The other faults are on a link, on Redis or on the process. The kubelet can restart a pod after them.
|
|
2801
|
+
* A zone loss freezes the pods of its node and deletes none of them in its hold.
|
|
2684
2802
|
*/
|
|
2685
2803
|
const stopOf = (cell) => {
|
|
2686
2804
|
switch (cell.trigger.kind) {
|
|
@@ -2701,7 +2819,8 @@ const stopOf = (cell) => {
|
|
|
2701
2819
|
case "idle":
|
|
2702
2820
|
case "ztunnel-restart":
|
|
2703
2821
|
case "certificate-expiry":
|
|
2704
|
-
case "cannot-serve":
|
|
2822
|
+
case "cannot-serve":
|
|
2823
|
+
case "zone-loss": return null;
|
|
2705
2824
|
}
|
|
2706
2825
|
};
|
|
2707
2826
|
/**
|
|
@@ -2740,8 +2859,13 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
2740
2859
|
const slug = (name) => name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
|
|
2741
2860
|
/** This function tells if a cell has Redis: only a declaration of the service can have it. */
|
|
2742
2861
|
const hasRedis = (cell) => cell.declaration !== null && cell.declaration.redis !== null;
|
|
2743
|
-
/**
|
|
2744
|
-
|
|
2862
|
+
/**
|
|
2863
|
+
* This function gives the key of the cluster of a cell: its kube-proxy mode, its node count, and the
|
|
2864
|
+
* flavor of its mesh. A zone loss has clusters of its own, with a zone on each node.
|
|
2865
|
+
*/
|
|
2866
|
+
const clusterKey = (cell) => cell.mesh !== void 0 ? `${cell.mode}-${cell.workers}-${cell.mesh.flavor}` : zoned(cell) ? `${cell.mode}-${cell.workers}-zones` : `${cell.mode}-${cell.workers}`;
|
|
2867
|
+
/** This function tells if a cell needs a cluster with zones: a zone loss. */
|
|
2868
|
+
const zoned = (cell) => cell.trigger.kind === "zone-loss";
|
|
2745
2869
|
/**
|
|
2746
2870
|
* This tracker keeps the latest state of the cell from the lines: each pod, each slice and each
|
|
2747
2871
|
* client. It also has, for each node, the backends of its rules and the hosts of its client sidecar.
|
|
@@ -2804,7 +2928,8 @@ const unsteady = (ctx) => {
|
|
|
2804
2928
|
for (const node of ctx.cluster.nodes) if ((tracker.hosts.get(node) ?? []).join() !== expected) return `the client sidecar of ${node} has ${(tracker.hosts.get(node) ?? []).join(" ") || "no host"}`;
|
|
2805
2929
|
}
|
|
2806
2930
|
const nodes = new Set([...tracker.clients.values()].map((client) => client.node));
|
|
2807
|
-
|
|
2931
|
+
const clientNodes = ctx.cluster.nodes.length - (cell.trigger.kind === "zone-loss" ? 1 : 0);
|
|
2932
|
+
if (nodes.size !== clientNodes) return `clients answer on ${nodes.size} nodes, not ${clientNodes}`;
|
|
2808
2933
|
for (const [pod, client] of tracker.clients) {
|
|
2809
2934
|
if (tracker.last - client.ok > 1e3) return `client ${pod} has no answer in the last second`;
|
|
2810
2935
|
if (tracker.last - client.failed < 2e3) return `client ${pod} failed in the last 2 s`;
|
|
@@ -2821,6 +2946,37 @@ const steady = async (ctx, within) => {
|
|
|
2821
2946
|
}
|
|
2822
2947
|
return why;
|
|
2823
2948
|
};
|
|
2949
|
+
/** These taints of the node lifecycle controller mark a node that it does not see as ready. */
|
|
2950
|
+
const NODE_LOSS_TAINTS = ["node.kubernetes.io/unreachable", "node.kubernetes.io/not-ready"];
|
|
2951
|
+
/**
|
|
2952
|
+
* This function tells why a node is not at rest for the node lifecycle controller, or gives null. A
|
|
2953
|
+
* ready pod is not sufficient after a zone loss. The controller can still see the node as lost, and
|
|
2954
|
+
* then it does not mark the pods again at the next loss. The controller removes its taints only after
|
|
2955
|
+
* it sees the node ready.
|
|
2956
|
+
*/
|
|
2957
|
+
const nodeUnsteady = async (cluster, name) => {
|
|
2958
|
+
const node = JSON.parse(await cluster.kubectl([
|
|
2959
|
+
"get",
|
|
2960
|
+
"node",
|
|
2961
|
+
name,
|
|
2962
|
+
"-o",
|
|
2963
|
+
"json"
|
|
2964
|
+
]));
|
|
2965
|
+
const ready = node.status?.conditions?.find((condition) => condition.type === "Ready")?.status;
|
|
2966
|
+
if (ready !== "True") return `the node ${name} has the Ready condition ${ready ?? "none"}`;
|
|
2967
|
+
const taints = (node.spec?.taints ?? []).map((taint) => taint.key).filter((key) => NODE_LOSS_TAINTS.includes(key));
|
|
2968
|
+
return taints.length === 0 ? null : `the node ${name} has the taints ${taints.join(", ")}`;
|
|
2969
|
+
};
|
|
2970
|
+
/** This function waits until a node is at rest for the node lifecycle controller. It gives null, or the reason when the node is not at rest in time. */
|
|
2971
|
+
const nodeSteady = async (cluster, name, within) => {
|
|
2972
|
+
const end = Date.now() + within;
|
|
2973
|
+
let why = await nodeUnsteady(cluster, name);
|
|
2974
|
+
while (why !== null && Date.now() < end) {
|
|
2975
|
+
await sleep(1e3);
|
|
2976
|
+
why = await nodeUnsteady(cluster, name);
|
|
2977
|
+
}
|
|
2978
|
+
return why;
|
|
2979
|
+
};
|
|
2824
2980
|
/** This function samples the load of the VM for 4 s, and names the lab containers that are not of this run. */
|
|
2825
2981
|
const sample = async (ctx, ours) => {
|
|
2826
2982
|
const second = (await ctx.loader.exec([
|
|
@@ -3288,8 +3444,16 @@ const doStep = async (ctx, step, replicas, background) => {
|
|
|
3288
3444
|
case "middlebox":
|
|
3289
3445
|
if (step.target !== "servers") throw new Error(`a middlebox takes the target servers, not ${step.target}`);
|
|
3290
3446
|
return middleboxServers(ctx, step.idle, step.port, replicas);
|
|
3447
|
+
case "zone": return lab.act(lostNodeOf(cluster), {
|
|
3448
|
+
op: "zone",
|
|
3449
|
+
lost: step.lost
|
|
3450
|
+
});
|
|
3291
3451
|
}
|
|
3292
3452
|
};
|
|
3453
|
+
/** This function gives the node of the lost zone: the last worker (`zoneOf`). */
|
|
3454
|
+
const lostNodeOf = (cluster) => cluster.nodes.at(-1);
|
|
3455
|
+
/** This function puts the replicas of a node first, and keeps the sequence of the others. */
|
|
3456
|
+
const lostFirst = (replicas, node) => [...replicas.filter((replica) => replica.node === node), ...replicas.filter((replica) => replica.node !== node)];
|
|
3293
3457
|
/** This function tells if a run is over: the trigger is done, and the cell is at rest again. */
|
|
3294
3458
|
const over = (ctx, run, lines) => {
|
|
3295
3459
|
if (unsteady(ctx) !== null) return false;
|
|
@@ -3342,8 +3506,12 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
|
|
|
3342
3506
|
await probeThePods(ctx);
|
|
3343
3507
|
await pollListeners(ctx);
|
|
3344
3508
|
await quiet(ctx, ours);
|
|
3509
|
+
const zoneLoss = cell.trigger.kind === "zone-loss";
|
|
3510
|
+
const lost = zoneLoss ? await nodeSteady(cluster, lostNodeOf(cluster), 3e5) : null;
|
|
3511
|
+
if (lost !== null) return empty(`the node of the lost zone was not at rest: ${lost}`, await lab.now());
|
|
3345
3512
|
if (await steady(ctx, 3e5) !== null) return empty("the cell was not at rest after the wait for a quiet VM", await lab.now());
|
|
3346
|
-
const replicas = replicasOf(ctx);
|
|
3513
|
+
const replicas = zoneLoss ? lostFirst(replicasOf(ctx), lostNodeOf(cluster)) : replicasOf(ctx);
|
|
3514
|
+
if (zoneLoss && replicas[0]?.node !== lostNodeOf(cluster)) return empty(`no replica is on the node of the lost zone, ${lostNodeOf(cluster)}`, await lab.now());
|
|
3347
3515
|
const restarts = new Map(ctx.tracker.servers().map((pod) => [pod.name, pod.cs.reduce((sum, status) => sum + status.rc, 0)]));
|
|
3348
3516
|
const target = hasRedis(cell) && [
|
|
3349
3517
|
"rst",
|
|
@@ -3352,7 +3520,7 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
|
|
|
3352
3520
|
"peer-gone-without-close",
|
|
3353
3521
|
"rollout"
|
|
3354
3522
|
].includes(cell.trigger.kind) ? null : replicas[0];
|
|
3355
|
-
const paused = cell.template === "T07" && cell.trigger.kind === "freeze-process" ? cluster.nodes[1] ?? cluster.nodes[0] : null;
|
|
3523
|
+
const paused = cell.template === "T07" && cell.trigger.kind === "freeze-process" ? cluster.nodes[1] ?? cluster.nodes[0] : zoneLoss ? lostNodeOf(cluster) : null;
|
|
3356
3524
|
const cleared = await lab.forgetHandshakes(ctx.clusterIp, 80);
|
|
3357
3525
|
lab.take();
|
|
3358
3526
|
for (const key of Object.keys(lab.dropped)) delete lab.dropped[key];
|
|
@@ -3377,16 +3545,46 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
|
|
|
3377
3545
|
if (stamp !== null && fault === null) fault = stamp.t0;
|
|
3378
3546
|
}
|
|
3379
3547
|
let heal = null;
|
|
3380
|
-
|
|
3548
|
+
const healAfter = async (from, hold) => {
|
|
3381
3549
|
const now = await lab.now();
|
|
3382
|
-
await sleep(Math.max(0, valueIn(
|
|
3550
|
+
await sleep(Math.max(0, valueIn(hold, "ms") - (now - (from ?? now))));
|
|
3551
|
+
let stamped = null;
|
|
3383
3552
|
for (const step of plan.heal) try {
|
|
3384
3553
|
const stamp = await doStep(ctx, step, replicas, background);
|
|
3385
|
-
if (stamp !== null &&
|
|
3554
|
+
if (stamp !== null && stamped === null) stamped = stamp.t0;
|
|
3386
3555
|
} catch (error) {
|
|
3387
3556
|
notes.push(`the heal ${step.do} failed: ${String(error)}`);
|
|
3388
3557
|
}
|
|
3558
|
+
return stamped;
|
|
3559
|
+
};
|
|
3560
|
+
const cycles = [];
|
|
3561
|
+
if (plan.hold !== null) {
|
|
3562
|
+
heal = await healAfter(fault, plan.hold);
|
|
3563
|
+
if (fault !== null && heal !== null) cycles.push({
|
|
3564
|
+
fault,
|
|
3565
|
+
heal
|
|
3566
|
+
});
|
|
3567
|
+
for (const flap of plan.flaps ?? []) {
|
|
3568
|
+
const now = await lab.now();
|
|
3569
|
+
await sleep(Math.max(0, valueIn(flap.rest, "ms") - (now - (heal ?? now))));
|
|
3570
|
+
let again = null;
|
|
3571
|
+
for (const step of plan.inject) try {
|
|
3572
|
+
const stamp = await doStep(ctx, step, replicas, background);
|
|
3573
|
+
if (stamp !== null && again === null) again = stamp.t0;
|
|
3574
|
+
} catch (error) {
|
|
3575
|
+
notes.push(`the inject ${step.do} of a flap failed: ${String(error)}`);
|
|
3576
|
+
}
|
|
3577
|
+
const healed = await healAfter(again, flap.hold);
|
|
3578
|
+
if (again !== null && healed !== null) cycles.push({
|
|
3579
|
+
fault: again,
|
|
3580
|
+
heal: healed
|
|
3581
|
+
});
|
|
3582
|
+
heal = healed ?? heal;
|
|
3583
|
+
}
|
|
3389
3584
|
}
|
|
3585
|
+
const flapped = (plan.flaps ?? []).length > 0;
|
|
3586
|
+
if (flapped && cycles.length !== (plan.flaps ?? []).length + 1) notes.push(`the run has ${cycles.length} cycles with stamps of the ${(plan.flaps ?? []).length + 1} of the trigger`);
|
|
3587
|
+
const flapping = flapped ? { cycles } : {};
|
|
3390
3588
|
const settled = await Promise.allSettled(background);
|
|
3391
3589
|
for (const each of settled) if (each.status === "fulfilled" && typeof each.value === "object" && each.value !== null && "t1" in each.value) resume = each.value.t1;
|
|
3392
3590
|
const limit = Date.now() + 24e4 + valueIn(cell.pod.grace, "ms") + valueIn(cell.pod.preStop, "ms");
|
|
@@ -3401,7 +3599,8 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
|
|
|
3401
3599
|
acts: {
|
|
3402
3600
|
fault,
|
|
3403
3601
|
heal,
|
|
3404
|
-
resume
|
|
3602
|
+
resume,
|
|
3603
|
+
...flapping
|
|
3405
3604
|
},
|
|
3406
3605
|
target: stoppedOf(cell, replicas, target, lab.lines),
|
|
3407
3606
|
replicas,
|
|
@@ -3450,7 +3649,8 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
|
|
|
3450
3649
|
acts: {
|
|
3451
3650
|
fault,
|
|
3452
3651
|
heal,
|
|
3453
|
-
resume
|
|
3652
|
+
resume,
|
|
3653
|
+
...flapping
|
|
3454
3654
|
},
|
|
3455
3655
|
target: stopped,
|
|
3456
3656
|
paused,
|
|
@@ -3535,7 +3735,7 @@ const runKindCells = async (cells, options) => {
|
|
|
3535
3735
|
for (const [key, group] of groups) {
|
|
3536
3736
|
const partial = /* @__PURE__ */ new Map();
|
|
3537
3737
|
for (let env = 0; env < environments; env += 1) {
|
|
3538
|
-
const s = scenario(`lab-${key}-${env}`, group[0].mode, group[0].workers);
|
|
3738
|
+
const s = scenario(`lab-${key}-${env}`, group[0].mode, group[0].workers, void 0, zoned(group[0]));
|
|
3539
3739
|
const dir = join(options.work ?? join(tmpdir(), "systems-kind-work"), s.name);
|
|
3540
3740
|
const mesh = group[0].mesh;
|
|
3541
3741
|
options.log(`cluster ${s.name}: ${s.kubeProxy.mode}, ${s.workers} workers${mesh === void 0 ? "" : `, Istio ${mesh.flavor}`}`);
|
|
@@ -3668,7 +3868,7 @@ const runKindCells = async (cells, options) => {
|
|
|
3668
3868
|
};
|
|
3669
3869
|
/** This function gives the estimate of a cell in seconds, for the arithmetic of the shards. */
|
|
3670
3870
|
const estimate = (cell, environments = 2) => {
|
|
3671
|
-
const hold =
|
|
3871
|
+
const hold = valueIn(faultSpan(cell.trigger), "s");
|
|
3672
3872
|
const trigger = {
|
|
3673
3873
|
"liveness-restart": 100,
|
|
3674
3874
|
crash: 30,
|
|
@@ -3679,7 +3879,8 @@ const estimate = (cell, environments = 2) => {
|
|
|
3679
3879
|
idle: hold + 5,
|
|
3680
3880
|
"ztunnel-restart": 40,
|
|
3681
3881
|
"certificate-expiry": hold + 20,
|
|
3682
|
-
"cannot-serve": hold + 20
|
|
3882
|
+
"cannot-serve": hold + 20,
|
|
3883
|
+
"zone-loss": hold + 45
|
|
3683
3884
|
};
|
|
3684
3885
|
const after = observationAfter(cell);
|
|
3685
3886
|
const observed = after === null ? 0 : hold + valueIn(after, "s");
|
|
@@ -3717,6 +3918,6 @@ const shardOf = (cells, index, count) => {
|
|
|
3717
3918
|
return out;
|
|
3718
3919
|
};
|
|
3719
3920
|
//#endregion
|
|
3720
|
-
export { ACCESS_LOG_FORMAT, CHANNEL_USE, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, MIDDLEBOX, NETNS_DEFAULTS, NODE_IMAGES, O04, O09, O10, O16, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, SPREAD_LEAD, STAGES, SYSCTL, TCP_STATE, WATCHDOG, ZTUNNEL_PODS, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, certificateArgs, checkRun, cidrOverlap, cidrRange, clientObjects, clusterFilter, compose, createCluster, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, followsZtunnel, goDuration, hostsOf, installMesh, intakeAfter, intakeOf, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, labSource, latchBand, leafCertificates, learnWindow, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, middleboxCommands, namespaceObject, namespaceOf, nodeNames, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, setZtunnelEnv, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stopOf, stoppingOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, writeResult, ztunnelCertificates, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
|
|
3921
|
+
export { ACCESS_LOG_FORMAT, CHANNEL_USE, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, MIDDLEBOX, NETNS_DEFAULTS, NODE_IMAGES, NODE_LIFECYCLE, O04, O09, O10, O16, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, SPREAD_LEAD, STAGES, SYSCTL, TCP_STATE, WATCHDOG, ZONE_KEPT, ZONE_LABEL, ZONE_LOST, ZTUNNEL_PODS, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, certificateArgs, checkRun, cidrOverlap, cidrRange, clientObjects, clusterFilter, compose, createCluster, cyclesOf, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, faultSpan, followsZtunnel, goDuration, hostsOf, installMesh, intakeAfter, intakeOf, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, labSource, latchBand, leafCertificates, learnWindow, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, middleboxCommands, namespaceObject, namespaceOf, nodeLossBand, nodeNames, nodeReturnBand, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, setZtunnelEnv, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stopOf, stoppingOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, writeResult, zoneOf, ztunnelCertificates, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
|
|
3721
3922
|
|
|
3722
3923
|
//# sourceMappingURL=index.js.map
|