@mark1russell7/kind 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -1
- package/build/index.d.ts +94 -8
- package/build/index.js +373 -34
- package/build/index.js.map +1 -1
- package/build/{judge-CmiF-Fcs.js → judge-Cgvkh4JX.js} +88 -7
- package/build/judge-Cgvkh4JX.js.map +1 -0
- package/build/{judge-B1nSdbS0.d.ts → judge-gpC49mVw.d.ts} +73 -4
- package/build/judge.d.ts +2 -2
- package/build/judge.js +1 -1
- package/image/client-grpc.mjs +17 -5
- package/package.json +2 -2
- package/src/client-lines.ts +37 -6
- package/build/judge-CmiF-Fcs.js.map +0 -1
package/build/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as signalOf, A as medianInterval, B as emptyOf, C as propagationOf, D as unitOf, E as reusesReset, F as backendsAt, G as outcomesOf, H as intakeOf, I as byClients, J as recordsOf, K as podsAt, L as byNode, M as quantileOf, N as HBONE_PORT, O as updatesOf, P as TCP_STATE, Q as rulesOf, R as deletionOf, S as observationEnd, T as referenceOf, U as isTunnelState, V as failuresOf, W as kernelCounts, X as restartOf, Y as requestsOf, Z as restartsAt, _ as checkRun, _t as seconds, a as calibrationOf, at as stoppingOf, b as judgeKind, c as predictionText, ct as ztunnelPodsOf, d as FORMAT_V1, dt as HEALTH_PORT, et as signalsOf, f as judge, ft as LOG_LABEL, g as LAST_SLACK, gt as bandOf, h as KIND_NOISE, ht as SERVER, i as readRecord, it as socketsOf, j as middle, k as RELATIVE, l as timeSpread, lt as ztunnelReplaced, m as upgradeCell, mt as REDIS, n as kindCellRecordSchema, nt as sliceEmptyOf, o as cellsTable, ot as transitionOf, p as upgrade, pt as PORT, q as podsOf, r as kindCellRecordV1Schema, rt as slicesOf, s as medianText, st as tunnelsOf, t as STAGES, tt as sliceDropOf, u as FORMAT, ut as CLIENT, v as deletionsOf, vt as secondsOf, w as propagationSource, x as observationAfter, y as factsOf, yt as showBand, z as dropOf } from "./judge-Cgvkh4JX.js";
|
|
2
2
|
import { valueIn } from "@mark1russell7/quantity";
|
|
3
3
|
import { execFile, execFileSync } from "node:child_process";
|
|
4
4
|
import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
@@ -658,6 +658,186 @@ const policyObject = (account) => ({
|
|
|
658
658
|
rules: [{ from: [{ source: { principals: [`cluster.local/ns/{namespace}/sa/${account}`] } }] }]
|
|
659
659
|
}
|
|
660
660
|
});
|
|
661
|
+
/** These are the names of the sysctls that the steps of a plan set. Each one is a sysctl of a network namespace. */
|
|
662
|
+
const SYSCTL = { retries: "net.ipv4.tcp_retries2" };
|
|
663
|
+
/**
|
|
664
|
+
* These are the values of those sysctls in a new network namespace of Linux v6.17. A restore puts
|
|
665
|
+
* them back. `tcp_retries2` is TCP_RETR2 (include/net/tcp.h, line 105, and net/ipv4/tcp_ipv4.c,
|
|
666
|
+
* line 3604).
|
|
667
|
+
*/
|
|
668
|
+
const NETNS_DEFAULTS = { retries: "15" };
|
|
669
|
+
/**
|
|
670
|
+
* These are the names of the middlebox in the filter table of a replica. One chain takes the
|
|
671
|
+
* segments from the peers, and one chain takes the segments to them. The table of xt_recent holds
|
|
672
|
+
* the time of the last segment of each peer.
|
|
673
|
+
*/
|
|
674
|
+
const MIDDLEBOX = {
|
|
675
|
+
input: "LAB-MBX-IN",
|
|
676
|
+
output: "LAB-MBX-OUT",
|
|
677
|
+
table: "LAB-MBX"
|
|
678
|
+
};
|
|
679
|
+
/** This time is between the stop of the calls and the fault of a quiet black hole. The last answers and their ACKs, at most 200 ms late, come in it. */
|
|
680
|
+
const QUIET_LEAD = {
|
|
681
|
+
value: 1,
|
|
682
|
+
unit: "s"
|
|
683
|
+
};
|
|
684
|
+
/** This time is between the close of the old channel of each client and the fault of a gap with `close`. The stream of the old channel ends in it. */
|
|
685
|
+
const CLOSE_LEAD = {
|
|
686
|
+
value: 2,
|
|
687
|
+
unit: "s"
|
|
688
|
+
};
|
|
689
|
+
/** This time is how long each client calls on its new channel at the fault of a gap with `close`, before it closes it. */
|
|
690
|
+
const CHANNEL_USE = {
|
|
691
|
+
value: 5,
|
|
692
|
+
unit: "s"
|
|
693
|
+
};
|
|
694
|
+
const IPV4 = /^(25[0-5]|2[0-4]\d|1?\d?\d)(\.(25[0-5]|2[0-4]\d|1?\d?\d)){3}$/;
|
|
695
|
+
/**
|
|
696
|
+
* This function gives the command of the injector that sets one sysctl of its network namespace.
|
|
697
|
+
* Only a sysctl of the namespace (`net.`) and a whole number reach it.
|
|
698
|
+
*/
|
|
699
|
+
const sysctlCommand = (key, value) => {
|
|
700
|
+
if (!/^net\.[a-z0-9_.]+$/.test(key)) throw new Error(`not a sysctl of a network namespace: ${key}`);
|
|
701
|
+
if (!/^\d+$/.test(value)) throw new Error(`not a whole number: ${value}`);
|
|
702
|
+
return [
|
|
703
|
+
"sysctl",
|
|
704
|
+
"-w",
|
|
705
|
+
`${key}=${value}`
|
|
706
|
+
];
|
|
707
|
+
};
|
|
708
|
+
/**
|
|
709
|
+
* This function gives the commands of the injector that put a stateful middlebox in its network
|
|
710
|
+
* namespace, with each peer, on the local port `port`. With `idle` null, they remove it.
|
|
711
|
+
*
|
|
712
|
+
* The middlebox forgets a flow of the port after `idle` with no segment. Then it drops each later
|
|
713
|
+
* segment of it: the silent-drop form of the router of the harness. Conntrack stays as it is. The
|
|
714
|
+
* local leg from ztunnel to the app has no PING, and istio-cni needs its conntrack entry. The
|
|
715
|
+
* connmark of that entry sends each answer of the app back to ztunnel (cni/pkg/iptables/iptables.go
|
|
716
|
+
* of Istio 1.31.3, lines 172–178, 278–284 and 336–347).
|
|
717
|
+
*
|
|
718
|
+
* The match recent keeps a table in each network namespace (xt_recent.c of Linux v6.17). Its key is
|
|
719
|
+
* the address of the peer: the source of a segment from it, or the destination of a segment to it
|
|
720
|
+
* (lines 251–257). The chain of each direction has these rules, in sequence:
|
|
721
|
+
* - In the input chain only, a SYN sets the time of its peer and passes: a new flow.
|
|
722
|
+
* - A segment of a peer with a time in the last `idle` seconds passes, and it sets the time again
|
|
723
|
+
* (lines 298–321).
|
|
724
|
+
* - A segment of a peer with an older time is dropped, and the time stays old. Thus the flow stays forgotten.
|
|
725
|
+
* - A segment of a peer that the table does not hold sets its time and passes (lines 283–290). That
|
|
726
|
+
* is the flow that was open before the set.
|
|
727
|
+
*
|
|
728
|
+
* Each rule that passes a segment goes back to INPUT or OUTPUT (RETURN), thus the other rules of the
|
|
729
|
+
* namespace still act on it. The key is an address, not a flow. Thus a new SYN of a peer also refreshes an
|
|
730
|
+
* old flow of the peer. A cell of the middlebox opens no new flow in its gap.
|
|
731
|
+
*/
|
|
732
|
+
const middleboxCommands = (peers, idle, port) => {
|
|
733
|
+
if (peers.length === 0) throw new Error("a middlebox needs a peer");
|
|
734
|
+
for (const peer of peers) if (!IPV4.test(peer)) throw new Error(`not an address: ${peer}`);
|
|
735
|
+
if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`not a port: ${port}`);
|
|
736
|
+
const op = idle === null ? "-D" : "-I";
|
|
737
|
+
const jumps = peers.flatMap((peer) => [[
|
|
738
|
+
"iptables",
|
|
739
|
+
op,
|
|
740
|
+
"INPUT",
|
|
741
|
+
"-s",
|
|
742
|
+
`${peer}/32`,
|
|
743
|
+
"-p",
|
|
744
|
+
"tcp",
|
|
745
|
+
"--dport",
|
|
746
|
+
String(port),
|
|
747
|
+
"-j",
|
|
748
|
+
MIDDLEBOX.input
|
|
749
|
+
], [
|
|
750
|
+
"iptables",
|
|
751
|
+
op,
|
|
752
|
+
"OUTPUT",
|
|
753
|
+
"-d",
|
|
754
|
+
`${peer}/32`,
|
|
755
|
+
"-p",
|
|
756
|
+
"tcp",
|
|
757
|
+
"--sport",
|
|
758
|
+
String(port),
|
|
759
|
+
"-j",
|
|
760
|
+
MIDDLEBOX.output
|
|
761
|
+
]]);
|
|
762
|
+
const chains = [MIDDLEBOX.input, MIDDLEBOX.output];
|
|
763
|
+
if (idle === null) return [
|
|
764
|
+
...chains.map((chain) => [
|
|
765
|
+
"iptables",
|
|
766
|
+
"-F",
|
|
767
|
+
chain
|
|
768
|
+
]),
|
|
769
|
+
...jumps,
|
|
770
|
+
...chains.map((chain) => [
|
|
771
|
+
"iptables",
|
|
772
|
+
"-X",
|
|
773
|
+
chain
|
|
774
|
+
])
|
|
775
|
+
];
|
|
776
|
+
const time = valueIn(idle, "s");
|
|
777
|
+
if (!Number.isInteger(time) || time < 1) throw new Error(`the idle timeout of the middlebox is a whole number of seconds, not ${time}`);
|
|
778
|
+
const rules = (chain, side) => {
|
|
779
|
+
const recent = [
|
|
780
|
+
"-m",
|
|
781
|
+
"recent",
|
|
782
|
+
"--name",
|
|
783
|
+
MIDDLEBOX.table,
|
|
784
|
+
side
|
|
785
|
+
];
|
|
786
|
+
return [
|
|
787
|
+
...chain === MIDDLEBOX.input ? [[
|
|
788
|
+
"iptables",
|
|
789
|
+
"-A",
|
|
790
|
+
chain,
|
|
791
|
+
"-p",
|
|
792
|
+
"tcp",
|
|
793
|
+
"--syn",
|
|
794
|
+
...recent,
|
|
795
|
+
"--set",
|
|
796
|
+
"-j",
|
|
797
|
+
"RETURN"
|
|
798
|
+
]] : [],
|
|
799
|
+
[
|
|
800
|
+
"iptables",
|
|
801
|
+
"-A",
|
|
802
|
+
chain,
|
|
803
|
+
...recent,
|
|
804
|
+
"--update",
|
|
805
|
+
"--seconds",
|
|
806
|
+
String(time),
|
|
807
|
+
"-j",
|
|
808
|
+
"RETURN"
|
|
809
|
+
],
|
|
810
|
+
[
|
|
811
|
+
"iptables",
|
|
812
|
+
"-A",
|
|
813
|
+
chain,
|
|
814
|
+
...recent,
|
|
815
|
+
"--rcheck",
|
|
816
|
+
"-j",
|
|
817
|
+
"DROP"
|
|
818
|
+
],
|
|
819
|
+
[
|
|
820
|
+
"iptables",
|
|
821
|
+
"-A",
|
|
822
|
+
chain,
|
|
823
|
+
...recent,
|
|
824
|
+
"--set",
|
|
825
|
+
"-j",
|
|
826
|
+
"RETURN"
|
|
827
|
+
]
|
|
828
|
+
];
|
|
829
|
+
};
|
|
830
|
+
return [
|
|
831
|
+
...chains.map((chain) => [
|
|
832
|
+
"iptables",
|
|
833
|
+
"-N",
|
|
834
|
+
chain
|
|
835
|
+
]),
|
|
836
|
+
...rules(MIDDLEBOX.input, "--rsource"),
|
|
837
|
+
...rules(MIDDLEBOX.output, "--rdest"),
|
|
838
|
+
...jumps
|
|
839
|
+
];
|
|
840
|
+
};
|
|
661
841
|
/**
|
|
662
842
|
* This function gives the target of a reset or a black hole of a cell. In T07 and T18 it is Redis,
|
|
663
843
|
* with the replicas as peers. In T15 the fault is on the mesh leg between the client pods and the
|
|
@@ -827,20 +1007,49 @@ const planOf = (cell) => {
|
|
|
827
1007
|
}],
|
|
828
1008
|
restore: []
|
|
829
1009
|
};
|
|
830
|
-
case "blackhole":
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
1010
|
+
case "blackhole": {
|
|
1011
|
+
const quiet = trigger.quiet === true;
|
|
1012
|
+
const retries = trigger.retries;
|
|
1013
|
+
if (retries !== void 0 && !(Number.isInteger(retries) && retries >= 1 && retries <= 255)) throw new Error(`tcp_retries2 is a count from 1 to 255, not ${retries}`);
|
|
1014
|
+
const start = quiet ? [{
|
|
1015
|
+
do: "signal",
|
|
1016
|
+
target: "clients",
|
|
1017
|
+
signal: "SIGUSR2"
|
|
1018
|
+
}] : [];
|
|
1019
|
+
const prepare = [...retries === void 0 ? [] : [{
|
|
1020
|
+
do: "sysctl",
|
|
1021
|
+
target: "clients",
|
|
1022
|
+
key: SYSCTL.retries,
|
|
1023
|
+
value: String(retries)
|
|
1024
|
+
}], ...quiet ? [{
|
|
1025
|
+
do: "signal",
|
|
1026
|
+
target: "clients",
|
|
1027
|
+
signal: "SIGUSR1"
|
|
1028
|
+
}, {
|
|
1029
|
+
do: "wait",
|
|
1030
|
+
time: QUIET_LEAD
|
|
1031
|
+
}] : []];
|
|
1032
|
+
return {
|
|
1033
|
+
...prepare.length === 0 ? {} : { prepare },
|
|
1034
|
+
inject: [{
|
|
1035
|
+
do: "inject",
|
|
1036
|
+
target: linkOf(cell),
|
|
1037
|
+
action: "blackhole"
|
|
1038
|
+
}],
|
|
1039
|
+
hold: needs(trigger.hold, "a hold"),
|
|
1040
|
+
heal: [{
|
|
1041
|
+
do: "inject",
|
|
1042
|
+
target: linkOf(cell),
|
|
1043
|
+
action: "clear"
|
|
1044
|
+
}, ...start],
|
|
1045
|
+
restore: [...retries === void 0 ? [] : [{
|
|
1046
|
+
do: "sysctl",
|
|
1047
|
+
target: "clients",
|
|
1048
|
+
key: SYSCTL.retries,
|
|
1049
|
+
value: NETNS_DEFAULTS.retries
|
|
1050
|
+
}], ...start]
|
|
1051
|
+
};
|
|
1052
|
+
}
|
|
844
1053
|
case "peer-gone-without-close": return {
|
|
845
1054
|
inject: [{
|
|
846
1055
|
do: "inject",
|
|
@@ -952,24 +1161,60 @@ const planOf = (cell) => {
|
|
|
952
1161
|
]
|
|
953
1162
|
}]
|
|
954
1163
|
};
|
|
955
|
-
case "idle":
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
target: "clients",
|
|
959
|
-
signal: "SIGUSR1"
|
|
960
|
-
}],
|
|
961
|
-
hold: needs(trigger.hold, "a hold, the time of the gap"),
|
|
962
|
-
heal: [{
|
|
963
|
-
do: "signal",
|
|
964
|
-
target: "clients",
|
|
965
|
-
signal: "SIGUSR2"
|
|
966
|
-
}],
|
|
967
|
-
restore: [{
|
|
1164
|
+
case "idle": {
|
|
1165
|
+
const hold = needs(trigger.hold, "a hold, the time of the gap");
|
|
1166
|
+
const start = {
|
|
968
1167
|
do: "signal",
|
|
969
1168
|
target: "clients",
|
|
970
1169
|
signal: "SIGUSR2"
|
|
971
|
-
}
|
|
972
|
-
|
|
1170
|
+
};
|
|
1171
|
+
if (trigger.close === true) {
|
|
1172
|
+
const close = {
|
|
1173
|
+
do: "signal",
|
|
1174
|
+
target: "clients",
|
|
1175
|
+
signal: "SIGHUP"
|
|
1176
|
+
};
|
|
1177
|
+
return {
|
|
1178
|
+
prepare: [close, {
|
|
1179
|
+
do: "wait",
|
|
1180
|
+
time: CLOSE_LEAD
|
|
1181
|
+
}],
|
|
1182
|
+
inject: [
|
|
1183
|
+
start,
|
|
1184
|
+
{
|
|
1185
|
+
do: "wait",
|
|
1186
|
+
time: CHANNEL_USE
|
|
1187
|
+
},
|
|
1188
|
+
close
|
|
1189
|
+
],
|
|
1190
|
+
hold,
|
|
1191
|
+
heal: [start],
|
|
1192
|
+
restore: [start]
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
const middlebox = trigger.middlebox;
|
|
1196
|
+
return {
|
|
1197
|
+
...middlebox === void 0 ? {} : { prepare: [{
|
|
1198
|
+
do: "middlebox",
|
|
1199
|
+
target: "servers",
|
|
1200
|
+
idle: middlebox,
|
|
1201
|
+
port: HBONE_PORT
|
|
1202
|
+
}] },
|
|
1203
|
+
inject: [{
|
|
1204
|
+
do: "signal",
|
|
1205
|
+
target: "clients",
|
|
1206
|
+
signal: "SIGUSR1"
|
|
1207
|
+
}],
|
|
1208
|
+
hold,
|
|
1209
|
+
heal: [start],
|
|
1210
|
+
restore: [start, ...middlebox === void 0 ? [] : [{
|
|
1211
|
+
do: "middlebox",
|
|
1212
|
+
target: "servers",
|
|
1213
|
+
idle: null,
|
|
1214
|
+
port: HBONE_PORT
|
|
1215
|
+
}]]
|
|
1216
|
+
};
|
|
1217
|
+
}
|
|
973
1218
|
case "ztunnel-restart": return {
|
|
974
1219
|
inject: [{
|
|
975
1220
|
do: "kubectl",
|
|
@@ -1413,7 +1658,11 @@ const deletedOf = (text) => {
|
|
|
1413
1658
|
};
|
|
1414
1659
|
//#endregion
|
|
1415
1660
|
//#region src/lab.ts
|
|
1416
|
-
/**
|
|
1661
|
+
/**
|
|
1662
|
+
* These kinds of kernel record stay in the raw record of a run. The lab also keeps each change of the
|
|
1663
|
+
* state of a socket of the HBONE port (`isTunnelState`): the tunnels of ztunnel. It counts the other
|
|
1664
|
+
* records and drops them.
|
|
1665
|
+
*/
|
|
1417
1666
|
const KEPT_KERNEL = /* @__PURE__ */ new Set([
|
|
1418
1667
|
"rst_tx",
|
|
1419
1668
|
"rst_rx",
|
|
@@ -1423,6 +1672,8 @@ const KEPT_KERNEL = /* @__PURE__ */ new Set([
|
|
|
1423
1672
|
"begin",
|
|
1424
1673
|
"fail"
|
|
1425
1674
|
]);
|
|
1675
|
+
/** This function tells if the lab keeps a kernel record of the prober. */
|
|
1676
|
+
const keptKernel = (record) => KEPT_KERNEL.has(String(record["ev"])) || isTunnelState(record);
|
|
1426
1677
|
/** This container class can share the process namespace of another container. */
|
|
1427
1678
|
const shared = async () => {
|
|
1428
1679
|
const { GenericContainer } = await import("testcontainers");
|
|
@@ -1580,7 +1831,7 @@ const openKindLab = async (cluster, image) => {
|
|
|
1580
1831
|
const ev = String(record["ev"]);
|
|
1581
1832
|
if (ev === "begin") begun(true, "");
|
|
1582
1833
|
if (ev === "fail") begun(false, String(record["why"]));
|
|
1583
|
-
if (!
|
|
1834
|
+
if (!keptKernel(record) && ev !== "clock") {
|
|
1584
1835
|
dropped[ev] = (dropped[ev] ?? 0) + 1;
|
|
1585
1836
|
return;
|
|
1586
1837
|
}
|
|
@@ -1712,6 +1963,33 @@ const openKindLab = async (cluster, image) => {
|
|
|
1712
1963
|
});
|
|
1713
1964
|
return text;
|
|
1714
1965
|
},
|
|
1966
|
+
async netns(node, pid, commands) {
|
|
1967
|
+
for (const word of commands.flat()) if (!/^[A-Za-z0-9_.,:=/+-]+$/.test(word)) throw new Error(`not a plain word of a command: ${word}`);
|
|
1968
|
+
if (commands.length === 0) throw new Error("no command");
|
|
1969
|
+
const t = await lab.now();
|
|
1970
|
+
const text = await output(sidecarOf(node).injector, [
|
|
1971
|
+
"nsenter",
|
|
1972
|
+
"-t",
|
|
1973
|
+
String(pid),
|
|
1974
|
+
"-n",
|
|
1975
|
+
"sh",
|
|
1976
|
+
"-c",
|
|
1977
|
+
commands.map((argv) => argv.join(" ")).join(" && ")
|
|
1978
|
+
]);
|
|
1979
|
+
push({
|
|
1980
|
+
ev: "act",
|
|
1981
|
+
t,
|
|
1982
|
+
t1: await lab.now(),
|
|
1983
|
+
node,
|
|
1984
|
+
spec: {
|
|
1985
|
+
op: "netns",
|
|
1986
|
+
pid,
|
|
1987
|
+
commands
|
|
1988
|
+
},
|
|
1989
|
+
out: text.slice(0, 2e3)
|
|
1990
|
+
});
|
|
1991
|
+
return text;
|
|
1992
|
+
},
|
|
1715
1993
|
async now() {
|
|
1716
1994
|
return (await lab.act(firstNode, { op: "now" })).t0;
|
|
1717
1995
|
},
|
|
@@ -2718,6 +2996,46 @@ const signalClients = async (ctx, signal) => {
|
|
|
2718
2996
|
t1: Math.max(...stamps.map((stamp) => stamp.t1))
|
|
2719
2997
|
};
|
|
2720
2998
|
};
|
|
2999
|
+
/**
|
|
3000
|
+
* This function gives the network namespace of each pod of a target, as a node and a PID in it. The
|
|
3001
|
+
* pods are each client pod (`clients`), or each replica (`servers`).
|
|
3002
|
+
*/
|
|
3003
|
+
const namespacesOf = async (ctx, target, replicas) => {
|
|
3004
|
+
if (target === "servers") return Promise.all(replicas.map(async (replica) => ({
|
|
3005
|
+
node: replica.node,
|
|
3006
|
+
pod: replica.pod,
|
|
3007
|
+
pid: await ctx.lab.pidOf(replica.node, ctx.namespace, replica.pod, "app")
|
|
3008
|
+
})));
|
|
3009
|
+
if (target !== "clients") throw new Error(`a sysctl or a middlebox takes the target clients or servers, not ${target}`);
|
|
3010
|
+
const clients = [...ctx.tracker.pods.values()].filter((pod) => pod.labels["app"] === "client" && !pod.del && pod.node !== null);
|
|
3011
|
+
if (clients.length === 0) throw new Error("the step needs clients");
|
|
3012
|
+
return Promise.all(clients.map(async (pod) => ({
|
|
3013
|
+
node: pod.node,
|
|
3014
|
+
pod: pod.name,
|
|
3015
|
+
pid: await ctx.lab.pidOf(pod.node, ctx.namespace, pod.name, CLIENT)
|
|
3016
|
+
})));
|
|
3017
|
+
};
|
|
3018
|
+
/**
|
|
3019
|
+
* This function puts a stateful middlebox in the network namespace of each replica, or removes it.
|
|
3020
|
+
* Each client pod is a peer (`middleboxCommands`), and the nodes go at the same time. After the set,
|
|
3021
|
+
* the lab reads the filter table: it shows the chains of the middlebox. Before the removal, the lab
|
|
3022
|
+
* reads the filter table again: the counters show which segments the middlebox dropped.
|
|
3023
|
+
*/
|
|
3024
|
+
const middleboxServers = async (ctx, idle, port, replicas) => {
|
|
3025
|
+
const peers = [...ctx.tracker.pods.values()].filter((pod) => pod.labels["app"] === "client" && !pod.del && pod.ip !== null).map((pod) => pod.ip).sort();
|
|
3026
|
+
const commands = middleboxCommands(peers, idle, port);
|
|
3027
|
+
const t0 = await ctx.lab.now();
|
|
3028
|
+
const each = await namespacesOf(ctx, "servers", replicas);
|
|
3029
|
+
await Promise.all(each.map(async ({ node, pid }) => {
|
|
3030
|
+
if (idle === null) await ctx.lab.tables(node, pid, "filter");
|
|
3031
|
+
await ctx.lab.netns(node, pid, commands);
|
|
3032
|
+
if (idle !== null) await ctx.lab.tables(node, pid, "filter");
|
|
3033
|
+
}));
|
|
3034
|
+
return {
|
|
3035
|
+
t0,
|
|
3036
|
+
t1: await ctx.lab.now()
|
|
3037
|
+
};
|
|
3038
|
+
};
|
|
2721
3039
|
/** This function does one step of a fault plan. It gives the stamp of its act, or null for an act without a stamp. */
|
|
2722
3040
|
const doStep = async (ctx, step, replicas, background) => {
|
|
2723
3041
|
const { lab, cluster, namespace } = ctx;
|
|
@@ -2849,6 +3167,18 @@ const doStep = async (ctx, step, replicas, background) => {
|
|
|
2849
3167
|
t1: await lab.now()
|
|
2850
3168
|
};
|
|
2851
3169
|
}
|
|
3170
|
+
case "sysctl": {
|
|
3171
|
+
const command = sysctlCommand(step.key, step.value);
|
|
3172
|
+
const t0 = await lab.now();
|
|
3173
|
+
await Promise.all((await namespacesOf(ctx, step.target, replicas)).map(({ node, pid }) => lab.netns(node, pid, [command])));
|
|
3174
|
+
return {
|
|
3175
|
+
t0,
|
|
3176
|
+
t1: await lab.now()
|
|
3177
|
+
};
|
|
3178
|
+
}
|
|
3179
|
+
case "middlebox":
|
|
3180
|
+
if (step.target !== "servers") throw new Error(`a middlebox takes the target servers, not ${step.target}`);
|
|
3181
|
+
return middleboxServers(ctx, step.idle, step.port, replicas);
|
|
2852
3182
|
}
|
|
2853
3183
|
};
|
|
2854
3184
|
/** This function tells if a run is over: the trigger is done, and the cell is at rest again. */
|
|
@@ -2925,6 +3255,14 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
|
|
|
2925
3255
|
const notes = [];
|
|
2926
3256
|
const forgotten = Object.entries(cleared).filter(([, count]) => count > 0);
|
|
2927
3257
|
if (forgotten.length > 0) notes.push(`before the run, the lab deleted SYN_SENT entries of the ClusterIP: ${forgotten.map(([node, count]) => `${count} on ${node}`).join(", ")}`);
|
|
3258
|
+
for (const step of plan.prepare ?? []) try {
|
|
3259
|
+
await doStep(ctx, step, replicas, background);
|
|
3260
|
+
} catch (error) {
|
|
3261
|
+
const why = `the step ${step.do} before the fault failed: ${String(error).slice(0, 400)}`;
|
|
3262
|
+
ctx.log(` ${why}`);
|
|
3263
|
+
for (const each of plan.restore) await doStep(ctx, each, replicas, background).catch((failure) => ctx.log(` the restore ${each.do} failed: ${String(failure)}`));
|
|
3264
|
+
return empty(why, start);
|
|
3265
|
+
}
|
|
2928
3266
|
for (const step of plan.inject) {
|
|
2929
3267
|
const stamp = await doStep(ctx, step, replicas, background);
|
|
2930
3268
|
if (stamp !== null && fault === null) fault = stamp.t0;
|
|
@@ -3235,7 +3573,8 @@ const estimate = (cell, environments = 2) => {
|
|
|
3235
3573
|
const after = observationAfter(cell);
|
|
3236
3574
|
const observed = after === null ? 0 : hold + valueIn(after, "s");
|
|
3237
3575
|
const closes = cell.prediction.kind === "ends" && cell.prediction.band !== null ? valueIn(cell.prediction.band.hi, "s") + 3 : 0;
|
|
3238
|
-
const
|
|
3576
|
+
const quiet = cell.trigger.quiet === true ? hold + 10 : 0;
|
|
3577
|
+
const each = 25 + Math.max(observed, closes, quiet, trigger[cell.trigger.kind] ?? (cell.template === "T18" ? hold + 35 : 15 + valueIn(cell.pod.preStop, "s")));
|
|
3239
3578
|
return (cell.mesh === void 0 ? 60 : 180) * environments + (cell.runs + environments) * each;
|
|
3240
3579
|
};
|
|
3241
3580
|
/**
|
|
@@ -3267,6 +3606,6 @@ const shardOf = (cells, index, count) => {
|
|
|
3267
3606
|
return out;
|
|
3268
3607
|
};
|
|
3269
3608
|
//#endregion
|
|
3270
|
-
export { ACCESS_LOG_FORMAT, CLIENT, CLIENT_ACCOUNT, CONTROLLER, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HEALTH_PORT, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, NODE_IMAGES, O04, O09, O10, O16, POLICY, PORT, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, STAGES, WATCHDOG, ZTUNNEL_PODS, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, checkRun, cidrOverlap, cidrRange, clientObjects, clusterFilter, compose, createCluster, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, followsZtunnel, goDuration, hostsOf, installMesh, intakeAfter, intakeOf, istioOperator, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, latchBand, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, namespaceObject, namespaceOf, nodeNames, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, setZtunnelEnv, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stoppingOf, tarEntry, timeSpread, toolPath, transitionOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, writeResult, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
|
|
3609
|
+
export { ACCESS_LOG_FORMAT, CHANNEL_USE, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, MIDDLEBOX, NETNS_DEFAULTS, NODE_IMAGES, O04, O09, O10, O16, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, STAGES, SYSCTL, TCP_STATE, WATCHDOG, ZTUNNEL_PODS, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, checkRun, cidrOverlap, cidrRange, clientObjects, clusterFilter, compose, createCluster, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, followsZtunnel, goDuration, hostsOf, installMesh, intakeAfter, intakeOf, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, latchBand, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, middleboxCommands, namespaceObject, namespaceOf, nodeNames, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, setZtunnelEnv, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stoppingOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, writeResult, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
|
|
3271
3610
|
|
|
3272
3611
|
//# sourceMappingURL=index.js.map
|