@mark1russell7/kind 0.13.0 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.d.ts +163 -6
- package/build/index.js +608 -31
- package/build/index.js.map +1 -1
- package/build/{judge-Bc84w-no.js → judge-C7Zr4KoS.js} +43 -12
- package/build/judge-C7Zr4KoS.js.map +1 -0
- package/build/{judge-B_wVAwjx.d.ts → judge-DpdSgpjv.d.ts} +51 -4
- package/build/judge.d.ts +2 -2
- package/build/judge.js +1 -1
- package/image/Dockerfile +2 -1
- package/image/client.mjs +25 -0
- package/image/npm-lock.json +20 -0
- package/image/package.json +2 -0
- package/package.json +1 -1
- package/build/judge-Bc84w-no.js.map +0 -1
package/build/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as signalsOf, A as HBONE_PORT, B as isTunnelState, C as observationEnd, Ct as
|
|
1
|
+
import { $ as signalsOf, A as HBONE_PORT, At as secondsOf, B as isTunnelState, C as observationEnd, Ct as VIP_ANNOTATION, D as reusesReset, Dt as overshootBand, E as referenceOf, Et as drainBand, F as deletionOf, G as podsOf, H as outcomesOf, I as dropOf, J as restartOf, K as recordsOf, L as emptyOf, M as backendsAt, N as byClients, O as unitOf, Ot as requestBand, P as byNode, Q as signalOf, R as failuresOf, S as observationAfter, St as SERVER, T as propagationSource, Tt as bandOf, U as periodOf, V as kernelCounts, W as podsAt, X as rulesOf, Y as restartsAt, Z as sessionsAt, _ as checkRun, _t as LB_CLASS, a as calibrationOf, at as socketsOf, b as factsOf, bt as PORT, c as predictionText, ct as tunnelsOf, d as FORMAT_V1, dt as RELATIVE, et as sliceDropOf, f as judge, ft as medianInterval, g as LAST_SLACK, gt as HEALTH_PORT, h as KIND_NOISE, ht as CLIENT, i as readRecord, it as slicesOf, j as TCP_STATE, jt as showBand, k as updatesOf, kt as seconds, l as timeSpread, lt as ztunnelPodsOf, m as upgradeCell, mt as quantileOf, n as kindCellRecordSchema, nt as sliceReadyAt, o as cellsTable, ot as stoppingOf, p as upgrade, pt as middle, q as requestsOf, r as kindCellRecordV1Schema, rt as sliceReadyOf, s as medianText, st as transitionOf, t as STAGES, tt as sliceEmptyOf, u as FORMAT, ut as ztunnelReplaced, v as cyclesOf, vt as LOG_LABEL, w as propagationOf, wt as backlogEnd, x as judgeKind, xt as REDIS, y as deletionsOf, yt as OUTSIDE, z as intakeOf } from "./judge-C7Zr4KoS.js";
|
|
2
2
|
import { valueIn } from "@mark1russell7/quantity";
|
|
3
3
|
import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
4
4
|
import { dirname, join } from "node:path";
|
|
@@ -1029,6 +1029,29 @@ const sysctlCommand = (key, value) => {
|
|
|
1029
1029
|
`${key}=${value}`
|
|
1030
1030
|
];
|
|
1031
1031
|
};
|
|
1032
|
+
/** This port is the port of a DNS server: each answer of the cluster DNS comes from it. */
|
|
1033
|
+
const DNS_PORT = 53;
|
|
1034
|
+
/**
|
|
1035
|
+
* This function gives the commands of the injector that drop each answer of a DNS server in its
|
|
1036
|
+
* network namespace, on UDP and on TCP. With `on` false, they remove the drops. The rules go in
|
|
1037
|
+
* INPUT, thus each query still goes out, and CoreDNS answers it. Only the answer is lost. A send of
|
|
1038
|
+
* the resolver gets no error, as for a DNS server with no route back. The rules match the source port,
|
|
1039
|
+
* thus each DNS server of the namespace is in the fault.
|
|
1040
|
+
*/
|
|
1041
|
+
const dnsCommands = (on) => {
|
|
1042
|
+
const op = on ? "-I" : "-D";
|
|
1043
|
+
return ["udp", "tcp"].map((protocol) => [
|
|
1044
|
+
"iptables",
|
|
1045
|
+
op,
|
|
1046
|
+
"INPUT",
|
|
1047
|
+
"-p",
|
|
1048
|
+
protocol,
|
|
1049
|
+
"--sport",
|
|
1050
|
+
String(53),
|
|
1051
|
+
"-j",
|
|
1052
|
+
"DROP"
|
|
1053
|
+
]);
|
|
1054
|
+
};
|
|
1032
1055
|
/**
|
|
1033
1056
|
* This function gives the commands of the injector that put a stateful middlebox in its network
|
|
1034
1057
|
* namespace, with each peer, on the local port `port`. With `idle` null, they remove it.
|
|
@@ -1754,6 +1777,20 @@ const stepsOf = (cell) => {
|
|
|
1754
1777
|
restore: back
|
|
1755
1778
|
};
|
|
1756
1779
|
}
|
|
1780
|
+
case "dns-blackhole": return {
|
|
1781
|
+
inject: [{
|
|
1782
|
+
do: "dns",
|
|
1783
|
+
target: "clients",
|
|
1784
|
+
on: true
|
|
1785
|
+
}],
|
|
1786
|
+
hold: needs(trigger.hold, "a hold, the time that no answer of DNS gets to a client"),
|
|
1787
|
+
heal: [{
|
|
1788
|
+
do: "dns",
|
|
1789
|
+
target: "clients",
|
|
1790
|
+
on: false
|
|
1791
|
+
}],
|
|
1792
|
+
restore: []
|
|
1793
|
+
};
|
|
1757
1794
|
}
|
|
1758
1795
|
};
|
|
1759
1796
|
/** This function tells if a trigger stops a pod of the service with its grace: a deletion, an eviction, a scale-in or a rollout. */
|
|
@@ -2186,6 +2223,413 @@ const ztunnelCertificates = async (cluster) => {
|
|
|
2186
2223
|
return out;
|
|
2187
2224
|
};
|
|
2188
2225
|
//#endregion
|
|
2226
|
+
//#region src/kubevip.ts
|
|
2227
|
+
/**
|
|
2228
|
+
* This module installs kube-vip 1.3.1 in a kind cluster (SYS-51). The install gives a Service of type
|
|
2229
|
+
* LoadBalancer an address on the Docker network of the nodes: the VIP. In ARP mode, one node holds
|
|
2230
|
+
* the VIP on its link, and it sends a gratuitous ARP for it each `vip_arpRate`. A Lease elects that
|
|
2231
|
+
* node. Each value of the install is in `kubeVipObjects`, with its reason.
|
|
2232
|
+
*
|
|
2233
|
+
* The lab uses a VIP of a Service, not a VIP of the control plane. A VIP of the control plane needs
|
|
2234
|
+
* three control-plane nodes and TLS on port 6443. A VIP of a Service needs one DaemonSet on the
|
|
2235
|
+
* workers, and it serves the server of the cell.
|
|
2236
|
+
*
|
|
2237
|
+
* Each term of the election comes from the source of v1.3.1:
|
|
2238
|
+
* - With `svc_election` off, one election of the lease `plndr-svcs-lock` gives each VIP to one
|
|
2239
|
+
* node (pkg/manager/worker/common.go, `GlobalLeader`). The lease is in the namespace of the pod.
|
|
2240
|
+
* - The election is `leaderelection.RunOrDie` of client-go v0.37.1, with a Lease lock and
|
|
2241
|
+
* `ReleaseOnCancel` (pkg/election/election.go). The identity is the name of the node.
|
|
2242
|
+
* - A leader that loses its lease stops the process (`OnStoppedLeading`), and the kubelet starts it again.
|
|
2243
|
+
* - The ARP manager sends a gratuitous ARP for each VIP at each tick of one ticker of
|
|
2244
|
+
* `vip_arpRate`. A new VIP gets its first one at the next tick, not at once (pkg/arp/arp.go).
|
|
2245
|
+
*/
|
|
2246
|
+
/** This version is the release of kube-vip of the lab, from 2026-10-19: the tag of its image. */
|
|
2247
|
+
const KUBE_VIP_VERSION = "1.3.1";
|
|
2248
|
+
/**
|
|
2249
|
+
* This image is kube-vip 1.3.1 for linux/amd64, pinned by the digest of its manifest. The digest of
|
|
2250
|
+
* the index of the tag is sha256:1962f9daf226503f7e6b107f595cb28b7917fd2d0113957075e19a3e5a6c1e33.
|
|
2251
|
+
* The manifest of one platform loads into kind with no index.
|
|
2252
|
+
*/
|
|
2253
|
+
const KUBE_VIP_IMAGE = "ghcr.io/kube-vip/kube-vip:v1.3.1@sha256:47353c7bde7fec21642cf621bc3fda2b999c051342912efda468999a399be667";
|
|
2254
|
+
/** This is the name of kube-vip: of its ServiceAccount, its DaemonSet and its container. */
|
|
2255
|
+
const KUBE_VIP = "kube-vip";
|
|
2256
|
+
/** This is the Lease of the election of the VIPs of the Services (`svc_leasename`, cmd/kube-vip.go of v1.3.1). */
|
|
2257
|
+
const KUBE_VIP_LEASE = {
|
|
2258
|
+
namespace: "kube-system",
|
|
2259
|
+
name: "plndr-svcs-lock"
|
|
2260
|
+
};
|
|
2261
|
+
/**
|
|
2262
|
+
* These are the times of the election, as the install sets them. Each one is the default of the
|
|
2263
|
+
* flags of v1.3.1 (cmd/kube-vip.go, lines 91–94). They are a lease of 15 s, a renew deadline of 10 s
|
|
2264
|
+
* and a retry period of 2 s. The install sets each one in the environment, thus a new default cannot
|
|
2265
|
+
* change a cell.
|
|
2266
|
+
*/
|
|
2267
|
+
const KUBE_VIP_ELECTION = {
|
|
2268
|
+
lease: {
|
|
2269
|
+
value: 15,
|
|
2270
|
+
unit: "s"
|
|
2271
|
+
},
|
|
2272
|
+
renew: {
|
|
2273
|
+
value: 10,
|
|
2274
|
+
unit: "s"
|
|
2275
|
+
},
|
|
2276
|
+
retry: {
|
|
2277
|
+
value: 2,
|
|
2278
|
+
unit: "s"
|
|
2279
|
+
}
|
|
2280
|
+
};
|
|
2281
|
+
/** This time is the interval of the gratuitous ARPs, the default of the flag `arpRate` of v1.3.1 (cmd/kube-vip.go, line 61). */
|
|
2282
|
+
const KUBE_VIP_ARP_RATE = {
|
|
2283
|
+
value: 3e3,
|
|
2284
|
+
unit: "ms"
|
|
2285
|
+
};
|
|
2286
|
+
/** This label of a node selects the nodes of kube-vip: the DaemonSet has a pod only on a node with it. */
|
|
2287
|
+
const VIP_LABEL = "lab.systems/vip";
|
|
2288
|
+
/** This label of a pod of kube-vip selects them. */
|
|
2289
|
+
const APP = { "app.kubernetes.io/name": KUBE_VIP };
|
|
2290
|
+
const whole$1 = (amount) => String(Math.round(valueIn(amount, "s")));
|
|
2291
|
+
/**
|
|
2292
|
+
* This function gives the objects of kube-vip in kube-system. They are its ServiceAccount, its
|
|
2293
|
+
* ClusterRole and binding, and its DaemonSet. The rules of the role are those of the manifest that
|
|
2294
|
+
* v1.3.1 generates (pkg/kubevip/config_generator.go, lines 95–134). The DaemonSet:
|
|
2295
|
+
* - is on the host network, with NET_ADMIN and NET_RAW. It adds the VIP to `eth0` and sends ARP.
|
|
2296
|
+
* - has a pod only on a node with `VIP_LABEL`. Thus the lab chooses the holder of the VIP.
|
|
2297
|
+
* - sets ARP mode, the Services, one election for all VIPs, and each time of the election.
|
|
2298
|
+
* - takes the image from the node (`imagePullPolicy: Never`).
|
|
2299
|
+
*/
|
|
2300
|
+
const kubeVipObjects = (image) => {
|
|
2301
|
+
const env = {
|
|
2302
|
+
vip_arp: "true",
|
|
2303
|
+
vip_interface: "eth0",
|
|
2304
|
+
port: "6443",
|
|
2305
|
+
cp_enable: "false",
|
|
2306
|
+
svc_enable: "true",
|
|
2307
|
+
svc_election: "false",
|
|
2308
|
+
svc_leasename: KUBE_VIP_LEASE.name,
|
|
2309
|
+
vip_leaderelection: "true",
|
|
2310
|
+
vip_leaseduration: whole$1(KUBE_VIP_ELECTION.lease),
|
|
2311
|
+
vip_renewdeadline: whole$1(KUBE_VIP_ELECTION.renew),
|
|
2312
|
+
vip_retryperiod: whole$1(KUBE_VIP_ELECTION.retry),
|
|
2313
|
+
vip_arpRate: String(Math.round(valueIn(KUBE_VIP_ARP_RATE, "ms")))
|
|
2314
|
+
};
|
|
2315
|
+
const namespace = KUBE_VIP_LEASE.namespace;
|
|
2316
|
+
return [
|
|
2317
|
+
{
|
|
2318
|
+
apiVersion: "v1",
|
|
2319
|
+
kind: "ServiceAccount",
|
|
2320
|
+
metadata: {
|
|
2321
|
+
name: KUBE_VIP,
|
|
2322
|
+
namespace
|
|
2323
|
+
}
|
|
2324
|
+
},
|
|
2325
|
+
{
|
|
2326
|
+
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
2327
|
+
kind: "ClusterRole",
|
|
2328
|
+
metadata: { name: "system:kube-vip-role" },
|
|
2329
|
+
rules: [
|
|
2330
|
+
{
|
|
2331
|
+
apiGroups: [""],
|
|
2332
|
+
resources: ["services/status"],
|
|
2333
|
+
verbs: ["update"]
|
|
2334
|
+
},
|
|
2335
|
+
{
|
|
2336
|
+
apiGroups: [""],
|
|
2337
|
+
resources: ["services", "endpoints"],
|
|
2338
|
+
verbs: [
|
|
2339
|
+
"list",
|
|
2340
|
+
"get",
|
|
2341
|
+
"watch",
|
|
2342
|
+
"update"
|
|
2343
|
+
]
|
|
2344
|
+
},
|
|
2345
|
+
{
|
|
2346
|
+
apiGroups: [""],
|
|
2347
|
+
resources: ["nodes"],
|
|
2348
|
+
verbs: [
|
|
2349
|
+
"list",
|
|
2350
|
+
"get",
|
|
2351
|
+
"watch",
|
|
2352
|
+
"update",
|
|
2353
|
+
"patch"
|
|
2354
|
+
]
|
|
2355
|
+
},
|
|
2356
|
+
{
|
|
2357
|
+
apiGroups: ["coordination.k8s.io"],
|
|
2358
|
+
resources: ["leases"],
|
|
2359
|
+
verbs: [
|
|
2360
|
+
"list",
|
|
2361
|
+
"get",
|
|
2362
|
+
"watch",
|
|
2363
|
+
"update",
|
|
2364
|
+
"create"
|
|
2365
|
+
]
|
|
2366
|
+
},
|
|
2367
|
+
{
|
|
2368
|
+
apiGroups: ["discovery.k8s.io"],
|
|
2369
|
+
resources: ["endpointslices"],
|
|
2370
|
+
verbs: [
|
|
2371
|
+
"list",
|
|
2372
|
+
"get",
|
|
2373
|
+
"watch",
|
|
2374
|
+
"update"
|
|
2375
|
+
]
|
|
2376
|
+
},
|
|
2377
|
+
{
|
|
2378
|
+
apiGroups: [""],
|
|
2379
|
+
resources: ["pods"],
|
|
2380
|
+
verbs: ["list"]
|
|
2381
|
+
},
|
|
2382
|
+
{
|
|
2383
|
+
apiGroups: ["networking.k8s.io"],
|
|
2384
|
+
resources: ["servicecidrs"],
|
|
2385
|
+
verbs: [
|
|
2386
|
+
"list",
|
|
2387
|
+
"get",
|
|
2388
|
+
"watch"
|
|
2389
|
+
]
|
|
2390
|
+
}
|
|
2391
|
+
]
|
|
2392
|
+
},
|
|
2393
|
+
{
|
|
2394
|
+
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
2395
|
+
kind: "ClusterRoleBinding",
|
|
2396
|
+
metadata: { name: "system:kube-vip-binding" },
|
|
2397
|
+
roleRef: {
|
|
2398
|
+
apiGroup: "rbac.authorization.k8s.io",
|
|
2399
|
+
kind: "ClusterRole",
|
|
2400
|
+
name: "system:kube-vip-role"
|
|
2401
|
+
},
|
|
2402
|
+
subjects: [{
|
|
2403
|
+
kind: "ServiceAccount",
|
|
2404
|
+
name: KUBE_VIP,
|
|
2405
|
+
namespace
|
|
2406
|
+
}]
|
|
2407
|
+
},
|
|
2408
|
+
{
|
|
2409
|
+
apiVersion: "apps/v1",
|
|
2410
|
+
kind: "DaemonSet",
|
|
2411
|
+
metadata: {
|
|
2412
|
+
name: KUBE_VIP,
|
|
2413
|
+
namespace,
|
|
2414
|
+
labels: APP
|
|
2415
|
+
},
|
|
2416
|
+
spec: {
|
|
2417
|
+
selector: { matchLabels: APP },
|
|
2418
|
+
template: {
|
|
2419
|
+
metadata: { labels: APP },
|
|
2420
|
+
spec: {
|
|
2421
|
+
serviceAccountName: KUBE_VIP,
|
|
2422
|
+
hostNetwork: true,
|
|
2423
|
+
nodeSelector: { [VIP_LABEL]: "1" },
|
|
2424
|
+
containers: [{
|
|
2425
|
+
name: KUBE_VIP,
|
|
2426
|
+
image,
|
|
2427
|
+
imagePullPolicy: "Never",
|
|
2428
|
+
args: ["manager"],
|
|
2429
|
+
env: [...Object.entries(env).map(([name, value]) => ({
|
|
2430
|
+
name,
|
|
2431
|
+
value
|
|
2432
|
+
})), {
|
|
2433
|
+
name: "vip_nodename",
|
|
2434
|
+
valueFrom: { fieldRef: { fieldPath: "spec.nodeName" } }
|
|
2435
|
+
}],
|
|
2436
|
+
securityContext: { capabilities: { add: ["NET_ADMIN", "NET_RAW"] } }
|
|
2437
|
+
}]
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
}
|
|
2441
|
+
}
|
|
2442
|
+
];
|
|
2443
|
+
};
|
|
2444
|
+
/** This function gives the Lease of the election, or null when it is not there yet. */
|
|
2445
|
+
const vipLeaseOf = async (cluster) => {
|
|
2446
|
+
const text = await cluster.kubectl([
|
|
2447
|
+
"-n",
|
|
2448
|
+
KUBE_VIP_LEASE.namespace,
|
|
2449
|
+
"get",
|
|
2450
|
+
"lease",
|
|
2451
|
+
KUBE_VIP_LEASE.name,
|
|
2452
|
+
"-o",
|
|
2453
|
+
"json",
|
|
2454
|
+
"--ignore-not-found"
|
|
2455
|
+
]);
|
|
2456
|
+
if (text.trim() === "") return null;
|
|
2457
|
+
const spec = JSON.parse(text).spec ?? {};
|
|
2458
|
+
const text0 = (key) => typeof spec[key] === "string" && spec[key] !== "" ? spec[key] : null;
|
|
2459
|
+
const number0 = (key) => typeof spec[key] === "number" ? spec[key] : null;
|
|
2460
|
+
return {
|
|
2461
|
+
holder: text0("holderIdentity"),
|
|
2462
|
+
acquireTime: text0("acquireTime"),
|
|
2463
|
+
renewTime: text0("renewTime"),
|
|
2464
|
+
leaseDurationSeconds: number0("leaseDurationSeconds"),
|
|
2465
|
+
leaseTransitions: number0("leaseTransitions")
|
|
2466
|
+
};
|
|
2467
|
+
};
|
|
2468
|
+
/** This function reads the Docker network of a cluster: its IPv4 subnet, and the MAC address of each container. */
|
|
2469
|
+
const vipNetworkOf = async (network) => {
|
|
2470
|
+
const [info] = JSON.parse(await command("docker", [
|
|
2471
|
+
"network",
|
|
2472
|
+
"inspect",
|
|
2473
|
+
network
|
|
2474
|
+
]));
|
|
2475
|
+
const subnet = (info?.IPAM?.Config ?? []).map((config) => config.Subnet ?? "").find((each) => /^\d+\.\d+\.\d+\.\d+\/\d+$/.test(each));
|
|
2476
|
+
if (subnet === void 0) throw new Error(`the Docker network ${network} has no IPv4 subnet`);
|
|
2477
|
+
const macs = {};
|
|
2478
|
+
for (const container of Object.values(info?.Containers ?? {})) if (container.Name !== void 0 && container.MacAddress !== void 0) macs[container.Name] = container.MacAddress.toLowerCase();
|
|
2479
|
+
return {
|
|
2480
|
+
subnet,
|
|
2481
|
+
macs
|
|
2482
|
+
};
|
|
2483
|
+
};
|
|
2484
|
+
/**
|
|
2485
|
+
* This function gives the VIP of a subnet: the sixth address from its end. Docker gives the address
|
|
2486
|
+
* of each container from the start of the subnet, thus no container takes it.
|
|
2487
|
+
*/
|
|
2488
|
+
const vipOf = (subnet) => {
|
|
2489
|
+
const [first, last] = cidrRange(subnet);
|
|
2490
|
+
const address = last - 5;
|
|
2491
|
+
if (address <= first + 1) throw new Error(`the subnet ${subnet} is too small for a VIP`);
|
|
2492
|
+
return [
|
|
2493
|
+
24,
|
|
2494
|
+
16,
|
|
2495
|
+
8,
|
|
2496
|
+
0
|
|
2497
|
+
].map((shift) => Math.floor(address / 2 ** shift) % 256).join(".");
|
|
2498
|
+
};
|
|
2499
|
+
const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
2500
|
+
/** This function gives the pods of kube-vip that are not deleted. */
|
|
2501
|
+
const kubeVipPods = async (cluster) => {
|
|
2502
|
+
return JSON.parse(await cluster.kubectl([
|
|
2503
|
+
"-n",
|
|
2504
|
+
KUBE_VIP_LEASE.namespace,
|
|
2505
|
+
"get",
|
|
2506
|
+
"pods",
|
|
2507
|
+
"-l",
|
|
2508
|
+
`app.kubernetes.io/name=${KUBE_VIP}`,
|
|
2509
|
+
"-o",
|
|
2510
|
+
"json"
|
|
2511
|
+
])).items.filter((pod) => pod.metadata.deletionTimestamp === void 0).map((pod) => {
|
|
2512
|
+
const statuses = pod.status?.containerStatuses ?? [];
|
|
2513
|
+
return {
|
|
2514
|
+
name: pod.metadata.name,
|
|
2515
|
+
node: pod.spec.nodeName ?? "",
|
|
2516
|
+
ready: statuses.length > 0 && statuses.every((status) => status.ready),
|
|
2517
|
+
restarts: statuses.reduce((sum, status) => sum + (status.restartCount ?? 0), 0)
|
|
2518
|
+
};
|
|
2519
|
+
});
|
|
2520
|
+
};
|
|
2521
|
+
/** This function gives the nodes with a ready pod of kube-vip that is not deleted. */
|
|
2522
|
+
const kubeVipReady = async (cluster) => (await kubeVipPods(cluster)).filter((pod) => pod.ready && pod.node !== "").map((pod) => pod.node).sort();
|
|
2523
|
+
/** This function waits until a condition gives null, and gives null, or the last reason after `within` ms. */
|
|
2524
|
+
const until = async (within, check) => {
|
|
2525
|
+
const end = Date.now() + within;
|
|
2526
|
+
let why = await check();
|
|
2527
|
+
while (why !== null && Date.now() < end) {
|
|
2528
|
+
await sleep$1(500);
|
|
2529
|
+
why = await check();
|
|
2530
|
+
}
|
|
2531
|
+
return why;
|
|
2532
|
+
};
|
|
2533
|
+
const label = (cluster, node, on) => cluster.kubectl([
|
|
2534
|
+
"label",
|
|
2535
|
+
"node",
|
|
2536
|
+
node,
|
|
2537
|
+
on ? `${VIP_LABEL}=1` : `${VIP_LABEL}-`,
|
|
2538
|
+
...on ? ["--overwrite"] : []
|
|
2539
|
+
]);
|
|
2540
|
+
/**
|
|
2541
|
+
* This function makes `holder` the holder of the VIP, with a ready pod of kube-vip on each node of
|
|
2542
|
+
* `nodes`. It gives null, or why it could not in `within` ms. If another node holds the lease, the
|
|
2543
|
+
* function takes the label from each other node. The DaemonSet then deletes their pods. The leader
|
|
2544
|
+
* gives back its lease at its stop (`ReleaseOnCancel`). The function gives each label back after
|
|
2545
|
+
* `holder` takes the lease.
|
|
2546
|
+
*
|
|
2547
|
+
* A leader that lost its lease restarted its container. The kubelet gives a container a longer
|
|
2548
|
+
* back-off at each restart, thus the function first deletes each pod of kube-vip with a restart.
|
|
2549
|
+
* The DaemonSet then makes a new pod with no back-off.
|
|
2550
|
+
*/
|
|
2551
|
+
const holdVip = async (cluster, holder, nodes, within = 18e4) => {
|
|
2552
|
+
const others = nodes.filter((node) => node !== holder);
|
|
2553
|
+
for (const pod of await kubeVipPods(cluster)) if (pod.restarts > 0) await cluster.kubectl([
|
|
2554
|
+
"-n",
|
|
2555
|
+
KUBE_VIP_LEASE.namespace,
|
|
2556
|
+
"delete",
|
|
2557
|
+
"pod",
|
|
2558
|
+
pod.name,
|
|
2559
|
+
"--wait=true",
|
|
2560
|
+
"--timeout=120s"
|
|
2561
|
+
]);
|
|
2562
|
+
await label(cluster, holder, true);
|
|
2563
|
+
const own = await until(within, async () => (await kubeVipReady(cluster)).includes(holder) ? null : `kube-vip on ${holder} is not ready`);
|
|
2564
|
+
if (own !== null) return own;
|
|
2565
|
+
const held = async () => {
|
|
2566
|
+
const lease = await vipLeaseOf(cluster);
|
|
2567
|
+
return lease?.holder === holder ? null : `the lease ${KUBE_VIP_LEASE.name} is held by ${lease?.holder ?? "no node"}, not ${holder}`;
|
|
2568
|
+
};
|
|
2569
|
+
if (await held() !== null) {
|
|
2570
|
+
for (const node of others) await label(cluster, node, false);
|
|
2571
|
+
const moved = await until(within, held);
|
|
2572
|
+
if (moved !== null) return moved;
|
|
2573
|
+
}
|
|
2574
|
+
for (const node of others) await label(cluster, node, true);
|
|
2575
|
+
return until(within, async () => {
|
|
2576
|
+
const ready = await kubeVipReady(cluster);
|
|
2577
|
+
const missing = nodes.filter((node) => !ready.includes(node));
|
|
2578
|
+
return missing.length === 0 ? null : `kube-vip is not ready on ${missing.join(", ")}`;
|
|
2579
|
+
});
|
|
2580
|
+
};
|
|
2581
|
+
/**
|
|
2582
|
+
* This function installs kube-vip in a cluster, with its image on each node (`cluster.load`), and
|
|
2583
|
+
* makes `holder` the holder of the VIP. A failed install gives the pods of kube-vip and the tail of
|
|
2584
|
+
* their logs.
|
|
2585
|
+
*/
|
|
2586
|
+
const installKubeVip = async (cluster, image, holder, nodes) => {
|
|
2587
|
+
try {
|
|
2588
|
+
await label(cluster, holder, true);
|
|
2589
|
+
await cluster.apply(kubeVipObjects(image));
|
|
2590
|
+
const why = await holdVip(cluster, holder, nodes);
|
|
2591
|
+
if (why !== null) throw new Error(why);
|
|
2592
|
+
} catch (error) {
|
|
2593
|
+
const state = async (args) => (await cluster.kubectl([
|
|
2594
|
+
"-n",
|
|
2595
|
+
KUBE_VIP_LEASE.namespace,
|
|
2596
|
+
...args
|
|
2597
|
+
]).catch((failure) => String(failure))).slice(-4e3);
|
|
2598
|
+
const pods = await state([
|
|
2599
|
+
"get",
|
|
2600
|
+
"pods",
|
|
2601
|
+
"-l",
|
|
2602
|
+
`app.kubernetes.io/name=${KUBE_VIP}`,
|
|
2603
|
+
"-o",
|
|
2604
|
+
"wide"
|
|
2605
|
+
]);
|
|
2606
|
+
const logs = await state([
|
|
2607
|
+
"logs",
|
|
2608
|
+
"-l",
|
|
2609
|
+
`app.kubernetes.io/name=${KUBE_VIP}`,
|
|
2610
|
+
"--tail=40",
|
|
2611
|
+
"--prefix"
|
|
2612
|
+
]);
|
|
2613
|
+
throw new Error(`the install of kube-vip failed: ${String(error)}\nthe pods of kube-vip:\n${pods}\nthe logs:\n${logs}`);
|
|
2614
|
+
}
|
|
2615
|
+
};
|
|
2616
|
+
/**
|
|
2617
|
+
* This function waits until the Service of a VIP has the VIP in its status: kube-vip took it. It
|
|
2618
|
+
* gives null, or why not in `within` ms.
|
|
2619
|
+
*/
|
|
2620
|
+
const vipServed = (cluster, namespace, service, vip, within = 12e4) => until(within, async () => {
|
|
2621
|
+
const ingress = (await cluster.kubectl([
|
|
2622
|
+
"-n",
|
|
2623
|
+
namespace,
|
|
2624
|
+
"get",
|
|
2625
|
+
"service",
|
|
2626
|
+
service,
|
|
2627
|
+
"-o",
|
|
2628
|
+
"jsonpath={.status.loadBalancer.ingress[*].ip}"
|
|
2629
|
+
])).trim();
|
|
2630
|
+
return ingress.split(/\s+/).includes(vip) ? null : `the Service ${service} has the ingress ${ingress || "none"}, not ${vip}`;
|
|
2631
|
+
});
|
|
2632
|
+
//#endregion
|
|
2189
2633
|
//#region src/lds.ts
|
|
2190
2634
|
/**
|
|
2191
2635
|
* This filter of RE2 selects the two counters of the listener manager of Envoy that an update of LDS
|
|
@@ -2298,6 +2742,11 @@ const openKindLab = async (cluster, image) => {
|
|
|
2298
2742
|
const opened = [];
|
|
2299
2743
|
let watcher = null;
|
|
2300
2744
|
let prober = null;
|
|
2745
|
+
let outsiders = [];
|
|
2746
|
+
const stopOutside = async () => {
|
|
2747
|
+
for (const container of outsiders) await container.stop({ timeout: 0 }).catch(() => void 0);
|
|
2748
|
+
outsiders = [];
|
|
2749
|
+
};
|
|
2301
2750
|
try {
|
|
2302
2751
|
for (const node of cluster.nodes) {
|
|
2303
2752
|
const agent = await new Shared(image).withPidMode(`container:${node}`).withPrivilegedMode().withNetworkMode("none").withEnvironment({ NODE_NAME: node }).withCommand([
|
|
@@ -2364,7 +2813,8 @@ const openKindLab = async (cluster, image) => {
|
|
|
2364
2813
|
return [
|
|
2365
2814
|
...opened,
|
|
2366
2815
|
...watcher === null ? [] : [watcher],
|
|
2367
|
-
...prober === null ? [] : [prober]
|
|
2816
|
+
...prober === null ? [] : [prober],
|
|
2817
|
+
...outsiders
|
|
2368
2818
|
].map((container) => container.getName().replace(/^\//, ""));
|
|
2369
2819
|
},
|
|
2370
2820
|
take() {
|
|
@@ -2411,6 +2861,21 @@ const openKindLab = async (cluster, image) => {
|
|
|
2411
2861
|
})).start();
|
|
2412
2862
|
await Promise.race([ready, new Promise((_, reject) => setTimeout(() => reject(/* @__PURE__ */ new Error("no begin line from the prober in 60 s")), 6e4))]);
|
|
2413
2863
|
},
|
|
2864
|
+
async outside(spec) {
|
|
2865
|
+
await stopOutside();
|
|
2866
|
+
if (spec === null) return;
|
|
2867
|
+
for (let k = 0; k < spec.count; k += 1) {
|
|
2868
|
+
const client = await new GenericContainer(image).withNetworkMode(cluster.network).withEnvironment({
|
|
2869
|
+
TARGET: spec.target,
|
|
2870
|
+
EVERY_MS: String(valueIn(spec.every, "ms")),
|
|
2871
|
+
TIMEOUT_MS: String(valueIn(spec.timeout, "ms")),
|
|
2872
|
+
POD_NAME: `${OUTSIDE}-${k}`,
|
|
2873
|
+
NODE_NAME: OUTSIDE,
|
|
2874
|
+
ARP_WATCH: spec.arp
|
|
2875
|
+
}).withCommand(["node", "client.mjs"]).withLogConsumer(consume((line) => push(line))).start();
|
|
2876
|
+
outsiders.push(client);
|
|
2877
|
+
}
|
|
2878
|
+
},
|
|
2414
2879
|
async stopProbe() {
|
|
2415
2880
|
if (prober !== null) await prober.stop({ timeout: 0 }).catch(() => void 0);
|
|
2416
2881
|
prober = null;
|
|
@@ -2597,6 +3062,7 @@ const openKindLab = async (cluster, image) => {
|
|
|
2597
3062
|
},
|
|
2598
3063
|
async close() {
|
|
2599
3064
|
await lab.stopProbe();
|
|
3065
|
+
await stopOutside();
|
|
2600
3066
|
if (watcher !== null) await watcher.stop({ timeout: 0 }).catch(() => void 0);
|
|
2601
3067
|
watcher = null;
|
|
2602
3068
|
for (const container of opened.reverse()) await container.stop({ timeout: 0 }).catch(() => void 0);
|
|
@@ -2816,14 +3282,53 @@ const preStopOf = (preStop, grace) => {
|
|
|
2816
3282
|
`sleep ${preStop}`
|
|
2817
3283
|
] } } } };
|
|
2818
3284
|
};
|
|
3285
|
+
/** This affinity keeps a pod in the zone that a zone loss keeps. */
|
|
3286
|
+
const keptZone = { nodeAffinity: { requiredDuringSchedulingIgnoredDuringExecution: { nodeSelectorTerms: [{ matchExpressions: [{
|
|
3287
|
+
key: ZONE_LABEL,
|
|
3288
|
+
operator: "NotIn",
|
|
3289
|
+
values: [ZONE_LOST]
|
|
3290
|
+
}] }] } } };
|
|
3291
|
+
/**
|
|
3292
|
+
* This function gives the Service of a VIP (`kubevip.ts`). It is of type LoadBalancer, with the
|
|
3293
|
+
* class of kube-vip and the address of the VIP in its annotation. The policy `Cluster` lets each node
|
|
3294
|
+
* send a request to a replica on another node. The holder writes the address to the status of the
|
|
3295
|
+
* Service. Then kube-proxy has the rules of the address on each node.
|
|
3296
|
+
*/
|
|
3297
|
+
const vipService = (namespace, vip, ports) => ({
|
|
3298
|
+
apiVersion: "v1",
|
|
3299
|
+
kind: "Service",
|
|
3300
|
+
metadata: {
|
|
3301
|
+
name: "lab",
|
|
3302
|
+
namespace,
|
|
3303
|
+
annotations: { [VIP_ANNOTATION]: vip }
|
|
3304
|
+
},
|
|
3305
|
+
spec: {
|
|
3306
|
+
type: "LoadBalancer",
|
|
3307
|
+
loadBalancerClass: LB_CLASS,
|
|
3308
|
+
externalTrafficPolicy: "Cluster",
|
|
3309
|
+
selector: { app: "lab" },
|
|
3310
|
+
ports
|
|
3311
|
+
}
|
|
3312
|
+
});
|
|
2819
3313
|
/**
|
|
2820
3314
|
* These are the objects of the server: a Service of type ClusterIP, and the Deployment with the
|
|
2821
3315
|
* settings of the cell. The image is the lab image for the service, or the image of grpc-go.
|
|
3316
|
+
*
|
|
3317
|
+
* With a VIP (`KindCell.vip`), the Service is of type LoadBalancer, with the address `vip` and the
|
|
3318
|
+
* class of kube-vip (`vipService`). No replica is in the lost zone. Thus a zone loss takes away
|
|
3319
|
+
* only the holder of the VIP.
|
|
2822
3320
|
*/
|
|
2823
|
-
const serverObjects = (cell, namespace, image, version = 1) => {
|
|
3321
|
+
const serverObjects = (cell, namespace, image, version = 1, vip = null) => {
|
|
2824
3322
|
const preStop = whole(cell.pod.preStop, "the preStop sleep");
|
|
2825
3323
|
const grace = whole(cell.pod.grace, "the grace period");
|
|
2826
|
-
|
|
3324
|
+
if (cell.vip !== void 0 && vip === null) throw new Error(`${cell.name}: a cell with a VIP needs the address of the VIP`);
|
|
3325
|
+
const ports = [{
|
|
3326
|
+
name: stackOf(cell) === "grpc" ? "grpc" : "http",
|
|
3327
|
+
port: 80,
|
|
3328
|
+
targetPort: PORT
|
|
3329
|
+
}];
|
|
3330
|
+
const zoneLoss = cell.trigger.kind === "zone-loss";
|
|
3331
|
+
return [vip === null ? {
|
|
2827
3332
|
apiVersion: "v1",
|
|
2828
3333
|
kind: "Service",
|
|
2829
3334
|
metadata: {
|
|
@@ -2832,13 +3337,9 @@ const serverObjects = (cell, namespace, image, version = 1) => {
|
|
|
2832
3337
|
},
|
|
2833
3338
|
spec: {
|
|
2834
3339
|
selector: { app: "lab" },
|
|
2835
|
-
ports
|
|
2836
|
-
name: stackOf(cell) === "grpc" ? "grpc" : "http",
|
|
2837
|
-
port: 80,
|
|
2838
|
-
targetPort: PORT
|
|
2839
|
-
}]
|
|
3340
|
+
ports
|
|
2840
3341
|
}
|
|
2841
|
-
}, {
|
|
3342
|
+
} : vipService(namespace, vip, ports), {
|
|
2842
3343
|
apiVersion: "apps/v1",
|
|
2843
3344
|
kind: "Deployment",
|
|
2844
3345
|
metadata: {
|
|
@@ -2871,12 +3372,13 @@ const serverObjects = (cell, namespace, image, version = 1) => {
|
|
|
2871
3372
|
topologyKey: "kubernetes.io/hostname",
|
|
2872
3373
|
whenUnsatisfiable: "ScheduleAnyway",
|
|
2873
3374
|
labelSelector: { matchLabels: { app: "lab" } }
|
|
2874
|
-
}, ...cell.
|
|
3375
|
+
}, ...zoneLoss && cell.vip === void 0 ? [{
|
|
2875
3376
|
maxSkew: 1,
|
|
2876
3377
|
topologyKey: ZONE_LABEL,
|
|
2877
3378
|
whenUnsatisfiable: "DoNotSchedule",
|
|
2878
3379
|
labelSelector: { matchLabels: { app: "lab" } }
|
|
2879
3380
|
}] : []],
|
|
3381
|
+
...zoneLoss && cell.vip !== void 0 ? { affinity: keptZone } : {},
|
|
2880
3382
|
containers: [{
|
|
2881
3383
|
name: "app",
|
|
2882
3384
|
image,
|
|
@@ -2918,11 +3420,7 @@ const clientObjects = (cell, namespace, image, target) => [{
|
|
|
2918
3420
|
operator: "Exists",
|
|
2919
3421
|
effect: "NoSchedule"
|
|
2920
3422
|
}],
|
|
2921
|
-
...cell.trigger.kind === "zone-loss" ? { affinity:
|
|
2922
|
-
key: ZONE_LABEL,
|
|
2923
|
-
operator: "NotIn",
|
|
2924
|
-
values: [ZONE_LOST]
|
|
2925
|
-
}] }] } } } } : {},
|
|
3423
|
+
...cell.trigger.kind === "zone-loss" ? { affinity: keptZone } : {},
|
|
2926
3424
|
containers: [{
|
|
2927
3425
|
name: CLIENT,
|
|
2928
3426
|
image,
|
|
@@ -3183,7 +3681,7 @@ const scenarioOf = (cell, propagation) => ({
|
|
|
3183
3681
|
* - A deletion, a rolling update, a node drain, a scale-in and a rollout delete a pod with its grace.
|
|
3184
3682
|
* - In T07 a pause of kube-proxy comes before a deletion. In T18 the pause is on Redis.
|
|
3185
3683
|
* - The crash of the lab is SIGKILL from the node. Like a crash, it has no preStop hook and no drain.
|
|
3186
|
-
* - The other faults are on a link, on Redis or on the process. The kubelet can restart a pod after them.
|
|
3684
|
+
* - The other faults are on a link, on Redis, on the DNS of the clients or on the process. The kubelet can restart a pod after them.
|
|
3187
3685
|
* A zone loss freezes the pods of its node and deletes none of them in its hold.
|
|
3188
3686
|
*/
|
|
3189
3687
|
const stopOf = (cell) => {
|
|
@@ -3207,7 +3705,8 @@ const stopOf = (cell) => {
|
|
|
3207
3705
|
case "certificate-expiry":
|
|
3208
3706
|
case "cannot-serve":
|
|
3209
3707
|
case "zone-loss":
|
|
3210
|
-
case "latency":
|
|
3708
|
+
case "latency":
|
|
3709
|
+
case "dns-blackhole": return null;
|
|
3211
3710
|
}
|
|
3212
3711
|
};
|
|
3213
3712
|
/**
|
|
@@ -3250,7 +3749,7 @@ const hasRedis = (cell) => cell.declaration !== null && cell.declaration.redis !
|
|
|
3250
3749
|
* This function gives the key of the cluster of a cell: its kube-proxy mode, its node count, and the
|
|
3251
3750
|
* flavor of its mesh. A zone loss has clusters of its own, with a zone on each node.
|
|
3252
3751
|
*/
|
|
3253
|
-
const clusterKey = (cell) => cell.mesh !== void 0 ? `${cell.mode}-${cell.workers}-${cell.mesh.flavor}` : zoned(cell) ? `${cell.mode}-${cell.workers}-zones` : `${cell.mode}-${cell.workers}
|
|
3752
|
+
const clusterKey = (cell) => (cell.mesh !== void 0 ? `${cell.mode}-${cell.workers}-${cell.mesh.flavor}` : zoned(cell) ? `${cell.mode}-${cell.workers}-zones` : `${cell.mode}-${cell.workers}`) + (cell.vip === void 0 ? "" : "-vip");
|
|
3254
3753
|
/** This function tells if a cell needs a cluster with zones: a zone loss. */
|
|
3255
3754
|
const zoned = (cell) => cell.trigger.kind === "zone-loss";
|
|
3256
3755
|
/**
|
|
@@ -3315,8 +3814,12 @@ const unsteady = (ctx) => {
|
|
|
3315
3814
|
for (const node of ctx.cluster.nodes) if ((tracker.hosts.get(node) ?? []).join() !== expected) return `the client sidecar of ${node} has ${(tracker.hosts.get(node) ?? []).join(" ") || "no host"}`;
|
|
3316
3815
|
}
|
|
3317
3816
|
const nodes = new Set([...tracker.clients.values()].map((client) => client.node));
|
|
3318
|
-
const clientNodes = ctx.cluster.nodes.length - (cell.trigger.kind === "zone-loss" ? 1 : 0);
|
|
3817
|
+
const clientNodes = ctx.cluster.nodes.length - (cell.trigger.kind === "zone-loss" ? 1 : 0) + (cell.vip === void 0 ? 0 : 1);
|
|
3319
3818
|
if (nodes.size !== clientNodes) return `clients answer on ${nodes.size} nodes, not ${clientNodes}`;
|
|
3819
|
+
if (cell.vip !== void 0) {
|
|
3820
|
+
const outside = [...tracker.clients.values()].filter((client) => client.node === OUTSIDE).length;
|
|
3821
|
+
if (outside !== cell.vip.clients) return `${outside} clients outside answer, not ${cell.vip.clients}`;
|
|
3822
|
+
}
|
|
3320
3823
|
for (const [pod, client] of tracker.clients) {
|
|
3321
3824
|
if (tracker.last - client.ok > 1e3) return `client ${pod} has no answer in the last second`;
|
|
3322
3825
|
if (tracker.last - client.failed < 2e3) return `client ${pod} failed in the last 2 s`;
|
|
@@ -3419,7 +3922,7 @@ const quiet = async (ctx, ours, within = QUIET_WITHIN) => {
|
|
|
3419
3922
|
* clients. A client of gRPC starts after the server is ready: each failed connect of a channel makes
|
|
3420
3923
|
* the backoff of grpc-js longer, up to 120 s.
|
|
3421
3924
|
*/
|
|
3422
|
-
const deploy = async (lab, cluster, cell, namespace, images) => {
|
|
3925
|
+
const deploy = async (lab, cluster, cell, namespace, images, vip) => {
|
|
3423
3926
|
await cluster.apply([namespaceObject(cell, namespace)]);
|
|
3424
3927
|
await lab.watch([namespace], { ztunnel: followsZtunnel(cell) });
|
|
3425
3928
|
if (hasRedis(cell)) {
|
|
@@ -3435,7 +3938,7 @@ const deploy = async (lab, cluster, cell, namespace, images) => {
|
|
|
3435
3938
|
}
|
|
3436
3939
|
const grpc = stackOf(cell) === "grpc";
|
|
3437
3940
|
if (grpc && images.grpc === void 0) throw new Error(`${cell.name}: the cluster has no image of grpc-go`);
|
|
3438
|
-
await cluster.apply(serverObjects(cell, namespace, grpc ? images.grpc : lab.image));
|
|
3941
|
+
await cluster.apply(serverObjects(cell, namespace, grpc ? images.grpc : lab.image, 1, cell.vip === void 0 ? null : vip));
|
|
3439
3942
|
const clusterIp = (await cluster.kubectl([
|
|
3440
3943
|
"-n",
|
|
3441
3944
|
namespace,
|
|
@@ -3445,7 +3948,7 @@ const deploy = async (lab, cluster, cell, namespace, images) => {
|
|
|
3445
3948
|
"-o",
|
|
3446
3949
|
"jsonpath={.spec.clusterIP}"
|
|
3447
3950
|
])).trim();
|
|
3448
|
-
const clients = clientObjects(cell, namespace, lab.image, `${clusterIp}:80`);
|
|
3951
|
+
const clients = clientObjects(cell, namespace, lab.image, `${cell.workload.byName === true ? "lab" : clusterIp}:80`);
|
|
3449
3952
|
if (!grpc) await cluster.apply(clients);
|
|
3450
3953
|
await cluster.kubectl([
|
|
3451
3954
|
"-n",
|
|
@@ -3464,6 +3967,18 @@ const deploy = async (lab, cluster, cell, namespace, images) => {
|
|
|
3464
3967
|
`daemonset/${CLIENT}`,
|
|
3465
3968
|
"--timeout=300s"
|
|
3466
3969
|
]);
|
|
3970
|
+
if (cell.vip !== void 0) {
|
|
3971
|
+
if (vip === null) throw new Error(`${cell.name}: the cluster has no VIP`);
|
|
3972
|
+
const served = await vipServed(cluster, namespace, "lab", vip);
|
|
3973
|
+
if (served !== null) throw new Error(`${cell.name}: ${served}`);
|
|
3974
|
+
await lab.outside({
|
|
3975
|
+
target: `${vip}:80`,
|
|
3976
|
+
every: cell.workload.every,
|
|
3977
|
+
timeout: cell.workload.timeout,
|
|
3978
|
+
count: cell.vip.clients,
|
|
3979
|
+
arp: vip
|
|
3980
|
+
});
|
|
3981
|
+
}
|
|
3467
3982
|
await lab.rules({
|
|
3468
3983
|
mode: cluster.scenario.dataplane,
|
|
3469
3984
|
ns: namespace,
|
|
@@ -3847,6 +4362,16 @@ const doStep = async (ctx, step, replicas, background) => {
|
|
|
3847
4362
|
t1: await lab.now()
|
|
3848
4363
|
};
|
|
3849
4364
|
}
|
|
4365
|
+
case "dns": {
|
|
4366
|
+
if (step.target !== "clients") throw new Error(`a drop of DNS takes the target clients, not ${step.target}`);
|
|
4367
|
+
const each = await namespacesOf(ctx, step.target, replicas);
|
|
4368
|
+
const t0 = await lab.now();
|
|
4369
|
+
await Promise.all(each.map(({ node, pid }) => lab.netns(node, pid, dnsCommands(step.on))));
|
|
4370
|
+
return {
|
|
4371
|
+
t0,
|
|
4372
|
+
t1: await lab.now()
|
|
4373
|
+
};
|
|
4374
|
+
}
|
|
3850
4375
|
case "certificates": {
|
|
3851
4376
|
const t0 = await lab.now();
|
|
3852
4377
|
const certificates = await ztunnelCertificates(cluster).catch((error) => String(error).slice(0, 400));
|
|
@@ -3890,6 +4415,39 @@ const doStep = async (ctx, step, replicas, background) => {
|
|
|
3890
4415
|
};
|
|
3891
4416
|
/** This function gives the node of the lost zone: the last worker (`zoneOf`). */
|
|
3892
4417
|
const lostNodeOf = (cluster) => cluster.nodes.at(-1);
|
|
4418
|
+
/** This function gives the workers of a cluster: each node but the control plane, the first (`nodeNames`). */
|
|
4419
|
+
const workersOf = (cluster) => cluster.nodes.slice(1);
|
|
4420
|
+
/**
|
|
4421
|
+
* This function adds a line `vip` to the lines. The line has the Lease of the VIP and the MAC address
|
|
4422
|
+
* of each node on the Docker network. It also has the wall clock of the runner with its CLOCK_MONOTONIC. The times of the
|
|
4423
|
+
* Lease are on the wall clock, and each stamp of the lab is on CLOCK_MONOTONIC. The containers of
|
|
4424
|
+
* the runner share one kernel, thus the pair converts one into the other.
|
|
4425
|
+
*/
|
|
4426
|
+
const recordVip = async (ctx) => {
|
|
4427
|
+
const wall = Date.now();
|
|
4428
|
+
const t = Number(process.hrtime.bigint()) / 1e6;
|
|
4429
|
+
const lease = await vipLeaseOf(ctx.cluster);
|
|
4430
|
+
const { macs } = await vipNetworkOf(ctx.cluster.network);
|
|
4431
|
+
const log = await ctx.cluster.kubectl([
|
|
4432
|
+
"-n",
|
|
4433
|
+
KUBE_VIP_LEASE.namespace,
|
|
4434
|
+
"logs",
|
|
4435
|
+
"-l",
|
|
4436
|
+
"app.kubernetes.io/name=kube-vip",
|
|
4437
|
+
"--tail=30",
|
|
4438
|
+
"--prefix",
|
|
4439
|
+
"--timestamps"
|
|
4440
|
+
]).catch((error) => String(error));
|
|
4441
|
+
ctx.lab.record({
|
|
4442
|
+
ev: "vip",
|
|
4443
|
+
t,
|
|
4444
|
+
wall,
|
|
4445
|
+
vip: ctx.vip,
|
|
4446
|
+
lease,
|
|
4447
|
+
macs,
|
|
4448
|
+
log: log.slice(-6e3)
|
|
4449
|
+
});
|
|
4450
|
+
};
|
|
3893
4451
|
/** This function puts the replicas of a node first, and keeps the sequence of the others. */
|
|
3894
4452
|
const lostFirst = (replicas, node) => [...replicas.filter((replica) => replica.node === node), ...replicas.filter((replica) => replica.node !== node)];
|
|
3895
4453
|
/** This function tells if a run is over: the trigger is done, and the cell is at rest again. */
|
|
@@ -3921,7 +4479,6 @@ const stoppedOf = (cell, replicas, target, lines) => deletes(cell) ? replicas.fi
|
|
|
3921
4479
|
const runOnce = async (ctx, env, index, warmup, ours) => {
|
|
3922
4480
|
const { lab, cell, cluster, namespace } = ctx;
|
|
3923
4481
|
const plan = planOf(cell);
|
|
3924
|
-
const atRest = await steady(ctx, 3e5);
|
|
3925
4482
|
const empty = (why, start) => ({
|
|
3926
4483
|
env,
|
|
3927
4484
|
index,
|
|
@@ -3940,6 +4497,7 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
|
|
|
3940
4497
|
lines: lab.take(),
|
|
3941
4498
|
dropped: {}
|
|
3942
4499
|
});
|
|
4500
|
+
const atRest = await steady(ctx, 3e5);
|
|
3943
4501
|
if (atRest !== null) return empty(`the cell was not at rest: ${atRest}`, await lab.now());
|
|
3944
4502
|
await probeThePods(ctx);
|
|
3945
4503
|
await pollListeners(ctx);
|
|
@@ -3951,11 +4509,16 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
|
|
|
3951
4509
|
const zoneLoss = cell.trigger.kind === "zone-loss";
|
|
3952
4510
|
const lost = zoneLoss ? await nodeSteady(cluster, lostNodeOf(cluster), 3e5) : null;
|
|
3953
4511
|
if (lost !== null) return empty(`the node of the lost zone was not at rest: ${lost}`, await lab.now());
|
|
4512
|
+
if (cell.vip !== void 0) {
|
|
4513
|
+
const held = await holdVip(cluster, lostNodeOf(cluster), workersOf(cluster));
|
|
4514
|
+
if (held !== null) return empty(`the VIP is not on ${lostNodeOf(cluster)}: ${held}`, await lab.now());
|
|
4515
|
+
await sleep(valueIn(KUBE_VIP_ARP_RATE, "ms") + valueIn(cell.workload.timeout, "ms") + 1e3);
|
|
4516
|
+
}
|
|
3954
4517
|
if (await steady(ctx, 3e5) !== null) return empty("the cell was not at rest after the wait for a quiet VM", await lab.now());
|
|
3955
4518
|
const replicas = zoneLoss ? lostFirst(replicasOf(ctx), lostNodeOf(cluster)) : replicasOf(ctx);
|
|
3956
|
-
if (zoneLoss && replicas[0]?.node !== lostNodeOf(cluster)) return empty(`no replica is on the node of the lost zone, ${lostNodeOf(cluster)}`, await lab.now());
|
|
4519
|
+
if (zoneLoss && cell.vip === void 0 && replicas[0]?.node !== lostNodeOf(cluster)) return empty(`no replica is on the node of the lost zone, ${lostNodeOf(cluster)}`, await lab.now());
|
|
3957
4520
|
const restarts = new Map(ctx.tracker.servers().map((pod) => [pod.name, pod.cs.reduce((sum, status) => sum + status.rc, 0)]));
|
|
3958
|
-
const target = hasRedis(cell) && [
|
|
4521
|
+
const target = cell.vip !== void 0 || hasRedis(cell) && [
|
|
3959
4522
|
"rst",
|
|
3960
4523
|
"blackhole",
|
|
3961
4524
|
"freeze-process",
|
|
@@ -4055,6 +4618,7 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
|
|
|
4055
4618
|
}
|
|
4056
4619
|
const end = await lab.now();
|
|
4057
4620
|
const timedOut = Date.now() >= limit;
|
|
4621
|
+
if (cell.vip !== void 0) await recordVip(ctx).catch((error) => notes.push(`the record of the VIP failed: ${String(error)}`));
|
|
4058
4622
|
for (const step of plan.restore) try {
|
|
4059
4623
|
await doStep(ctx, step, replicas, background);
|
|
4060
4624
|
} catch (error) {
|
|
@@ -4172,6 +4736,7 @@ const runKindCells = async (cells, options) => {
|
|
|
4172
4736
|
operator: await localPin(CILIUM_IMAGES.operator),
|
|
4173
4737
|
envoy: await localPin(CILIUM_IMAGES.envoy)
|
|
4174
4738
|
} : void 0;
|
|
4739
|
+
const kubeVip = cells.some((cell) => cell.vip !== void 0) ? await localPin(KUBE_VIP_IMAGE) : void 0;
|
|
4175
4740
|
const images = {
|
|
4176
4741
|
lab: image,
|
|
4177
4742
|
redis: redisImage,
|
|
@@ -4217,6 +4782,15 @@ const runKindCells = async (cells, options) => {
|
|
|
4217
4782
|
await installMesh(cluster, mesh.flavor, images.mesh);
|
|
4218
4783
|
options.log(` Istio ${mesh.flavor} installed in ${Math.round((Date.now() - began) / 100) / 10} s`);
|
|
4219
4784
|
}
|
|
4785
|
+
let vip = null;
|
|
4786
|
+
if (group[0].vip !== void 0) {
|
|
4787
|
+
if (kubeVip === void 0) throw new Error(`${group[0].name}: the run has no image of kube-vip`);
|
|
4788
|
+
await cluster.load(kubeVip);
|
|
4789
|
+
const began = Date.now();
|
|
4790
|
+
await installKubeVip(cluster, kubeVip, lostNodeOf(cluster), workersOf(cluster));
|
|
4791
|
+
vip = vipOf((await vipNetworkOf(cluster.network)).subnet);
|
|
4792
|
+
options.log(` kube-vip installed in ${Math.round((Date.now() - began) / 100) / 10} s, the VIP ${vip}`);
|
|
4793
|
+
}
|
|
4220
4794
|
lab = await openKindLab(cluster, image);
|
|
4221
4795
|
const { GenericContainer } = await import("testcontainers");
|
|
4222
4796
|
const Shared = class extends GenericContainer {
|
|
@@ -4237,7 +4811,7 @@ const runKindCells = async (cells, options) => {
|
|
|
4237
4811
|
const ztunnelEnv = cell.mesh?.ztunnel;
|
|
4238
4812
|
try {
|
|
4239
4813
|
if (ztunnelEnv !== void 0) await setZtunnelEnv(cluster, ztunnelEnv, false);
|
|
4240
|
-
const clusterIp = await deploy(lab, cluster, cell, namespace, images);
|
|
4814
|
+
const clusterIp = await deploy(lab, cluster, cell, namespace, images, vip);
|
|
4241
4815
|
for (const name of [...lab.own(), loader.getName().replace(/^\//, "")]) ours.add(name);
|
|
4242
4816
|
const ctx = {
|
|
4243
4817
|
lab,
|
|
@@ -4246,6 +4820,7 @@ const runKindCells = async (cells, options) => {
|
|
|
4246
4820
|
namespace,
|
|
4247
4821
|
tracker,
|
|
4248
4822
|
clusterIp,
|
|
4823
|
+
vip: cell.vip === void 0 ? null : vip,
|
|
4249
4824
|
loads,
|
|
4250
4825
|
loader,
|
|
4251
4826
|
idle: options.idle ?? .6,
|
|
@@ -4274,7 +4849,8 @@ const runKindCells = async (cells, options) => {
|
|
|
4274
4849
|
redis: redisImage,
|
|
4275
4850
|
...stackOf(cell) === "grpc" ? { grpc: grpcImage } : {},
|
|
4276
4851
|
...cell.mesh === void 0 ? {} : Object.fromEntries(MESH_IMAGES[cell.mesh.flavor].map((name) => [`istio-${name}`, ISTIO_IMAGES[name]])),
|
|
4277
|
-
...s.cni === "cilium" ? Object.fromEntries(Object.entries(CILIUM_IMAGES).map(([name, ref]) => [`cilium-${name}`, ref])) : {}
|
|
4852
|
+
...s.cni === "cilium" ? Object.fromEntries(Object.entries(CILIUM_IMAGES).map(([name, ref]) => [`cilium-${name}`, ref])) : {},
|
|
4853
|
+
...cell.vip === void 0 ? {} : { "kube-vip": KUBE_VIP_IMAGE }
|
|
4278
4854
|
}
|
|
4279
4855
|
});
|
|
4280
4856
|
entry.loads.push(...loads);
|
|
@@ -4288,6 +4864,7 @@ const runKindCells = async (cells, options) => {
|
|
|
4288
4864
|
}));
|
|
4289
4865
|
} finally {
|
|
4290
4866
|
off();
|
|
4867
|
+
await lab.outside(null).catch(() => void 0);
|
|
4291
4868
|
await lab.rules(null).catch(() => void 0);
|
|
4292
4869
|
await lab.eds(null).catch(() => void 0);
|
|
4293
4870
|
await lab.lds(null).catch(() => void 0);
|
|
@@ -4347,8 +4924,8 @@ const estimate = (cell, environments = 2) => {
|
|
|
4347
4924
|
const observed = after === null ? 0 : hold + valueIn(after, "s");
|
|
4348
4925
|
const closes = cell.prediction.kind === "ends" && cell.prediction.band !== null ? valueIn(cell.prediction.band.hi, "s") + 3 : 0;
|
|
4349
4926
|
const quiet = cell.trigger.quiet === true ? hold + 10 : 0;
|
|
4350
|
-
const each = 25 + Math.max(observed, closes, quiet, trigger[cell.trigger.kind] ?? (cell.template === "T18" ? hold + 35 : 15 + valueIn(cell.pod.preStop, "s")));
|
|
4351
|
-
return (cell.mesh === void 0 ? 60 : 180) * environments + (cell.runs + environments) * each;
|
|
4927
|
+
const each = 25 + (cell.vip === void 0 ? 0 : 30) + Math.max(observed, closes, quiet, trigger[cell.trigger.kind] ?? (cell.template === "T18" ? hold + 35 : 15 + valueIn(cell.pod.preStop, "s")));
|
|
4928
|
+
return ((cell.mesh === void 0 ? 60 : 180) + (cell.vip === void 0 ? 0 : 60)) * environments + (cell.runs + environments) * each;
|
|
4352
4929
|
};
|
|
4353
4930
|
/**
|
|
4354
4931
|
* This function gives an estimate of the time of a cell: its setup, then each run and each
|
|
@@ -4379,6 +4956,6 @@ const shardOf = (cells, index, count) => {
|
|
|
4379
4956
|
return out;
|
|
4380
4957
|
};
|
|
4381
4958
|
//#endregion
|
|
4382
|
-
export { ACCESS_LOG_FORMAT, CHANNEL_USE, CILIUM_CHART, CILIUM_CONFIG_KEYS, CILIUM_IMAGES, CILIUM_VERSION, CILIUM_WORKLOADS, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DATAPLANES, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, HELM, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, MIDDLEBOX, NETEM_DRAIN, NETNS_DEFAULTS, NODE_IMAGES, NODE_LIFECYCLE, O04, O09, O10, O16, POD_LINK, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, SPREAD_LEAD, STAGES, SYSCTL, TCP_STATE, WATCHDOG, ZONE_KEPT, ZONE_LABEL, ZONE_LOST, ZTUNNEL_PODS, backendsAt, backendsOf, backlogEnd, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, certificateArgs, checkRun, cidrOverlap, cidrRange, ciliumBackendsOf, ciliumMode, ciliumValues, clientObjects, clusterFilter, compose, createCluster, cyclesOf, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, drainBand, dropOf, emptyOf, ensureChart, ensureTool, estimatedTime, factsOf, failuresOf, faultSpan, followsZtunnel, goDuration, hostsOf, installCilium, installMesh, intakeAfter, intakeOf, isKubeProxyMode, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, labSource, latchBand, leafCertificates, learnWindow, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, middleboxCommands, namesService, namespaceObject, namespaceOf, nodeLossBand, nodeNames, nodeReturnBand, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, overshootBand, pausedWindow, periodOf, placementOf, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, sessionsAt, setZtunnelEnv, sha256, shardOf, sharedGate, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stopOf, stoppingOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, writeResult, zoneOf, ztunnelCertificates, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
|
|
4959
|
+
export { ACCESS_LOG_FORMAT, CHANNEL_USE, CILIUM_CHART, CILIUM_CONFIG_KEYS, CILIUM_IMAGES, CILIUM_VERSION, CILIUM_WORKLOADS, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DATAPLANES, DNS_PORT, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, HELM, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, KUBE_VIP, KUBE_VIP_ARP_RATE, KUBE_VIP_ELECTION, KUBE_VIP_IMAGE, KUBE_VIP_LEASE, KUBE_VIP_VERSION, LAST_SLACK, LB_CLASS, LDS_EVERY, LDS_FILTER, LOG_LABEL, MESH_IMAGES, MESH_WORKLOADS, MIDDLEBOX, NETEM_DRAIN, NETNS_DEFAULTS, NODE_IMAGES, NODE_LIFECYCLE, O04, O09, O10, O16, OUTSIDE, POD_LINK, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, SPREAD_LEAD, STAGES, SYSCTL, TCP_STATE, VIP_ANNOTATION, VIP_LABEL, WATCHDOG, ZONE_KEPT, ZONE_LABEL, ZONE_LOST, ZTUNNEL_PODS, backendsAt, backendsOf, backlogEnd, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, certificateArgs, checkRun, cidrOverlap, cidrRange, ciliumBackendsOf, ciliumMode, ciliumValues, clientObjects, clusterFilter, compose, createCluster, cyclesOf, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dnsCommands, drainBand, dropOf, emptyOf, ensureChart, ensureTool, estimatedTime, factsOf, failuresOf, faultSpan, followsZtunnel, goDuration, holdVip, hostsOf, installCilium, installKubeVip, installMesh, intakeAfter, intakeOf, isKubeProxyMode, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeVipObjects, kubeVipPods, kubeVipReady, kubeletBand, labImage, labSource, latchBand, leafCertificates, learnWindow, lintCell, listenerCountsOf, localPin, medianInterval, medianText, meshKey, middle, middleboxCommands, namesService, namespaceObject, namespaceOf, nodeLossBand, nodeNames, nodeReturnBand, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, overshootBand, pausedWindow, periodOf, placementOf, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, sessionsAt, setZtunnelEnv, sha256, shardOf, sharedGate, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stackOf, stageLabImage, stallRemovalBand, stopOf, stoppingOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, vipLeaseOf, vipNetworkOf, vipOf, vipServed, vipService, writeResult, zoneOf, ztunnelCertificates, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
|
|
4383
4960
|
|
|
4384
4961
|
//# sourceMappingURL=index.js.map
|