@mark1russell7/harness 0.14.3 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/build/{cell-B3-BSTit.d.ts → cell-biffSiEJ.d.ts} +415 -10
- package/build/{image-BMdV7yWD.js → image-BxVTFZ1o.js} +30 -2
- package/build/image-BxVTFZ1o.js.map +1 -0
- package/build/index.d.ts +509 -5
- package/build/index.js +2208 -245
- package/build/index.js.map +1 -1
- package/build/schema.d.ts +5 -2
- package/build/schema.js +7 -3
- package/build/schema.js.map +1 -1
- package/image/Dockerfile +3 -3
- package/image/amqp/client-cluster.mjs +68 -31
- package/image/hikari/ClientHikari.java +81 -6
- package/image/kafka/AdminKafka.java +152 -0
- package/image/kafka/ClientKafka.java +234 -0
- package/image/kafka/ConsumerKafka.java +359 -0
- package/image/kafka/Dockerfile +23 -0
- package/image/kafka/jars.sha256 +5 -0
- package/image/native/Cargo.toml +1 -1
- package/image/native/src/bin/client.rs +164 -28
- package/image/native/src/bin/ctfull.rs +633 -0
- package/image/native/src/bin/inject.rs +16 -0
- package/image/native/src/bin/load.rs +481 -0
- package/image/native/src/bin/ports.rs +367 -0
- package/image/native/src/bin/queue.rs +542 -0
- package/image/native/src/lib.rs +51 -0
- package/image/node/client-sentinel.mjs +28 -12
- package/image/pg/client-pg.mjs +31 -4
- package/package.json +9 -2
- package/build/image-BMdV7yWD.js.map +0 -1
package/build/index.js
CHANGED
|
@@ -1,25 +1,251 @@
|
|
|
1
|
-
import { A as
|
|
1
|
+
import { A as unlockedManifests, B as isHopPlace, C as programOf, D as subjectProblems, E as subjectClient, F as jarVersions, H as requestFirst, I as locksOf, L as npmVersions, M as LOCK_NAMES, N as cargoVersions, O as targetImageOf, P as goVersions, R as versionsOf, S as planOf, T as shippedSource, V as latestEndOf, _ as kafkaClientImage, a as SHIPPED_NAMES, b as nodeHttpImage, c as buildRecordOf, d as harnessImage, f as harnessVersion, g as inContext, h as imageProblems, i as SHIPPED_CLIENTS, j as unpinnedBases, k as targetSourceOf, l as goHttpImage, m as imageOf, n as PINNED, o as amqpClientImage, p as hikariClientImage, r as REPOSITORY, s as basesOf, t as CLIENT_NAMES, u as grpcClientImage, v as mongoClientImage, w as routerImage, x as pgClientImage, y as nodeClientImage, z as CLIENT_CAPABILITIES } from "./image-BxVTFZ1o.js";
|
|
2
2
|
import { quantity, valueIn } from "@mark1russell7/quantity";
|
|
3
|
-
import { median, medianInterval as medianInterval$1, quantileSorted } from "@mark1russell7/rigor-statistics";
|
|
3
|
+
import { median, medianInterval as medianInterval$1, quantile, quantileSorted } from "@mark1russell7/rigor-statistics";
|
|
4
|
+
import { rungVerdict } from "@mark1russell7/rigor-bench";
|
|
5
|
+
import { measureLoad, schedule, scheduleHeld } from "@mark1russell7/rigor-chaos";
|
|
4
6
|
import { createHash, randomBytes } from "node:crypto";
|
|
5
7
|
import { execFile } from "node:child_process";
|
|
6
8
|
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
7
9
|
import { release, tmpdir } from "node:os";
|
|
8
10
|
import { join } from "node:path";
|
|
9
11
|
import { existsSync, readFileSync } from "node:fs";
|
|
10
|
-
//#region src/
|
|
11
|
-
/** This function tells if a fault place is a hop of a topology. */
|
|
12
|
-
const isHopPlace = (place) => typeof place === "object" && "hop" in place;
|
|
12
|
+
//#region src/load.ts
|
|
13
13
|
/**
|
|
14
|
-
*
|
|
15
|
-
*
|
|
14
|
+
* The measure and the verdict of an open-loop cell (SYS-68). A client on a schedule sends each
|
|
15
|
+
* arrival at its time, thus a stall counts in each arrival that it delayed. The package rigor-chaos
|
|
16
|
+
* counts each latency from the planned time. The package rigor-bench judges a quantile by a
|
|
17
|
+
* distribution-free interval.
|
|
16
18
|
*/
|
|
17
|
-
|
|
19
|
+
/** This path is where the client of an open-loop cell finds its schedule. */
|
|
20
|
+
const SCHEDULE_PATH = "/tmp/schedule.json";
|
|
21
|
+
/** This function gives the time of a line of the client, in ms from the start of the schedule. */
|
|
22
|
+
const fromStart = (start, t) => (t - start) / 1e6;
|
|
18
23
|
/**
|
|
19
|
-
* This function gives the
|
|
20
|
-
* the
|
|
24
|
+
* This function gives the measure of one run of a schedule from the lines of its client. It gives
|
|
25
|
+
* null when the client did not start the schedule. The first `sent` line of a client is its request
|
|
26
|
+
* before `ready`, and it names no arrival. The end of the run is its last line.
|
|
21
27
|
*/
|
|
22
|
-
const
|
|
28
|
+
const loadOf = (events, plan) => {
|
|
29
|
+
const start = events.find((event) => event.ev === "scheduled")?.t;
|
|
30
|
+
if (start === void 0) return null;
|
|
31
|
+
return measureOf(events, start, plan, void 0);
|
|
32
|
+
};
|
|
33
|
+
/** This function tells if a status of a reply is a failure: a number outside 200 to 299. */
|
|
34
|
+
const failedStatus = (status) => typeof status === "number" && (status < 200 || status > 299);
|
|
35
|
+
/** This function gives the measure of the arrivals of one stream: the lines with the key `key`, or each line for no key. */
|
|
36
|
+
const measureOf = (events, start, plan, key) => {
|
|
37
|
+
const sent = /* @__PURE__ */ new Map();
|
|
38
|
+
const done = /* @__PURE__ */ new Map();
|
|
39
|
+
let failed = 0;
|
|
40
|
+
for (const event of events) {
|
|
41
|
+
const index = event["i"];
|
|
42
|
+
if (typeof index !== "number" || key !== void 0 && event["k"] !== key) continue;
|
|
43
|
+
if (event.ev === "sent") sent.set(index, fromStart(start, event.t));
|
|
44
|
+
if (event.ev === "replied") {
|
|
45
|
+
done.set(index, fromStart(start, event.t));
|
|
46
|
+
if (failedStatus(event["status"])) failed += 1;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
const completions = [...sent].map(([index, at]) => ({
|
|
50
|
+
index,
|
|
51
|
+
sent: at,
|
|
52
|
+
done: done.get(index) ?? null
|
|
53
|
+
}));
|
|
54
|
+
const end = events.reduce((latest, event) => Math.max(latest, fromStart(start, event.t)), 0);
|
|
55
|
+
return {
|
|
56
|
+
...measureLoad(plan, completions, end),
|
|
57
|
+
failed
|
|
58
|
+
};
|
|
59
|
+
};
|
|
60
|
+
/** This path is where the client `load` finds its plan. */
|
|
61
|
+
const LOAD_PATH = "/tmp/load.json";
|
|
62
|
+
/** This text is the format of the plan of the client `load`. */
|
|
63
|
+
const LOAD_FORMAT = "systems-load/1";
|
|
64
|
+
/**
|
|
65
|
+
* This function gives the schedule of each stream of a cell, and the text of the plan of the client
|
|
66
|
+
* `load`. It throws for no stream, for two streams with one key and for a stream with no request.
|
|
67
|
+
*/
|
|
68
|
+
const loadPlanOf = (streams) => {
|
|
69
|
+
if (streams.length === 0) throw new RangeError("a plan of the client load has no stream");
|
|
70
|
+
const keys = /* @__PURE__ */ new Set();
|
|
71
|
+
for (const stream of streams) {
|
|
72
|
+
if (keys.has(stream.key)) throw new RangeError(`two streams have the key ${stream.key}`);
|
|
73
|
+
if (stream.requests.length === 0) throw new RangeError(`the stream ${stream.key} has no request`);
|
|
74
|
+
keys.add(stream.key);
|
|
75
|
+
}
|
|
76
|
+
const scheduled = streams.map((stream) => ({
|
|
77
|
+
key: stream.key,
|
|
78
|
+
schedule: schedule(stream.plan.profile, stream.plan.options),
|
|
79
|
+
requests: stream.requests
|
|
80
|
+
}));
|
|
81
|
+
return {
|
|
82
|
+
streams: scheduled.map(({ key, schedule: each }) => ({
|
|
83
|
+
key,
|
|
84
|
+
schedule: each
|
|
85
|
+
})),
|
|
86
|
+
content: JSON.stringify({
|
|
87
|
+
format: LOAD_FORMAT,
|
|
88
|
+
streams: scheduled
|
|
89
|
+
})
|
|
90
|
+
};
|
|
91
|
+
};
|
|
92
|
+
/**
|
|
93
|
+
* This function gives the measure of each stream of a run of the client `load`, and their pool. Each
|
|
94
|
+
* stream counts its arrivals from 0, thus the lines of a stream are the lines with its key. The pool
|
|
95
|
+
* sums the counts and joins the times, in the order of the streams. It gives null when the client did
|
|
96
|
+
* not start its schedule.
|
|
97
|
+
*/
|
|
98
|
+
const streamsLoadOf = (events, streams) => {
|
|
99
|
+
const start = events.find((event) => event.ev === "scheduled")?.t;
|
|
100
|
+
if (start === void 0) return null;
|
|
101
|
+
const each = streams.map((stream) => [stream.key, measureOf(events, start, stream.schedule, stream.key)]);
|
|
102
|
+
const measures = each.map(([, measure]) => measure);
|
|
103
|
+
const sum = (count) => measures.reduce((total, measure) => total + count(measure), 0);
|
|
104
|
+
const join = (times) => measures.flatMap(times);
|
|
105
|
+
return {
|
|
106
|
+
pool: {
|
|
107
|
+
planned: sum((measure) => measure.planned),
|
|
108
|
+
unsent: sum((measure) => measure.unsent),
|
|
109
|
+
unfinished: sum((measure) => measure.unfinished),
|
|
110
|
+
failed: sum((measure) => measure.failed ?? 0),
|
|
111
|
+
latency: join((measure) => measure.latency),
|
|
112
|
+
service: join((measure) => measure.service),
|
|
113
|
+
lag: join((measure) => measure.lag),
|
|
114
|
+
censored: join((measure) => measure.censored)
|
|
115
|
+
},
|
|
116
|
+
streams: Object.fromEntries(each)
|
|
117
|
+
};
|
|
118
|
+
};
|
|
119
|
+
/** This function gives the spread of a list of times in ms, or null for no time. */
|
|
120
|
+
const spreadOf$1 = (values) => values.length === 0 ? null : {
|
|
121
|
+
p50: quantity(quantile(values, .5, "nearest-rank"), "ms"),
|
|
122
|
+
p99: quantity(quantile(values, .99, "nearest-rank"), "ms"),
|
|
123
|
+
max: quantity(values.reduce((most, value) => Math.max(most, value), 0), "ms")
|
|
124
|
+
};
|
|
125
|
+
/** This time is the date of the record of a band and the date of its verdict. The record has no age, because it comes from this run. */
|
|
126
|
+
const NOW = "2026-01-01T00:00:00.000Z";
|
|
127
|
+
/**
|
|
128
|
+
* This function judges one band of the latency over the pooled arrivals of the counted runs. The
|
|
129
|
+
* top of the band reads the interval of rigor-bench: the band holds only if the interval is under its top.
|
|
130
|
+
* The latencies of one run can follow each other, thus the width of the interval is an estimate.
|
|
131
|
+
* The bottom of the band reads the quantile itself. It gives the text of the verdict, and if it holds.
|
|
132
|
+
*/
|
|
133
|
+
const bandVerdict = (band, values, censored) => {
|
|
134
|
+
const high = valueIn(band.high, "ms");
|
|
135
|
+
const rung = {
|
|
136
|
+
name: "band",
|
|
137
|
+
at: {},
|
|
138
|
+
budgets: [{
|
|
139
|
+
kind: "quantile",
|
|
140
|
+
measure: "latency",
|
|
141
|
+
unit: "ms",
|
|
142
|
+
quantile: band.quantile,
|
|
143
|
+
limit: high
|
|
144
|
+
}],
|
|
145
|
+
test: "the cell"
|
|
146
|
+
};
|
|
147
|
+
const top = rungVerdict(rung, {
|
|
148
|
+
rung: "band",
|
|
149
|
+
commit: "the cell",
|
|
150
|
+
at: NOW,
|
|
151
|
+
real: ["the counted runs"],
|
|
152
|
+
synthetic: [],
|
|
153
|
+
evidence: [{
|
|
154
|
+
kind: "quantile",
|
|
155
|
+
measure: "latency",
|
|
156
|
+
values,
|
|
157
|
+
censored
|
|
158
|
+
}]
|
|
159
|
+
}, { now: NOW });
|
|
160
|
+
const read = values.length + censored.length === 0 ? null : quantile([...values, ...censored], band.quantile, "nearest-rank");
|
|
161
|
+
const low = band.low === void 0 ? null : valueIn(band.low, "ms");
|
|
162
|
+
const above = low === null || read !== null && read >= low;
|
|
163
|
+
const place = low === null ? `under ${high} ms` : `in [${low}, ${high}] ms`;
|
|
164
|
+
const outcome = top.outcome === "within" && !above ? "under the bottom of the band" : top.outcome;
|
|
165
|
+
return {
|
|
166
|
+
holds: top.outcome === "within" && above,
|
|
167
|
+
words: `p${band.quantile * 100} of the latency ${read === null ? "has no value" : `is ${read} ms`}, wanted ${place}: ${outcome}. ${top.reason}`
|
|
168
|
+
};
|
|
169
|
+
};
|
|
170
|
+
/**
|
|
171
|
+
* This function judges the share of failed arrivals over the counted runs against a limit. Each
|
|
172
|
+
* planned arrival is a trial. An arrival fails if the client did not send it. It also fails with no
|
|
173
|
+
* reply at the end of its run, or with the status of a failure. The rate budget of rigor-bench
|
|
174
|
+
* reads the upper end of the Wilson interval at 0.95. Thus 381 arrivals with no failure hold a limit
|
|
175
|
+
* of 1 %, and fewer arrivals cannot decide.
|
|
176
|
+
*/
|
|
177
|
+
const failuresVerdict = (limit, failures, trials) => {
|
|
178
|
+
const verdict = rungVerdict({
|
|
179
|
+
name: "failures",
|
|
180
|
+
at: {},
|
|
181
|
+
budgets: [{
|
|
182
|
+
kind: "rate",
|
|
183
|
+
measure: "failures",
|
|
184
|
+
limit
|
|
185
|
+
}],
|
|
186
|
+
test: "the cell"
|
|
187
|
+
}, {
|
|
188
|
+
rung: "failures",
|
|
189
|
+
commit: "the cell",
|
|
190
|
+
at: NOW,
|
|
191
|
+
real: ["the counted runs"],
|
|
192
|
+
synthetic: [],
|
|
193
|
+
evidence: [{
|
|
194
|
+
kind: "rate",
|
|
195
|
+
measure: "failures",
|
|
196
|
+
failures,
|
|
197
|
+
trials
|
|
198
|
+
}]
|
|
199
|
+
}, { now: NOW });
|
|
200
|
+
return {
|
|
201
|
+
holds: verdict.outcome === "within",
|
|
202
|
+
words: `${failures} of ${trials} arrivals failed, wanted a share under ${limit * 100} %: ${verdict.outcome}. ${verdict.reason}`
|
|
203
|
+
};
|
|
204
|
+
};
|
|
205
|
+
/**
|
|
206
|
+
* This function judges the load of an open-loop cell over its counted runs. A run that did not keep
|
|
207
|
+
* its schedule measures its driver, not the target, thus it is a finding of that run. Each
|
|
208
|
+
* band reads all the arrivals of the counted runs.
|
|
209
|
+
*/
|
|
210
|
+
const loadVerdict = (prediction, runs) => {
|
|
211
|
+
const limit = valueIn(prediction.lag, "ms");
|
|
212
|
+
const outside = runs.flatMap((run) => {
|
|
213
|
+
if (run.load === void 0) return [{
|
|
214
|
+
run: run.name,
|
|
215
|
+
why: "the client never started its schedule"
|
|
216
|
+
}];
|
|
217
|
+
const check = scheduleHeld(run.load, limit);
|
|
218
|
+
return check.held ? [] : [{
|
|
219
|
+
run: run.name,
|
|
220
|
+
why: check.reason
|
|
221
|
+
}];
|
|
222
|
+
});
|
|
223
|
+
const measures = runs.flatMap((run) => run.load === void 0 ? [] : [run.load]);
|
|
224
|
+
const latency = measures.flatMap((measure) => measure.latency);
|
|
225
|
+
const censored = measures.flatMap((measure) => measure.censored);
|
|
226
|
+
const bands = prediction.bands.map((band) => bandVerdict(band, latency, censored));
|
|
227
|
+
const total = (count) => measures.reduce((sum, measure) => sum + count(measure), 0);
|
|
228
|
+
const planned = total((measure) => measure.planned);
|
|
229
|
+
const unsent = total((measure) => measure.unsent);
|
|
230
|
+
const unfinished = total((measure) => measure.unfinished);
|
|
231
|
+
const failed = total((measure) => measure.failed ?? 0);
|
|
232
|
+
const failures = prediction.failures === void 0 ? void 0 : failuresVerdict(prediction.failures, unsent + unfinished + failed, planned);
|
|
233
|
+
return {
|
|
234
|
+
holds: runs.length > 0 && outside.length === 0 && bands.every((band) => band.holds) && (failures?.holds ?? true),
|
|
235
|
+
outside,
|
|
236
|
+
load: {
|
|
237
|
+
planned,
|
|
238
|
+
unsent,
|
|
239
|
+
unfinished,
|
|
240
|
+
failed,
|
|
241
|
+
latency: spreadOf$1([...latency, ...censored]),
|
|
242
|
+
service: spreadOf$1(measures.flatMap((measure) => measure.service)),
|
|
243
|
+
lag: spreadOf$1(measures.flatMap((measure) => measure.lag.map((lag) => Math.abs(lag)))),
|
|
244
|
+
bands: bands.map((band) => band.words),
|
|
245
|
+
...failures === void 0 ? {} : { failures: failures.words }
|
|
246
|
+
}
|
|
247
|
+
};
|
|
248
|
+
};
|
|
23
249
|
//#endregion
|
|
24
250
|
//#region src/verdict.ts
|
|
25
251
|
/** This function gives the words of the outcome of a second request: its connection, a retry, and its end. */
|
|
@@ -288,6 +514,10 @@ const groupVerdictOf = (prediction, counted, delta = NOISE) => {
|
|
|
288
514
|
const errors = /* @__PURE__ */ new Map();
|
|
289
515
|
for (const run of counted) for (const error of run.group?.errors ?? []) errors.set(error.name, [...errors.get(error.name) ?? [], valueIn(error.first, "s")]);
|
|
290
516
|
const lags = counted.flatMap((run) => run.group?.lags ?? []);
|
|
517
|
+
const copyCounts = counted.flatMap((run) => {
|
|
518
|
+
const copies = run.group?.writes.duplicates;
|
|
519
|
+
return copies === null || copies === void 0 ? [] : [copies.length];
|
|
520
|
+
});
|
|
291
521
|
return {
|
|
292
522
|
stages,
|
|
293
523
|
medians,
|
|
@@ -310,7 +540,13 @@ const groupVerdictOf = (prediction, counted, delta = NOISE) => {
|
|
|
310
540
|
runs: firsts.length,
|
|
311
541
|
first: s(median(firsts.sort((a, b) => a - b)))
|
|
312
542
|
})),
|
|
313
|
-
...lags.length === 0 ? {} : { lags: spreadOf(lags) }
|
|
543
|
+
...lags.length === 0 ? {} : { lags: spreadOf(lags) },
|
|
544
|
+
...prediction.group?.duplicates === void 0 ? {} : { duplicates: {
|
|
545
|
+
runs: copyCounts.length,
|
|
546
|
+
total: copyCounts.reduce((sum, n) => sum + n, 0),
|
|
547
|
+
median: medianCount(copyCounts),
|
|
548
|
+
max: copyCounts.length === 0 ? null : Math.max(...copyCounts)
|
|
549
|
+
} }
|
|
314
550
|
};
|
|
315
551
|
};
|
|
316
552
|
/** This function gives the verdict of the band of a cell: its containment and the equivalence of its median. */
|
|
@@ -335,6 +571,17 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
|
|
|
335
571
|
why: "no run counted"
|
|
336
572
|
};
|
|
337
573
|
if (prediction.detects === null) {
|
|
574
|
+
const law = prediction.queue !== void 0 ? "queue" : prediction.ports !== void 0 ? "port range" : prediction.pause !== void 0 ? "paused consumer" : prediction.conntrack !== void 0 ? "conntrack table" : void 0;
|
|
575
|
+
if (law !== void 0) {
|
|
576
|
+
const off = new Set(outside.map((each) => each.run)).size;
|
|
577
|
+
return {
|
|
578
|
+
...empty,
|
|
579
|
+
decision: off === 0 ? "holds" : "fails",
|
|
580
|
+
band: null,
|
|
581
|
+
outside,
|
|
582
|
+
why: `${counted.length - off} of ${counted.length} runs hold the law of the ${law}; ${prediction.why}`
|
|
583
|
+
};
|
|
584
|
+
}
|
|
338
585
|
const crossed = counted.flatMap((run) => run.crossings === void 0 ? [] : [run.crossings]);
|
|
339
586
|
if (crossed.length > 0) {
|
|
340
587
|
const hop = crossed[0].hop;
|
|
@@ -363,16 +610,30 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
|
|
|
363
610
|
}
|
|
364
611
|
const outcomes = [...kinds].map(([kind, n]) => `${n} ${kind}`).join(", ");
|
|
365
612
|
const timed = times.length === 0 ? "no time" : `median ${s(median(times)).value} s from ${prediction.from}, range [${s(times[0]).value}, ${s(times.at(-1)).value}] s`;
|
|
366
|
-
const counts = prediction
|
|
613
|
+
const { counts } = prediction;
|
|
367
614
|
if (counts !== void 0) {
|
|
615
|
+
const exact = counts.map((each) => `${each.what} ${each.count}`).join(", ");
|
|
616
|
+
return {
|
|
617
|
+
...empty,
|
|
618
|
+
decision: outside.length === 0 ? "holds" : "fails",
|
|
619
|
+
band: null,
|
|
620
|
+
outside,
|
|
621
|
+
median: times.length === 0 ? null : s(median(times)),
|
|
622
|
+
why: `${counted.length - outside.length} of ${counted.length} runs gave the exact counts (${exact}); ${outcomes}; ${timed}. ${prediction.why}`
|
|
623
|
+
};
|
|
624
|
+
}
|
|
625
|
+
const lost = prediction.group?.lost;
|
|
626
|
+
const copies = prediction.group?.duplicates;
|
|
627
|
+
if (lost !== void 0 || copies !== void 0) {
|
|
368
628
|
const off = new Set(outside.map((each) => each.run)).size;
|
|
629
|
+
const bands = [...lost === void 0 ? [] : [`lost writes [${lost.low}, ${lost.high}]`], ...copies === void 0 ? [] : [`extra copies [${copies.low}, ${copies.high}]`]];
|
|
369
630
|
return {
|
|
370
631
|
...empty,
|
|
371
632
|
decision: off === 0 ? "holds" : "fails",
|
|
372
633
|
band: null,
|
|
373
634
|
outside,
|
|
374
635
|
median: times.length === 0 ? null : s(median(times)),
|
|
375
|
-
why: `${counted.length - off} of ${counted.length} runs inside the band of
|
|
636
|
+
why: `${counted.length - off} of ${counted.length} runs inside the band of ${bands.join(" and the band of ")}; ${outcomes}; ${timed}. ${prediction.why}`
|
|
376
637
|
};
|
|
377
638
|
}
|
|
378
639
|
const missed = outside.filter((each) => counted.some((run) => name(run) === each.run && run.reconnect !== void 0));
|
|
@@ -386,6 +647,33 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
|
|
|
386
647
|
why: `${counted.length} runs measured: ${outcomes}; ${timed}.${late} ${prediction.why}`
|
|
387
648
|
};
|
|
388
649
|
}
|
|
650
|
+
if (counted.some((run) => run.load !== void 0)) {
|
|
651
|
+
const judged = loadVerdict(prediction.load ?? {
|
|
652
|
+
lag: quantity(Number.MAX_SAFE_INTEGER, "ms"),
|
|
653
|
+
bands: []
|
|
654
|
+
}, counted.map((run) => ({
|
|
655
|
+
name: name(run),
|
|
656
|
+
load: run.load
|
|
657
|
+
})));
|
|
658
|
+
const middleOfLoad = judged.load.latency?.p50;
|
|
659
|
+
const failed = judged.load.failed === 0 ? "" : `, ${judged.load.failed} with the status of a failure`;
|
|
660
|
+
const read = [
|
|
661
|
+
`${counted.length} runs on a schedule, ${judged.load.planned} arrivals, ${judged.load.unsent} not sent, ${judged.load.unfinished} with no reply${failed}.`,
|
|
662
|
+
...judged.load.bands,
|
|
663
|
+
...judged.load.failures === void 0 ? [] : [judged.load.failures],
|
|
664
|
+
prediction.why
|
|
665
|
+
].join(" ");
|
|
666
|
+
return {
|
|
667
|
+
...empty,
|
|
668
|
+
decision: prediction.load === void 0 ? "measured" : judged.holds ? "holds" : "fails",
|
|
669
|
+
band: null,
|
|
670
|
+
outside: judged.outside,
|
|
671
|
+
median: middleOfLoad === void 0 ? null : s(valueIn(middleOfLoad, "s")),
|
|
672
|
+
...judged.load.latency === null ? {} : { slowest: s(valueIn(judged.load.latency.max, "s")) },
|
|
673
|
+
load: judged.load,
|
|
674
|
+
why: read
|
|
675
|
+
};
|
|
676
|
+
}
|
|
389
677
|
const samples = counted.flatMap((run) => (run.samples ?? []).map((sample) => valueIn(sample, "s"))).sort((a, b) => a - b);
|
|
390
678
|
const retransmissions = counted.reduce((sum, run) => sum + run.retransmissions, 0);
|
|
391
679
|
const ended = counted.filter((run) => run.outcome.kind === "error" || run.outcome.kind === "eof");
|
|
@@ -529,7 +817,7 @@ const DROP_HANDLING = quantity(1.5, "ms");
|
|
|
529
817
|
/** This function gives the window of the act of a drop, in seconds: `DROP_HANDLING`, or the pairing window (`delta`) when it is longer. */
|
|
530
818
|
const dropWindow = (delta) => Math.max(valueIn(DROP_HANDLING, "s"), valueIn(delta, "s"));
|
|
531
819
|
/** This function gives the interval between two instants on the one clock, in seconds to the microsecond. */
|
|
532
|
-
const between$
|
|
820
|
+
const between$4 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
|
|
533
821
|
/** This function gives the state of a state record after the change, and before it. TCP_ESTABLISHED is 1, TCP_SYN_SENT 2. */
|
|
534
822
|
const newState$2 = (record) => record.new;
|
|
535
823
|
const oldState$2 = (record) => record.old;
|
|
@@ -571,7 +859,7 @@ const eventOf = (socket, line, matches, delta = NOISE) => {
|
|
|
571
859
|
*/
|
|
572
860
|
const leftAfter = (socket, line) => {
|
|
573
861
|
const left = socket.find((record) => record.ev === "state" && oldState$2(record) === 1 && (newState$2(record) === 4 || newState$2(record) === 7) && record.mono >= line.t);
|
|
574
|
-
return left === void 0 ? null : between$
|
|
862
|
+
return left === void 0 ? null : between$4(line.t, left.mono);
|
|
575
863
|
};
|
|
576
864
|
/**
|
|
577
865
|
* This function gives the end of the client process in a run, as the kernel stamps it. The process
|
|
@@ -663,19 +951,19 @@ const reconnectOf = (run, detected, cookie, from = detected) => {
|
|
|
663
951
|
const waits = starts.flatMap((start, at) => {
|
|
664
952
|
const next = starts[at + 1]?.t ?? Number.POSITIVE_INFINITY;
|
|
665
953
|
const dial = dials.find((each) => each.t >= start.t && each.t < next);
|
|
666
|
-
return dial === void 0 ? [] : [between$
|
|
954
|
+
return dial === void 0 ? [] : [between$4(start.t, dial.t)];
|
|
667
955
|
});
|
|
668
956
|
const fresh = run.kernel.filter((record) => record.netns === run.netns && record.cookie !== cookie && record.ev === "state" && record.mono > detected);
|
|
669
957
|
const syns = fresh.filter((record) => newState$2(record) === 2);
|
|
670
958
|
const established = fresh.find((record) => newState$2(record) === 1);
|
|
671
959
|
const connects = dials.flatMap((dial) => {
|
|
672
960
|
const syn = syns.find((record) => record.mono >= dial.t);
|
|
673
|
-
return syn === void 0 ? [] : [between$
|
|
961
|
+
return syn === void 0 ? [] : [between$4(dial.t, syn.mono)];
|
|
674
962
|
});
|
|
675
963
|
return {
|
|
676
964
|
waits,
|
|
677
|
-
syn: syns[0] === void 0 ? null : between$
|
|
678
|
-
established: established === void 0 ? null : between$
|
|
965
|
+
syn: syns[0] === void 0 ? null : between$4(detected, syns[0].mono),
|
|
966
|
+
established: established === void 0 ? null : between$4(detected, established.mono),
|
|
679
967
|
connects
|
|
680
968
|
};
|
|
681
969
|
};
|
|
@@ -702,7 +990,7 @@ const callOf = (events, request, detected) => {
|
|
|
702
990
|
};
|
|
703
991
|
return {
|
|
704
992
|
outcome: end.ev === "data" ? "a reply" : String(end["message"] ?? end["code"] ?? "an error"),
|
|
705
|
-
after: detected === void 0 ? null : between$
|
|
993
|
+
after: detected === void 0 ? null : between$4(detected, end.t)
|
|
706
994
|
};
|
|
707
995
|
};
|
|
708
996
|
//#endregion
|
|
@@ -717,12 +1005,22 @@ const ERRNO$1 = {
|
|
|
717
1005
|
111: "ECONNREFUSED",
|
|
718
1006
|
113: "EHOSTUNREACH"
|
|
719
1007
|
};
|
|
720
|
-
const between$
|
|
1008
|
+
const between$3 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
|
|
721
1009
|
const round = (seconds) => Math.round(seconds * 1e6) / 1e6;
|
|
722
1010
|
/** This function gives the value of a field of a line by its name. A dotted name is a field in a field. */
|
|
723
1011
|
const fieldOf = (line, path) => path.split(".").reduce((value, key) => value !== null && typeof value === "object" ? value[key] : void 0, line);
|
|
724
1012
|
/** This function tells if a line of the client matches a line match: its `ev`, and each field that the match names. */
|
|
725
1013
|
const matches = (line, match) => line.ev === match.ev && Object.entries(match.where ?? {}).every(([path, value]) => fieldOf(line, path) === value);
|
|
1014
|
+
/** This function gives the count of the lines that match. */
|
|
1015
|
+
const countOf = (lines, match) => lines.filter((line) => matches(line, match)).length;
|
|
1016
|
+
/** This function gives the finding of the line counts of a run, or null when each count is the exact one. */
|
|
1017
|
+
const countFinding = (counts, lines) => {
|
|
1018
|
+
const off = counts.flatMap((each) => {
|
|
1019
|
+
const seen = countOf(lines, each.match);
|
|
1020
|
+
return seen === each.count ? [] : [`${each.what}: ${seen} lines, and the law gives ${each.count}`];
|
|
1021
|
+
});
|
|
1022
|
+
return off.length === 0 ? null : off.join("; ");
|
|
1023
|
+
};
|
|
726
1024
|
/** This function says a line match in words, for a finding. */
|
|
727
1025
|
const wordsOf = (match) => {
|
|
728
1026
|
const where = Object.entries(match.where ?? {}).map(([path, value]) => `${path} ${String(value)}`);
|
|
@@ -1025,7 +1323,7 @@ const measureDriver = (cell, run) => {
|
|
|
1025
1323
|
const until = prediction.detects ? Infinity : after + valueIn(prediction.window, "s") * 1e9;
|
|
1026
1324
|
for (const match of driver.quiet ?? []) {
|
|
1027
1325
|
const line = events.find((event) => event.t > after && event.t <= until && matches(event, match));
|
|
1028
|
-
if (line !== void 0) findings.push(`${wordsOf(match)} came ${between$
|
|
1326
|
+
if (line !== void 0) findings.push(`${wordsOf(match)} came ${between$3(after, line.t).value} s after the fault, where none should`);
|
|
1029
1327
|
}
|
|
1030
1328
|
const expected = prediction.detects ? void 0 : prediction.middlebox?.forgets;
|
|
1031
1329
|
if (!prediction.detects && (run.middlebox !== void 0 || expected !== void 0)) {
|
|
@@ -1035,14 +1333,14 @@ const measureDriver = (cell, run) => {
|
|
|
1035
1333
|
for (const socket of socketsOf(events)) {
|
|
1036
1334
|
const forgot = middlebox.find((line) => line.ev === "ct" && line["type"] === "DESTROY" && String(line["orig"] ?? "").startsWith(`${socket.local}>`) && line.t > socket.opened && (socket.ended === null || line.t < socket.ended) && line.t <= until);
|
|
1037
1335
|
if (expected === void 0 || socket.local !== own) {
|
|
1038
|
-
if (forgot !== void 0) findings.push(`a router forgot the flow of the ${socket.role} socket ${between$
|
|
1336
|
+
if (forgot !== void 0) findings.push(`a router forgot the flow of the ${socket.role} socket ${between$3(after, forgot.t).value} s after the fault, where it should keep it`);
|
|
1039
1337
|
continue;
|
|
1040
1338
|
}
|
|
1041
1339
|
const last = referenceOf(run, "last-received", driver.socket, false);
|
|
1042
1340
|
if (forgot === void 0) findings.push(`no router forgot the flow of the ${socket.role} socket in the window, where it should forget it`);
|
|
1043
1341
|
else if (last === void 0) findings.push(`the ${socket.role} socket got no segment before the fault`);
|
|
1044
1342
|
else {
|
|
1045
|
-
const at = between$
|
|
1343
|
+
const at = between$3(last, forgot.t).value;
|
|
1046
1344
|
const [low, high] = [round(valueIn(expected.low, "s") - noise), round(valueIn(expected.high, "s") + noise)];
|
|
1047
1345
|
if (at < low || at > high) findings.push(`a router forgot the flow of the ${socket.role} socket ${at} s after its last segment, outside [${low}, ${high}] s`);
|
|
1048
1346
|
}
|
|
@@ -1059,7 +1357,7 @@ const measureDriver = (cell, run) => {
|
|
|
1059
1357
|
elapsed: null
|
|
1060
1358
|
};
|
|
1061
1359
|
}
|
|
1062
|
-
const elapsed = between$
|
|
1360
|
+
const elapsed = between$3(from, line.t);
|
|
1063
1361
|
const name = nameOf$1(line, step.ends);
|
|
1064
1362
|
if (step.name !== "OK" && name !== step.name) findings.push(`${step.what}: it ended with ${name}, not ${step.name}`);
|
|
1065
1363
|
const [low, high] = [round(valueIn(step.low, "s") - noise), round(valueIn(step.high, "s") + noise)];
|
|
@@ -1093,7 +1391,7 @@ const measureDriver = (cell, run) => {
|
|
|
1093
1391
|
...nothing,
|
|
1094
1392
|
...stepped,
|
|
1095
1393
|
outcome: { kind: "answered" },
|
|
1096
|
-
elapsed: between$
|
|
1394
|
+
elapsed: between$3(from, line.t),
|
|
1097
1395
|
void: voided,
|
|
1098
1396
|
finding: findings[0] ?? null
|
|
1099
1397
|
};
|
|
@@ -1136,15 +1434,545 @@ const measureDriver = (cell, run) => {
|
|
|
1136
1434
|
...base,
|
|
1137
1435
|
...stepped,
|
|
1138
1436
|
outcome,
|
|
1139
|
-
elapsed: between$
|
|
1140
|
-
socket: seen === void 0 ? null : between$
|
|
1141
|
-
noticed: report === void 0 || seen === void 0 ? null : between$
|
|
1142
|
-
steps: (prediction.from === "last-received" ? records.filter((record) => record.ev === "probe" && record.what === "keepalive" && record.mono > reference) : []).map((record) => between$
|
|
1437
|
+
elapsed: between$3(reference, ended),
|
|
1438
|
+
socket: seen === void 0 ? null : between$3(reference, seen.t),
|
|
1439
|
+
noticed: report === void 0 || seen === void 0 ? null : between$3(report.at, seen.t),
|
|
1440
|
+
steps: (prediction.from === "last-received" ? records.filter((record) => record.ev === "probe" && record.what === "keepalive" && record.mono > reference) : []).map((record) => between$3(reference, record.mono)),
|
|
1143
1441
|
void: voided,
|
|
1144
1442
|
finding: findings[0] ?? null
|
|
1145
1443
|
};
|
|
1146
1444
|
};
|
|
1147
1445
|
//#endregion
|
|
1446
|
+
//#region src/queue.ts
|
|
1447
|
+
/** This bit is the SYN of the flags of an `rx` record of the prober. */
|
|
1448
|
+
const SYN = 2;
|
|
1449
|
+
/** This function gives the time between two stamps of CLOCK_MONOTONIC. */
|
|
1450
|
+
const between$2 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
|
|
1451
|
+
const numberOf$1 = (line, field) => {
|
|
1452
|
+
const value = line[field];
|
|
1453
|
+
return typeof value === "number" ? value : 0;
|
|
1454
|
+
};
|
|
1455
|
+
/** This function says if a time is inside a band. */
|
|
1456
|
+
const inside = (time, band) => {
|
|
1457
|
+
const seconds = valueIn(time, "s");
|
|
1458
|
+
return seconds >= valueIn(band.low, "s") && seconds <= valueIn(band.high, "s");
|
|
1459
|
+
};
|
|
1460
|
+
/** This function gives the words of a band. */
|
|
1461
|
+
const bandWords = (band) => `[${valueIn(band.low, "s")}, ${valueIn(band.high, "s")}] s`;
|
|
1462
|
+
const waitsOf = (from, stamps) => stamps.map((stamp, at) => between$2(at === 0 ? from : stamps[at - 1], stamp));
|
|
1463
|
+
/** This function finds the records of the sockets of the client that had this local address. */
|
|
1464
|
+
const socketsAt = (run, local) => {
|
|
1465
|
+
const cookies = new Set(run.kernel.filter((record) => record.netns === run.netns && record.src === local).map((record) => record.cookie));
|
|
1466
|
+
return run.kernel.filter((record) => record.netns === run.netns && cookies.has(record.cookie));
|
|
1467
|
+
};
|
|
1468
|
+
const readConnection = (name, run, expected) => {
|
|
1469
|
+
const lineOf = (ev) => run.events.find((event) => event.ev === ev && event["name"] === name);
|
|
1470
|
+
const dial = lineOf("dial");
|
|
1471
|
+
const connected = lineOf("connected");
|
|
1472
|
+
const failed = lineOf("failed");
|
|
1473
|
+
const state = lineOf("state");
|
|
1474
|
+
const socket = dial === void 0 ? [] : socketsAt(run, dial["local"]);
|
|
1475
|
+
const sent = socket.filter((record) => record.ev === "retrans").map((record) => record.mono);
|
|
1476
|
+
const rx = socket.filter((record) => record.ev === "rx" && ((record.seg?.flags ?? 0) & SYN) !== 0);
|
|
1477
|
+
const from = dial?.t ?? 0;
|
|
1478
|
+
const start = expected?.ends === null || expected?.ends === void 0 ? void 0 : lineOf(expected.ends.from);
|
|
1479
|
+
const last = sent.at(-1);
|
|
1480
|
+
return {
|
|
1481
|
+
name,
|
|
1482
|
+
connected: connected !== void 0,
|
|
1483
|
+
syns: waitsOf(from, connected === void 0 ? sent : sent.filter((stamp) => stamp < connected.t)),
|
|
1484
|
+
synAcks: waitsOf(from, rx.map((record) => record.mono)),
|
|
1485
|
+
retransmissions: sent.length,
|
|
1486
|
+
errno: failed === void 0 ? null : String(failed["errno"]),
|
|
1487
|
+
ended: failed === void 0 || start === void 0 ? null : between$2(start.t, failed.t),
|
|
1488
|
+
afterLast: failed === void 0 || last === void 0 ? null : between$2(last, failed.t),
|
|
1489
|
+
unacked: state === void 0 ? null : numberOf$1(state, "unacked")
|
|
1490
|
+
};
|
|
1491
|
+
};
|
|
1492
|
+
/** This function reads one run against the band of each wait in a list, and gives the words of each miss. */
|
|
1493
|
+
const waitFindings = (what, seen, bands) => {
|
|
1494
|
+
if (seen.length !== bands.length) return [`${what}: ${seen.length} waits, not ${bands.length}`];
|
|
1495
|
+
return seen.flatMap((wait, at) => inside(wait, bands[at]) ? [] : [`${what}: wait ${at} of ${valueIn(wait, "s")} s, outside ${bandWords(bands[at])}`]);
|
|
1496
|
+
};
|
|
1497
|
+
const connectionFindings = (expected, read, expiredAt, stamps) => {
|
|
1498
|
+
const { name, ends } = expected;
|
|
1499
|
+
const found = [];
|
|
1500
|
+
if (read.connected !== expected.connects) found.push(`${name}: ${read.connected ? "connected" : "did not connect"}, against the prediction`);
|
|
1501
|
+
if (expected.syns !== void 0) found.push(...waitFindings(`${name} SYN`, read.syns, expected.syns));
|
|
1502
|
+
if (expected.synAcks !== void 0) found.push(...waitFindings(`${name} SYN-ACK`, read.synAcks, expected.synAcks));
|
|
1503
|
+
if (expected.unacked !== void 0 && read.unacked !== expected.unacked) found.push(`${name}: ${read.unacked ?? "no"} segments without an ACK, not ${expected.unacked}`);
|
|
1504
|
+
if (ends === null) {
|
|
1505
|
+
if (read.errno !== null) found.push(`${name}: ended with ${read.errno}, and the prediction has no end`);
|
|
1506
|
+
return found;
|
|
1507
|
+
}
|
|
1508
|
+
if (read.errno !== ends.errno) found.push(`${name}: ended with ${read.errno ?? "nothing"}, not ${ends.errno}`);
|
|
1509
|
+
if (read.ended === null) return [...found, `${name}: no time for its end`];
|
|
1510
|
+
if (!inside(read.ended, ends.within)) found.push(`${name}: its end came at ${valueIn(read.ended, "s")} s, outside ${bandWords(ends.within)}`);
|
|
1511
|
+
if (ends.retransmitted === void 0) return found;
|
|
1512
|
+
if (read.afterLast === null || !inside(read.afterLast, ends.retransmitted)) found.push(`${name}: its end came ${read.afterLast === null ? "with no retransmission" : `${valueIn(read.afterLast, "s")} s after its last retransmission`}, outside ${bandWords(ends.retransmitted)}`);
|
|
1513
|
+
if (expiredAt === null) return found;
|
|
1514
|
+
const before = stamps.at(-2);
|
|
1515
|
+
const last = stamps.at(-1);
|
|
1516
|
+
if (last === void 0 || last < expiredAt || before !== void 0 && before > expiredAt) found.push(`${name}: its last two retransmissions do not span the expiry`);
|
|
1517
|
+
return found;
|
|
1518
|
+
};
|
|
1519
|
+
/**
|
|
1520
|
+
* This function reads one run of a queue cell. The client lines give the connections and the
|
|
1521
|
+
* counters at the two ends of the run. The target lines give the count of the queue. The records of
|
|
1522
|
+
* the prober give the waits of each socket.
|
|
1523
|
+
*/
|
|
1524
|
+
const measureQueue = (run, expected) => {
|
|
1525
|
+
const { events, kernel } = run;
|
|
1526
|
+
const lineOf = (ev) => events.find((event) => event.ev === ev);
|
|
1527
|
+
const base = {
|
|
1528
|
+
environment: run.environment,
|
|
1529
|
+
index: run.index,
|
|
1530
|
+
outcome: { kind: "answered" },
|
|
1531
|
+
elapsed: null,
|
|
1532
|
+
socket: null,
|
|
1533
|
+
noticed: null,
|
|
1534
|
+
steps: [],
|
|
1535
|
+
probes: 0,
|
|
1536
|
+
retransmissions: kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length,
|
|
1537
|
+
acks: 0
|
|
1538
|
+
};
|
|
1539
|
+
const voided = (why) => ({
|
|
1540
|
+
...base,
|
|
1541
|
+
void: why,
|
|
1542
|
+
finding: null
|
|
1543
|
+
});
|
|
1544
|
+
const opened = lineOf("opened");
|
|
1545
|
+
const stats = lineOf("stats");
|
|
1546
|
+
if (opened === void 0) return voided("the target did not open its listener");
|
|
1547
|
+
if (stats === void 0 || lineOf("done") === void 0) return voided("the client did not reach its end");
|
|
1548
|
+
if (run.lost === null) return voided("the prober wrote no clock line after the run, so nothing says what it dropped");
|
|
1549
|
+
if (run.lost > 0) return voided(`the prober dropped ${run.lost} records`);
|
|
1550
|
+
const lines = (run.server ?? []).filter((line) => line.ev === "queue" && line.t >= opened.t);
|
|
1551
|
+
if (lines.length === 0) return voided("the target wrote no queue line in the run");
|
|
1552
|
+
const named = [...new Set(events.filter((event) => event.ev === "dial").map((event) => String(event["name"])))];
|
|
1553
|
+
const predicted = new Map(expected.connections.map((each) => [each.name, each]));
|
|
1554
|
+
const connections = named.map((name) => readConnection(name, run, predicted.get(name)));
|
|
1555
|
+
const admitted = expected.admitted === void 0 ? void 0 : connections.find((each) => each.name === expected.admitted);
|
|
1556
|
+
if (expected.admitted !== void 0 && (admitted === void 0 || !admitted.connected || admitted.unacked !== 0)) return voided(`${expected.admitted} did not take the last place in the queue: the burst did not land in its sequence`);
|
|
1557
|
+
const change = (field) => numberOf$1(stats, field) - numberOf$1(opened, field);
|
|
1558
|
+
const burst = events.find((event) => event.ev === "dial" && event["name"] === "in")?.t ?? opened.t;
|
|
1559
|
+
const counted = expected.expiry?.timeouts;
|
|
1560
|
+
const expired = counted === void 0 ? void 0 : lines.find((line) => numberOf$1(line, "timeouts") - numberOf$1(opened, "timeouts") >= counted);
|
|
1561
|
+
const accepted = events.find((event) => event.ev === "accepted");
|
|
1562
|
+
const rise = expected.rise?.count ?? 0;
|
|
1563
|
+
const low = accepted === void 0 ? -1 : lines.findIndex((line) => line.t >= accepted.t && numberOf$1(line, "count") < rise);
|
|
1564
|
+
const risen = low < 0 ? void 0 : lines.slice(low).find((line) => numberOf$1(line, "count") >= rise);
|
|
1565
|
+
const drained = events.filter((event) => event.ev === "accepted").reduce((sum, event) => sum + (Array.isArray(event["peers"]) ? event["peers"].length : 0), 0);
|
|
1566
|
+
const queue = {
|
|
1567
|
+
max: numberOf$1(opened, "max"),
|
|
1568
|
+
held: Math.max(...lines.map((line) => numberOf$1(line, "count"))),
|
|
1569
|
+
connected: connections.filter((each) => each.connected).length,
|
|
1570
|
+
overflows: change("overflows"),
|
|
1571
|
+
drops: change("drops"),
|
|
1572
|
+
cookies: change("cookies"),
|
|
1573
|
+
timeouts: change("timeouts"),
|
|
1574
|
+
resets: change("resets"),
|
|
1575
|
+
drained,
|
|
1576
|
+
expiry: expired === void 0 ? null : between$2(burst, expired.t),
|
|
1577
|
+
rise: risen === void 0 ? null : between$2(burst, risen.t),
|
|
1578
|
+
connections
|
|
1579
|
+
};
|
|
1580
|
+
const found = [];
|
|
1581
|
+
if (queue.max !== expected.max) found.push(`the limit is ${queue.max}, not ${expected.max}`);
|
|
1582
|
+
if (queue.held !== expected.held) found.push(`the queue held ${queue.held}, not ${expected.held}`);
|
|
1583
|
+
if (queue.connected !== expected.connected) found.push(`${queue.connected} connected, not ${expected.connected}`);
|
|
1584
|
+
if (queue.overflows < expected.overflows.low || queue.overflows > expected.overflows.high) found.push(`${queue.overflows} overflows, outside [${expected.overflows.low}, ${expected.overflows.high}]`);
|
|
1585
|
+
if (queue.resets !== expected.resets) found.push(`${queue.resets} resets, not ${expected.resets}`);
|
|
1586
|
+
if (queue.cookies !== 0) found.push(`${queue.cookies} SYN cookies, and the law has none`);
|
|
1587
|
+
if (expected.drained !== void 0 && queue.drained !== expected.drained) found.push(`${queue.drained} accepted, not ${expected.drained}`);
|
|
1588
|
+
if (expected.expiry !== void 0 && (queue.expiry === null || !inside(queue.expiry, expected.expiry.within))) found.push(`the expiry came at ${queue.expiry === null ? "no time" : `${valueIn(queue.expiry, "s")} s`}, outside ${bandWords(expected.expiry.within)}`);
|
|
1589
|
+
if (expected.rise !== void 0 && (queue.rise === null || !inside(queue.rise, expected.rise.within))) found.push(`the count got back to ${expected.rise.count} at ${queue.rise === null ? "no time" : `${valueIn(queue.rise, "s")} s`}, outside ${bandWords(expected.rise.within)}`);
|
|
1590
|
+
for (const each of expected.connections) {
|
|
1591
|
+
const read = connections.find((connection) => connection.name === each.name);
|
|
1592
|
+
if (read === void 0) {
|
|
1593
|
+
found.push(`${each.name}: the client did not dial it`);
|
|
1594
|
+
continue;
|
|
1595
|
+
}
|
|
1596
|
+
const dial = events.find((event) => event.ev === "dial" && event["name"] === each.name);
|
|
1597
|
+
const stamps = socketsAt(run, dial?.["local"]).filter((record) => record.ev === "retrans").map((record) => record.mono);
|
|
1598
|
+
found.push(...connectionFindings(each, read, expired?.t ?? null, stamps));
|
|
1599
|
+
}
|
|
1600
|
+
return {
|
|
1601
|
+
...base,
|
|
1602
|
+
void: null,
|
|
1603
|
+
finding: found.length === 0 ? null : found.join(". "),
|
|
1604
|
+
queue
|
|
1605
|
+
};
|
|
1606
|
+
};
|
|
1607
|
+
//#endregion
|
|
1608
|
+
//#region src/conntrack.ts
|
|
1609
|
+
/** This errno tells that a connect got a slot. */
|
|
1610
|
+
const NONE = "none";
|
|
1611
|
+
const numberOr = (line, field) => {
|
|
1612
|
+
const value = line?.[field];
|
|
1613
|
+
return typeof value === "number" ? value : null;
|
|
1614
|
+
};
|
|
1615
|
+
/** This function gives the time from one stamp to a stamp of a field of a line, when the line has it. */
|
|
1616
|
+
const untilField = (from, line, field) => {
|
|
1617
|
+
const stamp = numberOr(line, field);
|
|
1618
|
+
return from === void 0 || stamp === null ? null : between$2(from.t, stamp);
|
|
1619
|
+
};
|
|
1620
|
+
const stallOf = (line) => line === void 0 ? null : {
|
|
1621
|
+
errno: String(line["errno"]),
|
|
1622
|
+
time: quantity(Math.round((numberOr(line, "elapsed") ?? 0) / 1e3) / 1e6, "s"),
|
|
1623
|
+
drops: numberOr(line, "drops") ?? 0,
|
|
1624
|
+
retransFail: numberOr(line, "retrans_fail") ?? 0
|
|
1625
|
+
};
|
|
1626
|
+
/** This function tells where a reading differs from the law of its cell. */
|
|
1627
|
+
const findingsOf$1 = (read, failed, expected) => {
|
|
1628
|
+
const found = [];
|
|
1629
|
+
const { max, neighbor, refused, drained, pressure } = expected;
|
|
1630
|
+
if (typeof failed === "string") found.push(`a connect failed with ${failed} before the client held its flows`);
|
|
1631
|
+
if (read.flows !== max) found.push(`the client held ${read.flows} flows, not ${max}`);
|
|
1632
|
+
if (read.count !== max) found.push(`the count read ${read.count} at the fill, not ${max}`);
|
|
1633
|
+
if (neighbor !== void 0 && read.neighbor !== neighbor) found.push(`the namespace of the target read ${read.neighbor ?? "nothing"}, not ${neighbor}`);
|
|
1634
|
+
if (refused !== void 0) {
|
|
1635
|
+
const stall = read.stall;
|
|
1636
|
+
if (stall === null) found.push("no connect after the fill ended");
|
|
1637
|
+
else {
|
|
1638
|
+
if (stall.errno !== refused.errno) found.push(`the connect after the fill ended with ${stall.errno}, not ${refused.errno}`);
|
|
1639
|
+
if (!inside(stall.time, refused.within)) found.push(`the connect after the fill took ${valueIn(stall.time, "s")} s, outside ${bandWords(refused.within)}`);
|
|
1640
|
+
if (stall.drops !== refused.drops) found.push(`the field drop rose by ${stall.drops}, not ${refused.drops}`);
|
|
1641
|
+
if (stall.retransFail !== refused.retransFail) found.push(`TCPRetransFail rose by ${stall.retransFail}, not ${refused.retransFail}`);
|
|
1642
|
+
}
|
|
1643
|
+
}
|
|
1644
|
+
if (drained !== void 0) {
|
|
1645
|
+
for (const [name, time] of [["first fall", read.firstFall], ["zero", read.zero]]) if (time === null || !inside(time, drained)) found.push(`the ${name} of the count came at ${time === null ? "no time" : `${valueIn(time, "s")} s`}, outside ${bandWords(drained)}`);
|
|
1646
|
+
}
|
|
1647
|
+
if (pressure !== void 0) {
|
|
1648
|
+
const stall = read.stall;
|
|
1649
|
+
if (read.closed !== max) found.push(`the count read ${read.closed ?? "nothing"} after each side closed, not ${max}`);
|
|
1650
|
+
const { highest } = pressure;
|
|
1651
|
+
if (highest === void 0) {
|
|
1652
|
+
if (read.lowest !== pressure.lowest) found.push(`the lowest count was ${read.lowest ?? "nothing"}, not ${pressure.lowest}`);
|
|
1653
|
+
if (read.after !== pressure.after) found.push(`the count read ${read.after ?? "nothing"} after the connect, not ${pressure.after}`);
|
|
1654
|
+
} else if (read.lowest === null || read.lowest < pressure.lowest || read.lowest > highest) found.push(`the lowest count was ${read.lowest ?? "nothing"}, outside ${pressure.lowest} to ${highest}`);
|
|
1655
|
+
else if (read.after !== read.lowest + 1) found.push(`the count read ${read.after ?? "nothing"} after the connect, not ${read.lowest + 1}`);
|
|
1656
|
+
if (stall === null) found.push("no connect after the close ended");
|
|
1657
|
+
else {
|
|
1658
|
+
if (stall.errno !== NONE) found.push(`the connect after the close ended with ${stall.errno}`);
|
|
1659
|
+
if (!pressure.points.some((point) => inside(stall.time, point))) found.push(`the connect after the close took ${valueIn(stall.time, "s")} s, on no point of the schedule`);
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
return found;
|
|
1663
|
+
};
|
|
1664
|
+
/**
|
|
1665
|
+
* This function reads one run of a conntrack cell from the lines of its client and its target. An
|
|
1666
|
+
* `unsettled` line makes the run void. The entries of the run before it were still in the table, thus
|
|
1667
|
+
* the run did not start from an empty table.
|
|
1668
|
+
*/
|
|
1669
|
+
const measureConntrack = (run, expected) => {
|
|
1670
|
+
const { events, kernel } = run;
|
|
1671
|
+
const retransmissions = kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length;
|
|
1672
|
+
const base = {
|
|
1673
|
+
environment: run.environment,
|
|
1674
|
+
index: run.index,
|
|
1675
|
+
outcome: { kind: "answered" },
|
|
1676
|
+
elapsed: null,
|
|
1677
|
+
socket: null,
|
|
1678
|
+
noticed: null,
|
|
1679
|
+
steps: [],
|
|
1680
|
+
probes: 0,
|
|
1681
|
+
retransmissions,
|
|
1682
|
+
acks: 0
|
|
1683
|
+
};
|
|
1684
|
+
const voided = (why) => ({
|
|
1685
|
+
...base,
|
|
1686
|
+
void: why,
|
|
1687
|
+
finding: null
|
|
1688
|
+
});
|
|
1689
|
+
const unsettled = events.find((event) => event.ev === "unsettled");
|
|
1690
|
+
if (unsettled !== void 0) return voided(`the table held ${String(unsettled["count"])} entries at the end of the wait before the run`);
|
|
1691
|
+
if (events.find((event) => event.ev === "done") === void 0) {
|
|
1692
|
+
const error = events.find((event) => event.ev === "error");
|
|
1693
|
+
return voided(error === void 0 ? "the client did not reach its end" : `the client did not reach its end: ${String(error["what"])}`);
|
|
1694
|
+
}
|
|
1695
|
+
if (run.lost === null) return voided("the prober wrote no clock line after the run, so nothing says what it dropped");
|
|
1696
|
+
if (run.lost > 0) return voided(`the prober dropped ${run.lost} records`);
|
|
1697
|
+
const full = events.find((event) => event.ev === "full");
|
|
1698
|
+
if (full === void 0) return voided("the client wrote no line for its fill");
|
|
1699
|
+
const stall = events.find((event) => event.ev === "stall");
|
|
1700
|
+
const closing = events.find((event) => event.ev === "closing");
|
|
1701
|
+
const drained = events.find((event) => event.ev === "drained");
|
|
1702
|
+
const closed = events.find((event) => event.ev === "closed");
|
|
1703
|
+
const neighbor = (run.server ?? []).filter((line) => line.ev === "count" && line.t >= full.t && (stall === void 0 || line.t <= stall.t)).at(-1);
|
|
1704
|
+
const conntrack = {
|
|
1705
|
+
flows: numberOr(full, "flows") ?? 0,
|
|
1706
|
+
count: numberOr(full, "count") ?? 0,
|
|
1707
|
+
neighbor: numberOr(neighbor, "count"),
|
|
1708
|
+
stall: stallOf(stall),
|
|
1709
|
+
firstFall: untilField(closing, drained, "first_fall"),
|
|
1710
|
+
zero: untilField(closing, drained, "zero"),
|
|
1711
|
+
closed: numberOr(closed, "count"),
|
|
1712
|
+
lowest: numberOr(drained ?? stall, "lowest"),
|
|
1713
|
+
after: numberOr(stall, "after")
|
|
1714
|
+
};
|
|
1715
|
+
const found = findingsOf$1(conntrack, full["errno"], expected);
|
|
1716
|
+
return {
|
|
1717
|
+
...base,
|
|
1718
|
+
void: null,
|
|
1719
|
+
finding: found.length === 0 ? null : found.join(". "),
|
|
1720
|
+
conntrack
|
|
1721
|
+
};
|
|
1722
|
+
};
|
|
1723
|
+
//#endregion
|
|
1724
|
+
//#region src/pause.ts
|
|
1725
|
+
/** This expression finds the stamp of a log line of a broker, as its console writes it, on UTC. */
|
|
1726
|
+
const LOG_TIME = /\[(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d),(\d{3})\]/;
|
|
1727
|
+
/** This function gives the time of a log line on the monotonic clock of a member, in ns, from the `start` line of that member. It is null for a line with no stamp. */
|
|
1728
|
+
const logTimeOf = (line, start) => {
|
|
1729
|
+
const found = LOG_TIME.exec(line);
|
|
1730
|
+
if (found === null) return null;
|
|
1731
|
+
const wall = Date.UTC(Number(found[1]), Number(found[2]) - 1, Number(found[3]), Number(found[4]), Number(found[5]), Number(found[6]), Number(found[7]));
|
|
1732
|
+
return start.t + (wall - Number(start["wall"])) * 1e6;
|
|
1733
|
+
};
|
|
1734
|
+
/** This function gives the time between two instants of the monotonic clock, in s, to 1 microsecond. */
|
|
1735
|
+
const secondsBetween = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
|
|
1736
|
+
const isInside = (time, band) => {
|
|
1737
|
+
const seconds = valueIn(time, "s");
|
|
1738
|
+
return seconds >= valueIn(band.low, "s") && seconds <= valueIn(band.high, "s");
|
|
1739
|
+
};
|
|
1740
|
+
const bandText = (band) => `[${valueIn(band.low, "s")}, ${valueIn(band.high, "s")}] s`;
|
|
1741
|
+
const isChange = (event) => event.ev === "assigned" || event.ev === "revoked" || event.ev === "lost";
|
|
1742
|
+
/** This function gives the partitions that a line of the rebalance listener names, in its field `partitions` or `held`. */
|
|
1743
|
+
const partitionsOf = (event, field) => event[field] ?? [];
|
|
1744
|
+
/** This function gives the last line of the rebalance listener in a list of lines. */
|
|
1745
|
+
const lastChange = (events) => events.findLast(isChange);
|
|
1746
|
+
/** This function gives the offset that a commit names for a partition, or null. */
|
|
1747
|
+
const offsetOf = (event, partition) => {
|
|
1748
|
+
const value = event["offsets"]?.[String(partition)];
|
|
1749
|
+
return typeof value === "number" ? value : null;
|
|
1750
|
+
};
|
|
1751
|
+
/**
|
|
1752
|
+
* This function tells if the two members can take the fault. Each member holds one partition, the
|
|
1753
|
+
* two partitions differ, and the first member has two acknowledged commits on its partition after its
|
|
1754
|
+
* last change. Thus the first member reads and commits at a steady rate.
|
|
1755
|
+
*/
|
|
1756
|
+
const pauseReady = (first, other) => {
|
|
1757
|
+
const mine = lastChange(first);
|
|
1758
|
+
const theirs = lastChange(other);
|
|
1759
|
+
if (mine === void 0 || theirs === void 0) return false;
|
|
1760
|
+
const held = partitionsOf(mine, "held");
|
|
1761
|
+
const others = partitionsOf(theirs, "held");
|
|
1762
|
+
if (held.length !== 1 || others.length !== 1 || held[0] === others[0]) return false;
|
|
1763
|
+
return first.filter((event) => event.ev === "commit" && event.t > mine.t && offsetOf(event, held[0]) !== null).length >= 2;
|
|
1764
|
+
};
|
|
1765
|
+
/** This function tells if a log line of a broker tells that a member left a group, under either protocol. */
|
|
1766
|
+
const isFenceLine = (line) => /Member \S+ fenced from the group because|Member \S+ in group \S+ has failed, removing it from the group/.test(line);
|
|
1767
|
+
/** This function tells if a fence line names a group. The two protocols write the name in different words. */
|
|
1768
|
+
const namesGroup = (line, group) => line.includes(`[GroupId ${group}]`) || line.includes(`in group ${group} has`);
|
|
1769
|
+
/** This function compares a reading to the law, and gives each disagreement. */
|
|
1770
|
+
const findingsOf = (reading, prediction, fences, otherReports) => {
|
|
1771
|
+
const found = [];
|
|
1772
|
+
const { commit, fence, takeover } = reading;
|
|
1773
|
+
if (prediction.fence === null) {
|
|
1774
|
+
if (fences.length > 0) found.push(`the group removed a member: ${fences[0]}`);
|
|
1775
|
+
if (otherReports) found.push("the other member reported the partition");
|
|
1776
|
+
} else {
|
|
1777
|
+
if (fence === null) found.push("no log line tells of a fence, and the other member did not report the partition");
|
|
1778
|
+
else if (!isInside(fence, prediction.fence)) found.push(`the fence is ${valueIn(fence, "s")} s, outside ${bandText(prediction.fence)}`);
|
|
1779
|
+
if (reading.otherFirst !== reading.lastAck) found.push(`the other member started at offset ${reading.otherFirst}, and the last commit is ${reading.lastAck}`);
|
|
1780
|
+
}
|
|
1781
|
+
if (prediction.takeover !== null) {
|
|
1782
|
+
if (takeover === null) found.push("the other member did not report the partition");
|
|
1783
|
+
else if (!isInside(takeover, prediction.takeover)) found.push(`the other member reported the partition at ${valueIn(takeover, "s")} s, outside ${bandText(prediction.takeover)}`);
|
|
1784
|
+
}
|
|
1785
|
+
if (commit === null) {
|
|
1786
|
+
found.push("the member made no commit after the stop");
|
|
1787
|
+
return found;
|
|
1788
|
+
}
|
|
1789
|
+
switch (prediction.thaw) {
|
|
1790
|
+
case "fails":
|
|
1791
|
+
if (commit.name !== "CommitFailedException") found.push(`the commit after the stop gave ${commit.name ?? "an acknowledgment"}, not CommitFailedException`);
|
|
1792
|
+
break;
|
|
1793
|
+
case "acknowledged":
|
|
1794
|
+
if (!commit.acked) found.push(`the commit after the stop failed with ${commit.name}`);
|
|
1795
|
+
if (commit.epoch !== reading.epochBefore) found.push(`the epoch changed from ${reading.epochBefore} to ${commit.epoch}`);
|
|
1796
|
+
break;
|
|
1797
|
+
case "rejoin": {
|
|
1798
|
+
const within = prediction.rejoinWithin;
|
|
1799
|
+
if (reading.rejoin === null) found.push("the member did not get a new epoch");
|
|
1800
|
+
else if (within !== void 0 && valueIn(reading.rejoin, "s") > valueIn(within, "s")) found.push(`the new epoch came at ${valueIn(reading.rejoin, "s")} s, above ${valueIn(within, "s")} s`);
|
|
1801
|
+
if (!commit.acked) found.push(`the stale commit failed with ${commit.name}`);
|
|
1802
|
+
if (reading.reassigned) found.push("the member held the partition again before its commit");
|
|
1803
|
+
if (reading.committed !== reading.stale) found.push(`the committed offset is ${reading.committed}, and the stale commit named ${reading.stale}`);
|
|
1804
|
+
if (reading.stale === null || reading.otherLast === null || reading.stale >= reading.otherLast) found.push(`the stale commit ${reading.stale} is not below the progress ${reading.otherLast} of the other member`);
|
|
1805
|
+
break;
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1808
|
+
return found;
|
|
1809
|
+
};
|
|
1810
|
+
/** This function reads one run of a paused consumer against its law. */
|
|
1811
|
+
const measurePause = (run, prediction) => {
|
|
1812
|
+
const retransmissions = run.kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length;
|
|
1813
|
+
const base = {
|
|
1814
|
+
environment: run.environment,
|
|
1815
|
+
index: run.index,
|
|
1816
|
+
outcome: { kind: "answered" },
|
|
1817
|
+
elapsed: null,
|
|
1818
|
+
socket: null,
|
|
1819
|
+
noticed: null,
|
|
1820
|
+
steps: [],
|
|
1821
|
+
probes: 0,
|
|
1822
|
+
retransmissions,
|
|
1823
|
+
acks: 0
|
|
1824
|
+
};
|
|
1825
|
+
const voided = (why) => ({
|
|
1826
|
+
...base,
|
|
1827
|
+
void: why,
|
|
1828
|
+
finding: null
|
|
1829
|
+
});
|
|
1830
|
+
const { fault, heal, pause } = run;
|
|
1831
|
+
if (fault === null || pause === void 0) return voided("the partitions were not one for each member before the stop");
|
|
1832
|
+
if (heal === void 0 || heal === null) return voided("the stop did not end");
|
|
1833
|
+
const stop = fault.t0;
|
|
1834
|
+
const thaw = heal.t0;
|
|
1835
|
+
const before = run.events.filter((event) => event.t < stop);
|
|
1836
|
+
const after = run.events.filter((event) => event.t >= stop);
|
|
1837
|
+
const start = run.events.find((event) => event.ev === "start");
|
|
1838
|
+
if (start === void 0) return voided("the member wrote no start line");
|
|
1839
|
+
const mine = lastChange(before);
|
|
1840
|
+
const theirs = lastChange(pause.other.filter((event) => event.t < stop));
|
|
1841
|
+
const held = mine === void 0 ? [] : partitionsOf(mine, "held");
|
|
1842
|
+
const others = theirs === void 0 ? [] : partitionsOf(theirs, "held");
|
|
1843
|
+
const partition = held[0];
|
|
1844
|
+
if (partition === void 0 || held.length !== 1 || others.length !== 1 || others[0] === partition) return voided("the partitions were not one for each member before the stop");
|
|
1845
|
+
if (after.find((event) => event.ev === "pause")?.["in"] !== "process") return voided("the stop landed outside the processing of a batch");
|
|
1846
|
+
if (before.findLast((event) => event.ev === "read")?.["partition"] !== partition) return voided("the last read before the stop was on another partition");
|
|
1847
|
+
const lastCommit = before.findLast((event) => event.ev === "commit" && offsetOf(event, partition) !== null);
|
|
1848
|
+
if (lastCommit === void 0) return voided("the member had no acknowledged commit on its partition before the stop");
|
|
1849
|
+
const member = String(mine?.["member"]);
|
|
1850
|
+
const fenceAt = pause.fences.filter((line) => line.includes(member)).map((line) => logTimeOf(line, start)).find((time) => time !== null && time >= stop) ?? null;
|
|
1851
|
+
const taken = pause.other.find((event) => event.ev === "assigned" && event.t >= stop && partitionsOf(event, "partitions").includes(partition));
|
|
1852
|
+
if (prediction.fence !== null && fenceAt === null && taken !== void 0) return voided("the broker log lost the line of the fence");
|
|
1853
|
+
const thawed = after.find((event) => event.ev === "commit" || event.ev === "commit-fail");
|
|
1854
|
+
const rejoined = after.find((event) => event.ev === "rejoined");
|
|
1855
|
+
const ackAt = thawed?.t ?? Number.POSITIVE_INFINITY;
|
|
1856
|
+
const otherCommit = pause.other.findLast((event) => event.ev === "commit" && event.t < ackAt && offsetOf(event, partition) !== null);
|
|
1857
|
+
const committed = after.find((event) => event.ev === "committed");
|
|
1858
|
+
const epochOf = (event) => typeof event?.["epoch"] === "number" ? event["epoch"] : null;
|
|
1859
|
+
const pauseReading = {
|
|
1860
|
+
partition,
|
|
1861
|
+
fence: fenceAt === null ? null : secondsBetween(stop, fenceAt),
|
|
1862
|
+
takeover: taken === void 0 ? null : secondsBetween(stop, taken.t),
|
|
1863
|
+
commit: thawed === void 0 ? null : {
|
|
1864
|
+
acked: thawed.ev === "commit",
|
|
1865
|
+
name: typeof thawed["name"] === "string" ? thawed["name"] : null,
|
|
1866
|
+
message: typeof thawed["message"] === "string" ? thawed["message"] : null,
|
|
1867
|
+
epoch: epochOf(thawed)
|
|
1868
|
+
},
|
|
1869
|
+
epochBefore: epochOf(lastCommit),
|
|
1870
|
+
lastAck: offsetOf(lastCommit, partition),
|
|
1871
|
+
otherFirst: pause.other.find((event) => event.ev === "read" && event.t >= stop && event["partition"] === partition)?.["first"] ?? null,
|
|
1872
|
+
otherLast: otherCommit === void 0 ? null : offsetOf(otherCommit, partition),
|
|
1873
|
+
rejoin: rejoined === void 0 ? null : secondsBetween(thaw, rejoined.t),
|
|
1874
|
+
reassigned: after.some((event) => event.ev === "assigned" && event.t >= thaw && event.t <= ackAt && partitionsOf(event, "partitions").includes(partition)),
|
|
1875
|
+
committed: committed === void 0 ? null : offsetOf(committed, partition),
|
|
1876
|
+
stale: thawed === void 0 ? null : offsetOf(thawed, partition)
|
|
1877
|
+
};
|
|
1878
|
+
const findings = findingsOf(pauseReading, prediction, pause.fences, taken !== void 0);
|
|
1879
|
+
return {
|
|
1880
|
+
...base,
|
|
1881
|
+
void: null,
|
|
1882
|
+
finding: findings.length === 0 ? null : findings.join(". "),
|
|
1883
|
+
pause: pauseReading
|
|
1884
|
+
};
|
|
1885
|
+
};
|
|
1886
|
+
//#endregion
|
|
1887
|
+
//#region src/ports.ts
|
|
1888
|
+
/** This errno ends a search that found no port. */
|
|
1889
|
+
const NO_PORT = "EADDRNOTAVAIL";
|
|
1890
|
+
const portsOf = (line) => Array.isArray(line["ports"]) ? line["ports"].filter((port) => typeof port === "number") : [];
|
|
1891
|
+
/** This function says if the ports of the parity of the low port come first in one search. */
|
|
1892
|
+
const lowFirst = (ports, low) => {
|
|
1893
|
+
const odd = ports.findIndex((port) => (port - low) % 2 !== 0);
|
|
1894
|
+
return odd < 0 || ports.slice(odd).every((port) => (port - low) % 2 !== 0);
|
|
1895
|
+
};
|
|
1896
|
+
/**
|
|
1897
|
+
* This function reads one run of a port cell. The `round` lines give each search. The `closing` and
|
|
1898
|
+
* `freed` lines give when each held port comes back. The records of the prober give the
|
|
1899
|
+
* retransmissions.
|
|
1900
|
+
*/
|
|
1901
|
+
const measurePorts = (run, expected) => {
|
|
1902
|
+
const { events, kernel } = run;
|
|
1903
|
+
const retransmissions = kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length;
|
|
1904
|
+
const base = {
|
|
1905
|
+
environment: run.environment,
|
|
1906
|
+
index: run.index,
|
|
1907
|
+
outcome: { kind: "answered" },
|
|
1908
|
+
elapsed: null,
|
|
1909
|
+
socket: null,
|
|
1910
|
+
noticed: null,
|
|
1911
|
+
steps: [],
|
|
1912
|
+
probes: 0,
|
|
1913
|
+
retransmissions,
|
|
1914
|
+
acks: 0
|
|
1915
|
+
};
|
|
1916
|
+
const voided = (why) => ({
|
|
1917
|
+
...base,
|
|
1918
|
+
void: why,
|
|
1919
|
+
finding: null
|
|
1920
|
+
});
|
|
1921
|
+
if (events.find((event) => event.ev === "done") === void 0) return voided("the client did not reach its end");
|
|
1922
|
+
if (run.lost === null) return voided("the prober wrote no clock line after the run, so nothing says what it dropped");
|
|
1923
|
+
if (run.lost > 0) return voided(`the prober dropped ${run.lost} records`);
|
|
1924
|
+
const rounds = events.filter((event) => event.ev === "round");
|
|
1925
|
+
const closing = events.find((event) => event.ev === "closing");
|
|
1926
|
+
const freedLines = events.filter((event) => event.ev === "freed");
|
|
1927
|
+
const counts = {};
|
|
1928
|
+
for (const round of rounds) {
|
|
1929
|
+
const count = String(portsOf(round).length);
|
|
1930
|
+
counts[count] = (counts[count] ?? 0) + 1;
|
|
1931
|
+
}
|
|
1932
|
+
const seen = [...new Set(rounds.flatMap((round) => portsOf(round).map((port) => port - expected.low)))].sort((a, b) => a - b);
|
|
1933
|
+
const refusal = Math.max(0, ...rounds.map((round) => typeof round["refusal"] === "number" ? round["refusal"] : 0));
|
|
1934
|
+
const first = freedLines[0];
|
|
1935
|
+
const last = freedLines.at(-1);
|
|
1936
|
+
const ports = {
|
|
1937
|
+
rounds: rounds.length,
|
|
1938
|
+
counts,
|
|
1939
|
+
seen,
|
|
1940
|
+
errnos: [...new Set(rounds.map((round) => String(round["errno"])))],
|
|
1941
|
+
refusal: quantity(Math.round(refusal / 1e3) / 1e6, "s"),
|
|
1942
|
+
firstFreed: closing === void 0 || first === void 0 ? null : between$2(closing.t, first.t),
|
|
1943
|
+
lastFreed: closing === void 0 || last === void 0 ? null : between$2(closing.t, last.t),
|
|
1944
|
+
freed: freedLines.length,
|
|
1945
|
+
tries: last === void 0 || typeof last["tries"] !== "number" ? 0 : last["tries"]
|
|
1946
|
+
};
|
|
1947
|
+
const found = [];
|
|
1948
|
+
const searches = expected.rounds * expected.destinations;
|
|
1949
|
+
if (ports.rounds !== searches) found.push(`${ports.rounds} searches, not ${searches}`);
|
|
1950
|
+
const stray = Object.keys(counts).filter((count) => !expected.counts.includes(Number(count)));
|
|
1951
|
+
if (stray.length > 0) found.push(`a search gave ${stray.join(" or ")} connects, outside ${expected.counts.join(" or ")}`);
|
|
1952
|
+
if (seen.join() !== expected.seen.join()) found.push(`the ports at offsets ${seen.join(" ")} were used, not ${expected.seen.join(" ")}`);
|
|
1953
|
+
if (ports.errnos.join() !== NO_PORT) found.push(`the searches ended with ${ports.errnos.join(" and ")}, not ${NO_PORT}`);
|
|
1954
|
+
if (valueIn(ports.refusal, "s") > valueIn(expected.refusedWithin, "s")) found.push(`a refused connect took ${valueIn(ports.refusal, "s")} s, above ${valueIn(expected.refusedWithin, "s")} s`);
|
|
1955
|
+
if (expected.share !== void 0) {
|
|
1956
|
+
const { count, low, high } = expected.share;
|
|
1957
|
+
const share = (counts[String(count)] ?? 0) / Math.max(1, ports.rounds);
|
|
1958
|
+
if (share < low || share > high) found.push(`the share of ${count} connects is ${share}, outside [${low}, ${high}]`);
|
|
1959
|
+
}
|
|
1960
|
+
if (expected.lowFirst === true && !rounds.every((round) => lowFirst(portsOf(round), expected.low))) found.push("a search got a port of the other parity before each port of the parity of the low port");
|
|
1961
|
+
if (expected.retransmissions !== void 0 && retransmissions !== expected.retransmissions) found.push(`${retransmissions} retransmissions, not ${expected.retransmissions}`);
|
|
1962
|
+
if (expected.freed !== void 0) {
|
|
1963
|
+
const held = Math.max(...expected.counts);
|
|
1964
|
+
if (ports.freed !== held) found.push(`${ports.freed} ports came back, not ${held}`);
|
|
1965
|
+
for (const [name, time] of [["first", ports.firstFreed], ["last", ports.lastFreed]]) if (time === null || !inside(time, expected.freed)) found.push(`the ${name} port came back at ${time === null ? "no time" : `${valueIn(time, "s")} s`}, outside ${bandWords(expected.freed)}`);
|
|
1966
|
+
if (ports.tries === 0) found.push("no connect was refused before the ports came back");
|
|
1967
|
+
}
|
|
1968
|
+
return {
|
|
1969
|
+
...base,
|
|
1970
|
+
void: null,
|
|
1971
|
+
finding: found.length === 0 ? null : found.join(". "),
|
|
1972
|
+
ports
|
|
1973
|
+
};
|
|
1974
|
+
};
|
|
1975
|
+
//#endregion
|
|
1148
1976
|
//#region src/recovery.ts
|
|
1149
1977
|
/**
|
|
1150
1978
|
* This module holds the pure parts of a recovery drill. These are the scripts of the shell, the reader
|
|
@@ -1516,12 +2344,32 @@ const writesOf = (run, after, primary) => {
|
|
|
1516
2344
|
held: kept === null ? null : kept.has(n)
|
|
1517
2345
|
};
|
|
1518
2346
|
});
|
|
2347
|
+
const duplicates = holds === null ? null : [...holds].sort((a, b) => a - b).filter((n, at, all) => all[at - 1] === n);
|
|
2348
|
+
const oldHolds = run.group?.oldHolds;
|
|
1519
2349
|
return {
|
|
1520
2350
|
sent: sent.length,
|
|
1521
2351
|
acked: acks.size,
|
|
1522
2352
|
old,
|
|
1523
2353
|
lost,
|
|
1524
|
-
unknown
|
|
2354
|
+
unknown,
|
|
2355
|
+
duplicates,
|
|
2356
|
+
...oldHolds === void 0 ? {} : { sets: setsOf(run, acks.keys(), unknown, new Set(oldHolds)) }
|
|
2357
|
+
};
|
|
2358
|
+
};
|
|
2359
|
+
/**
|
|
2360
|
+
* This function splits the writes of a pair by its two final reads. A write with no reply is `kept`
|
|
2361
|
+
* on the new primary. It is `stranded` on the old primary alone, or `absent` from each server.
|
|
2362
|
+
*/
|
|
2363
|
+
const setsOf = (run, acked, unknown, old) => {
|
|
2364
|
+
const kept = unknown.filter((each) => each.held === true).length;
|
|
2365
|
+
const stranded = unknown.filter((each) => each.held !== true && old.has(each.n)).length;
|
|
2366
|
+
return {
|
|
2367
|
+
unknown: unknown.length,
|
|
2368
|
+
kept,
|
|
2369
|
+
stranded,
|
|
2370
|
+
absent: unknown.length - kept - stranded,
|
|
2371
|
+
missing: [...acked].filter((n) => !old.has(n)).length,
|
|
2372
|
+
warnings: run.events.filter((event) => event.ev === "notice" && event["severity"] === "WARNING").length
|
|
1525
2373
|
};
|
|
1526
2374
|
};
|
|
1527
2375
|
/** This function gives the errors that the client met after an instant: each kind, its first time, and its count. */
|
|
@@ -1632,9 +2480,10 @@ const measureGroup = (cell, run) => {
|
|
|
1632
2480
|
const heal = run.heal ?? null;
|
|
1633
2481
|
const lands = expected?.lands ?? "new";
|
|
1634
2482
|
const cluster = cell.cluster !== void 0;
|
|
1635
|
-
const
|
|
2483
|
+
const kafka = cell.kafka !== void 0;
|
|
2484
|
+
const replica = cell.replicaSet !== void 0 || cluster || kafka;
|
|
1636
2485
|
const stall = prediction.detects === false && prediction.recovers !== void 0;
|
|
1637
|
-
const stages = cluster ? clusterStagesOf(run, t0, run.fault.t1) : replica ? replicaSetStagesOf(run, t0, run.fault.t1, lands) : stagesOf(timeline, t0);
|
|
2486
|
+
const stages = kafka ? {} : cluster ? clusterStagesOf(run, t0, run.fault.t1) : replica ? replicaSetStagesOf(run, t0, run.fault.t1, lands) : stagesOf(timeline, t0);
|
|
1638
2487
|
const after = lands === "same" && heal !== null && (!replica || stall) ? heal.t0 : t0;
|
|
1639
2488
|
const sentAt = /* @__PURE__ */ new Map();
|
|
1640
2489
|
for (const event of events) if (event.ev === "sent") sentAt.set(Number(event["n"]), event.t);
|
|
@@ -1666,14 +2515,14 @@ const measureGroup = (cell, run) => {
|
|
|
1666
2515
|
if (voided === null && split !== void 0 && (elected === void 0 || elected.t > split.t)) voided = "the election split its votes and runs again after 2 × failover-timeout, which the band does not bound";
|
|
1667
2516
|
const odown = stages["+odown"];
|
|
1668
2517
|
if (voided === null && odown !== void 0 && stages["+try-failover"] === void 0) voided = "no Sentinel started a failover after +odown: each was in the hold-off of 2 × failover-timeout after an earlier attempt";
|
|
1669
|
-
if (voided === null && cell.hold !== void 0 && prediction.detects !== null) {
|
|
2518
|
+
if (voided === null && cell.hold !== void 0 && (prediction.detects !== null || kafka)) {
|
|
1670
2519
|
if (heal === null) voided = "the fault never lifted";
|
|
1671
2520
|
else {
|
|
1672
2521
|
const late = (heal.t0 - t0) / 1e9 - valueIn(cell.hold, "s");
|
|
1673
2522
|
if (late > valueIn(INJECT, "s")) voided = `the lift came ${Math.round(late * 1e4) / 10} ms after its hold: the injector was late, not the group`;
|
|
1674
2523
|
}
|
|
1675
2524
|
}
|
|
1676
|
-
const firstAddress = cluster ? String(events.find((event) => event.ev === "connected")?.["at"] ?? "") : `${run.group?.members[primary]?.ip}:${(cell.group ?? cell.replicaSet)?.port}`;
|
|
2525
|
+
const firstAddress = cluster ? String(events.find((event) => event.ev === "connected")?.["at"] ?? "") : `${run.group?.members[primary]?.ip}:${(cell.group ?? cell.replicaSet ?? cell.kafka)?.port}`;
|
|
1677
2526
|
const assumed = prediction.detects === null ? void 0 : prediction.rtt;
|
|
1678
2527
|
const first = kernel.find((record) => record.netns === run.netns && record.dst === firstAddress && record.tcp?.srtt !== void 0);
|
|
1679
2528
|
const rtt = first?.tcp?.srtt === void 0 ? null : valueIn(first.tcp.srtt, "us") / 1e3;
|
|
@@ -1693,11 +2542,13 @@ const measureGroup = (cell, run) => {
|
|
|
1693
2542
|
const findings = [];
|
|
1694
2543
|
const switched = replica ? stages["elected"] : stages["+switch-master"];
|
|
1695
2544
|
if (expected?.failover === false && switched !== void 0) findings.push(`the group failed over ${switched.value} s after the fault, where it should not`);
|
|
2545
|
+
const leaderAfter = run.group?.after ?? null;
|
|
2546
|
+
if (kafka && expected !== void 0 && leaderAfter !== null && expected.failover !== (leaderAfter !== primary)) findings.push(expected.failover ? `the leader stayed on ${primary}, where it should change` : `the leader moved to ${leaderAfter}, where it should stay on ${primary}`);
|
|
1696
2547
|
for (const stage of expected?.quiet ?? []) {
|
|
1697
2548
|
const at = stages[stage];
|
|
1698
2549
|
if (at !== void 0) findings.push(`${stage} came ${at.value} s after the fault, where it should not`);
|
|
1699
2550
|
}
|
|
1700
|
-
if (accepted !== void 0 && lands === "same" && !cluster && to !== primary) findings.push(`the client wrote to ${to ?? String(accepted["at"])}, where it should stay on ${primary}`);
|
|
2551
|
+
if (accepted !== void 0 && lands === "same" && !cluster && !kafka && to !== primary) findings.push(`the client wrote to ${to ?? String(accepted["at"])}, where it should stay on ${primary}`);
|
|
1701
2552
|
for (const error of expected?.errors ?? []) {
|
|
1702
2553
|
const met = group.errors.find((each) => each.name === error.name);
|
|
1703
2554
|
const noise = valueIn(noiseOf(cell), "s");
|
|
@@ -1710,9 +2561,22 @@ const measureGroup = (cell, run) => {
|
|
|
1710
2561
|
if (lost === null) findings.push("the group did not converge, thus the count of its lost writes is not known");
|
|
1711
2562
|
else if (lost.length < low || lost.length > high) findings.push(`the group lost ${lost.length} accepted writes, outside [${low}, ${high}]`);
|
|
1712
2563
|
}
|
|
2564
|
+
if (expected?.duplicates !== void 0) {
|
|
2565
|
+
const { low, high } = expected.duplicates;
|
|
2566
|
+
const copies = group.writes.duplicates ?? null;
|
|
2567
|
+
if (copies === null) findings.push("the group did not converge, thus the count of its extra copies is not known");
|
|
2568
|
+
else if (run.group?.holds?.length === 0) findings.push("the group holds no write, thus the count of its extra copies says nothing");
|
|
2569
|
+
else if (copies.length < low || copies.length > high) findings.push(`the group holds ${copies.length} extra copies, outside [${low}, ${high}]`);
|
|
2570
|
+
}
|
|
2571
|
+
const { sets } = group.writes;
|
|
2572
|
+
for (const [name, band] of Object.entries(expected?.outcomes ?? {})) {
|
|
2573
|
+
const count = sets?.[name];
|
|
2574
|
+
if (count === void 0) findings.push(`the run has no count of ${name}`);
|
|
2575
|
+
else if (count < band.low || count > band.high) findings.push(`the count of ${name} is ${count}, outside [${band.low}, ${band.high}]`);
|
|
2576
|
+
}
|
|
1713
2577
|
if (group.recovery !== void 0) findings.push(...recoveryFindings(group.recovery, group.writes));
|
|
1714
2578
|
if (accepted === void 0) {
|
|
1715
|
-
if (prediction.detects !== null) findings.push(lands === "new" ? "the client never wrote to a new primary in the window" : "the client never wrote again in the window");
|
|
2579
|
+
if (prediction.detects !== null || kafka) findings.push(lands === "new" ? "the client never wrote to a new primary in the window" : "the client never wrote again in the window");
|
|
1716
2580
|
return {
|
|
1717
2581
|
...silent,
|
|
1718
2582
|
void: voided,
|
|
@@ -1981,7 +2845,22 @@ const requestOf = (events) => {
|
|
|
1981
2845
|
};
|
|
1982
2846
|
/** This function reads the socket of one run against its cell, as `measure` does without the second request of an HTTP client. */
|
|
1983
2847
|
const measureSocket = (cell, run) => {
|
|
1984
|
-
if (
|
|
2848
|
+
if (run.missing !== void 0) return {
|
|
2849
|
+
environment: run.environment,
|
|
2850
|
+
index: run.index,
|
|
2851
|
+
outcome: { kind: "answered" },
|
|
2852
|
+
elapsed: null,
|
|
2853
|
+
socket: null,
|
|
2854
|
+
noticed: null,
|
|
2855
|
+
steps: [],
|
|
2856
|
+
probes: 0,
|
|
2857
|
+
retransmissions: 0,
|
|
2858
|
+
acks: 0,
|
|
2859
|
+
void: `the environment cannot do this cell: ${run.missing}`,
|
|
2860
|
+
finding: null
|
|
2861
|
+
};
|
|
2862
|
+
if (cell.prediction.detects === null && cell.prediction.pause !== void 0) return measurePause(run, cell.prediction.pause);
|
|
2863
|
+
if (cell.group !== void 0 || cell.replicaSet !== void 0 || cell.cluster !== void 0 || cell.pair !== void 0 || cell.kafka !== void 0) {
|
|
1985
2864
|
const read = measureGroup(cell, run);
|
|
1986
2865
|
return {
|
|
1987
2866
|
environment: run.environment,
|
|
@@ -1999,6 +2878,9 @@ const measureSocket = (cell, run) => {
|
|
|
1999
2878
|
group: read.group
|
|
2000
2879
|
};
|
|
2001
2880
|
}
|
|
2881
|
+
if (cell.prediction.detects === null && cell.prediction.queue !== void 0) return measureQueue(run, cell.prediction.queue);
|
|
2882
|
+
if (cell.prediction.detects === null && cell.prediction.ports !== void 0) return measurePorts(run, cell.prediction.ports);
|
|
2883
|
+
if (cell.prediction.detects === null && cell.prediction.conntrack !== void 0) return measureConntrack(run, cell.prediction.conntrack);
|
|
2002
2884
|
if (cell.prediction.detects !== null && cell.prediction.driver !== void 0) return measureDriver(cell, run);
|
|
2003
2885
|
const { events, kernel } = run;
|
|
2004
2886
|
const ready = events.find((event) => event.ev === "ready");
|
|
@@ -2129,7 +3011,13 @@ const measureSocket = (cell, run) => {
|
|
|
2129
3011
|
forgot: forgotten === void 0 ? null : between(lastRx, forgotten.t)
|
|
2130
3012
|
};
|
|
2131
3013
|
if (prediction.detects === null) {
|
|
2132
|
-
const
|
|
3014
|
+
const load = run.streams !== void 0 ? streamsLoadOf(events, run.streams)?.pool ?? null : run.schedule === void 0 ? void 0 : loadOf(events, run.schedule);
|
|
3015
|
+
if (load === null) return {
|
|
3016
|
+
...base,
|
|
3017
|
+
...nothing,
|
|
3018
|
+
void: "the client never started its schedule"
|
|
3019
|
+
};
|
|
3020
|
+
const sends = load === void 0 ? events.filter((event) => event.ev === "sent").slice(1) : [];
|
|
2133
3021
|
const answers = events.some((event) => event.ev === "replied") ? "replied" : "data";
|
|
2134
3022
|
const samples = sends.flatMap((send, n) => {
|
|
2135
3023
|
const next = sends[n + 1]?.t ?? Infinity;
|
|
@@ -2155,17 +3043,19 @@ const measureSocket = (cell, run) => {
|
|
|
2155
3043
|
}, run)) };
|
|
2156
3044
|
const reconnect = prediction.reconnects === void 0 || ended === void 0 ? void 0 : reconnectOf(run, ended, cookie, drop?.line.t ?? ended);
|
|
2157
3045
|
const late = reconnect === void 0 || prediction.reconnects === void 0 ? null : reconnectFinding(prediction.reconnects, reconnect, noiseOf(cell));
|
|
3046
|
+
const counted = prediction.counts === void 0 ? null : countFinding(prediction.counts, run.server ?? []);
|
|
2158
3047
|
return {
|
|
2159
3048
|
...base,
|
|
2160
3049
|
...nothing,
|
|
2161
3050
|
...chain,
|
|
2162
3051
|
...reconnect === void 0 ? {} : { reconnect },
|
|
2163
|
-
finding: late,
|
|
3052
|
+
finding: late ?? counted,
|
|
2164
3053
|
outcome: ended !== void 0 ? outcome : { kind: "answered" },
|
|
2165
3054
|
...from === void 0 || until === void 0 ? {} : { elapsed: between(from, until) },
|
|
2166
3055
|
noticed: prediction.from === void 0 ? null : noticed,
|
|
2167
3056
|
steps: retransmitted.filter((record) => record.mono > request.t).map((record) => between(request.t, record.mono)),
|
|
2168
3057
|
samples,
|
|
3058
|
+
...load === void 0 ? {} : { load },
|
|
2169
3059
|
reordering: reorderingOf(socket.filter((record) => record.mono > request.t), peerRecords.filter((record) => record.mono > request.t)),
|
|
2170
3060
|
...crossings,
|
|
2171
3061
|
void: voided
|
|
@@ -2918,7 +3808,7 @@ const TAIL_LINES = 60;
|
|
|
2918
3808
|
const OUTPUT_LINES = 2e3;
|
|
2919
3809
|
/** This time is how long a write of a hop waits for the line that confirms it, in milliseconds. */
|
|
2920
3810
|
const CONFIRM_MS = 1e4;
|
|
2921
|
-
const sleep$
|
|
3811
|
+
const sleep$6 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
2922
3812
|
/** This function gives the arguments of the injector for the options of a fault. */
|
|
2923
3813
|
const optionArgs = (options) => [
|
|
2924
3814
|
...options.direction === void 0 ? [] : ["--direction", options.direction],
|
|
@@ -3112,6 +4002,7 @@ const clientOf = async (kind) => {
|
|
|
3112
4002
|
clientImage: await imageOf(client.image),
|
|
3113
4003
|
command: client.command,
|
|
3114
4004
|
clientKind: typeof kind === "string" ? kind : "image",
|
|
4005
|
+
...client.capabilities === void 0 || client.capabilities.length === 0 ? {} : { capabilities: client.capabilities },
|
|
3115
4006
|
...typeof client.image === "string" ? {} : { source: client.image }
|
|
3116
4007
|
};
|
|
3117
4008
|
};
|
|
@@ -3143,18 +4034,21 @@ const deviceOf = (route) => {
|
|
|
3143
4034
|
*/
|
|
3144
4035
|
const openLab = async (target, kind = "tcp", topology) => {
|
|
3145
4036
|
const image = await harnessImage();
|
|
3146
|
-
const { clientImage, command, clientKind, source } = await clientOf(kind);
|
|
4037
|
+
const { clientImage, command, clientKind, capabilities, source } = await clientOf(kind);
|
|
3147
4038
|
const hops = topology?.hops ?? [];
|
|
3148
4039
|
const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();
|
|
3149
4040
|
const runtime = await getContainerRuntimeClient();
|
|
3150
4041
|
const opened = [];
|
|
3151
4042
|
const networks = [];
|
|
3152
4043
|
const serverLines = lines();
|
|
4044
|
+
let restoreHost = () => Promise.resolve([]);
|
|
3153
4045
|
let fabric;
|
|
3154
4046
|
const close = async () => {
|
|
4047
|
+
const restored = await restoreHost().then(() => null, (error) => error);
|
|
3155
4048
|
await fabric?.close();
|
|
3156
4049
|
for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
|
|
3157
4050
|
for (const network of networks.reverse()) await network.stop().catch(() => void 0);
|
|
4051
|
+
if (restored !== null) throw restored;
|
|
3158
4052
|
};
|
|
3159
4053
|
/** This function gives the address of a container on a network, as Docker has it at this time. A container can join a network after its start. */
|
|
3160
4054
|
const ipOn = async (container, network) => {
|
|
@@ -3459,7 +4353,35 @@ const openLab = async (target, kind = "tcp", topology) => {
|
|
|
3459
4353
|
};
|
|
3460
4354
|
};
|
|
3461
4355
|
const proxies = inChain.filter((hop) => hop.hop.kind === "proxy");
|
|
4356
|
+
let hostAdmin;
|
|
4357
|
+
const hostChanges = [];
|
|
4358
|
+
let hostDone;
|
|
4359
|
+
restoreHost = async () => {
|
|
4360
|
+
if (hostDone !== void 0) return hostDone;
|
|
4361
|
+
const restored = [];
|
|
4362
|
+
for (const change of [...hostChanges].reverse()) {
|
|
4363
|
+
await output(hostAdmin, [
|
|
4364
|
+
"sysctl",
|
|
4365
|
+
"-q",
|
|
4366
|
+
"-w",
|
|
4367
|
+
`${change.name}=${change.before}`
|
|
4368
|
+
]);
|
|
4369
|
+
const read = await output(hostAdmin, [
|
|
4370
|
+
"sysctl",
|
|
4371
|
+
"-n",
|
|
4372
|
+
change.name
|
|
4373
|
+
]);
|
|
4374
|
+
if (read !== change.before) throw new Error(`${change.name} reads ${read} after the restore, not ${change.before}`);
|
|
4375
|
+
restored.unshift({
|
|
4376
|
+
...change,
|
|
4377
|
+
restored: read
|
|
4378
|
+
});
|
|
4379
|
+
}
|
|
4380
|
+
hostDone = restored;
|
|
4381
|
+
return restored;
|
|
4382
|
+
};
|
|
3462
4383
|
return {
|
|
4384
|
+
serverIp,
|
|
3463
4385
|
targetIp,
|
|
3464
4386
|
targetPort,
|
|
3465
4387
|
clientIp,
|
|
@@ -3504,6 +4426,53 @@ const openLab = async (target, kind = "tcp", topology) => {
|
|
|
3504
4426
|
]);
|
|
3505
4427
|
return read;
|
|
3506
4428
|
},
|
|
4429
|
+
async namespaceSetup(commands) {
|
|
4430
|
+
for (const command of commands) {
|
|
4431
|
+
await output(admin, [...command]);
|
|
4432
|
+
await output(injector, [...command]);
|
|
4433
|
+
}
|
|
4434
|
+
},
|
|
4435
|
+
async hostSysctl(settings) {
|
|
4436
|
+
if (settings.length === 0) return;
|
|
4437
|
+
if (hostAdmin === void 0) {
|
|
4438
|
+
hostAdmin = await new GenericContainer(image).withNetworkMode("host").withPrivilegedMode().start();
|
|
4439
|
+
opened.push(hostAdmin);
|
|
4440
|
+
}
|
|
4441
|
+
for (const { name, value, guard } of settings) {
|
|
4442
|
+
if (guard !== void 0) {
|
|
4443
|
+
const reads = Number(await output(hostAdmin, [
|
|
4444
|
+
"sysctl",
|
|
4445
|
+
"-n",
|
|
4446
|
+
guard.name
|
|
4447
|
+
]));
|
|
4448
|
+
if (!(reads <= guard.atMost)) throw new Error(`${guard.name} of the host reads ${reads}, above ${guard.atMost}`);
|
|
4449
|
+
}
|
|
4450
|
+
const before = await output(hostAdmin, [
|
|
4451
|
+
"sysctl",
|
|
4452
|
+
"-n",
|
|
4453
|
+
name
|
|
4454
|
+
]);
|
|
4455
|
+
hostChanges.push({
|
|
4456
|
+
name,
|
|
4457
|
+
before,
|
|
4458
|
+
set: value,
|
|
4459
|
+
restored: null
|
|
4460
|
+
});
|
|
4461
|
+
await output(hostAdmin, [
|
|
4462
|
+
"sysctl",
|
|
4463
|
+
"-q",
|
|
4464
|
+
"-w",
|
|
4465
|
+
`${name}=${value}`
|
|
4466
|
+
]);
|
|
4467
|
+
const set = await output(hostAdmin, [
|
|
4468
|
+
"sysctl",
|
|
4469
|
+
"-n",
|
|
4470
|
+
name
|
|
4471
|
+
]);
|
|
4472
|
+
if (set !== value) throw new Error(`${name} of the host reads ${set} after the write, not ${value}`);
|
|
4473
|
+
}
|
|
4474
|
+
},
|
|
4475
|
+
restoreHost: () => restoreHost(),
|
|
3507
4476
|
async configure(values) {
|
|
3508
4477
|
const read = {};
|
|
3509
4478
|
if (Object.keys(values).length === 0) return read;
|
|
@@ -3524,7 +4493,7 @@ const openLab = async (target, kind = "tcp", topology) => {
|
|
|
3524
4493
|
return stamped(where, [
|
|
3525
4494
|
"/opt/harness/inject",
|
|
3526
4495
|
kind,
|
|
3527
|
-
peer,
|
|
4496
|
+
options.peers?.join(",") ?? peer,
|
|
3528
4497
|
...optionArgs(options),
|
|
3529
4498
|
...via
|
|
3530
4499
|
]);
|
|
@@ -3534,7 +4503,7 @@ const openLab = async (target, kind = "tcp", topology) => {
|
|
|
3534
4503
|
await output(where, [
|
|
3535
4504
|
"/opt/harness/inject",
|
|
3536
4505
|
LIFT[kind],
|
|
3537
|
-
peer,
|
|
4506
|
+
options.peers?.join(",") ?? peer,
|
|
3538
4507
|
...optionArgs(options),
|
|
3539
4508
|
...via
|
|
3540
4509
|
]);
|
|
@@ -3586,6 +4555,18 @@ const openLab = async (target, kind = "tcp", topology) => {
|
|
|
3586
4555
|
if (target.close === void 0) throw new Error(`${target.image} names no way to close its clients`);
|
|
3587
4556
|
await output(server, [...target.close]);
|
|
3588
4557
|
},
|
|
4558
|
+
async exec(argv) {
|
|
4559
|
+
return output(server, [...argv]);
|
|
4560
|
+
},
|
|
4561
|
+
async sibling(siblingImage, argv, ready) {
|
|
4562
|
+
const started = await new GenericContainer(siblingImage).withNetwork(last).withWaitStrategy(Wait.forLogMessage(ready)).withCommand([...argv]).start();
|
|
4563
|
+
opened.push(started);
|
|
4564
|
+
return {
|
|
4565
|
+
ip: await ipOn(started, last),
|
|
4566
|
+
netns: await netnsOf(started),
|
|
4567
|
+
exec: (command) => output(started, [...command])
|
|
4568
|
+
};
|
|
4569
|
+
},
|
|
3589
4570
|
async agent(agentImage, argv, options = {}) {
|
|
3590
4571
|
const Shared = shared(GenericContainer);
|
|
3591
4572
|
const seen = lines();
|
|
@@ -3726,7 +4707,7 @@ const openLab = async (target, kind = "tcp", topology) => {
|
|
|
3726
4707
|
};
|
|
3727
4708
|
while (!confirmed()) {
|
|
3728
4709
|
if (Date.now() - started > CONFIRM_MS) throw new Error(`the hop ${event.hop} did not confirm the write of ${event.write.path}: no line "${confirm}" in ${CONFIRM_MS / 1e3} s`);
|
|
3729
|
-
await sleep$
|
|
4710
|
+
await sleep$6(50);
|
|
3730
4711
|
}
|
|
3731
4712
|
}
|
|
3732
4713
|
return stamp;
|
|
@@ -3743,9 +4724,10 @@ const openLab = async (target, kind = "tcp", topology) => {
|
|
|
3743
4724
|
query
|
|
3744
4725
|
]);
|
|
3745
4726
|
},
|
|
3746
|
-
async probe(args) {
|
|
4727
|
+
async probe(args, files = []) {
|
|
3747
4728
|
const events = lines();
|
|
3748
|
-
const
|
|
4729
|
+
const prober = new GenericContainer(clientImage).withNetworkMode(`container:${client.getId()}`).withCommand([...command, ...args]).withCopyContentToContainer(files.map((file) => ({ ...file }))).withLogConsumer(events.consume);
|
|
4730
|
+
const container = await (capabilities === void 0 ? prober : prober.withAddedCapabilities(...capabilities)).start();
|
|
3749
4731
|
return {
|
|
3750
4732
|
events: events.seen,
|
|
3751
4733
|
until: (kinds, within) => events.until(kinds, within, "client"),
|
|
@@ -3846,6 +4828,7 @@ const openLab = async (target, kind = "tcp", topology) => {
|
|
|
3846
4828
|
client: {
|
|
3847
4829
|
kind: clientKind,
|
|
3848
4830
|
image: clientImage,
|
|
4831
|
+
...capabilities === void 0 ? {} : { capabilities },
|
|
3849
4832
|
...source === void 0 ? {} : {
|
|
3850
4833
|
source,
|
|
3851
4834
|
...buildOf(source)
|
|
@@ -3864,33 +4847,240 @@ const openLab = async (target, kind = "tcp", topology) => {
|
|
|
3864
4847
|
}
|
|
3865
4848
|
};
|
|
3866
4849
|
//#endregion
|
|
3867
|
-
//#region src/
|
|
3868
|
-
/**
|
|
3869
|
-
* This module tells if a manifest pins each program that a run operated. A record that does not pin
|
|
3870
|
-
* them is not a record. A later run cannot operate the same programs, thus nobody can reproduce
|
|
3871
|
-
* its numbers (SYS-101). Each image is pulled by its digest, or the harness builds it. A build
|
|
3872
|
-
* records its bases by digest, the sha256 of each lockfile, and the versions of the packages that
|
|
3873
|
-
* its manifests name. The commit of the code that operated is a field of the record of a cell, and
|
|
3874
|
-
* `commitProblems` examines it.
|
|
3875
|
-
*/
|
|
3876
|
-
/** This pattern matches the id of an image: `sha256:` and 64 hex digits. */
|
|
3877
|
-
const IMAGE_ID = /^sha256:[0-9a-f]{64}$/;
|
|
3878
|
-
/** This pattern matches the tag of a build of the harness: its name, then 12 hex digits of the sha256 of its context. */
|
|
3879
|
-
const BUILD_TAG = /^[a-z0-9][a-z0-9._\-/]*:[0-9a-f]{12}$/;
|
|
3880
|
-
/** This pattern matches a commit of git: 40 hex digits. */
|
|
3881
|
-
const COMMIT = /^[0-9a-f]{40}$/;
|
|
3882
|
-
/** This function gives each problem of the record of a build. A problem is a base that no digest pins, or a tag of another form. */
|
|
3883
|
-
const buildProblems = (what, build) => [...BUILD_TAG.test(build.tag) ? [] : [`${what}: the build has the tag ${build.tag}, which is not a tag of a build`], ...build.bases.filter((base) => !PINNED.test(base)).map((base) => `${what}: the base ${base} is not pinned by digest`)];
|
|
4850
|
+
//#region src/standby.ts
|
|
3884
4851
|
/**
|
|
3885
|
-
* This
|
|
3886
|
-
*
|
|
3887
|
-
*
|
|
4852
|
+
* This module is the second server of a pair (`Cell.pair`). The lab of the cell has the primary, the
|
|
4853
|
+
* topology in front of it, the injector and the prober. This module adds one standby on the network
|
|
4854
|
+
* of the primary, and it does one run:
|
|
4855
|
+
*
|
|
4856
|
+
* 1. The client writes a stream of sequence numbers through the topology.
|
|
4857
|
+
* 2. The fault cuts the primary from the standby, or from each peer. The cut stays to the end.
|
|
4858
|
+
* 3. At the end of the hold, the injector stops each process of the primary with SIGKILL.
|
|
4859
|
+
* 4. The harness promotes the standby, and reads what it holds.
|
|
4860
|
+
* 5. The harness starts the old primary, and reads what its crash recovery gives.
|
|
4861
|
+
*
|
|
4862
|
+
* The standby has no way to the WAL of the old primary, because the primary stops under the cut. A
|
|
4863
|
+
* promotion cannot be undone, thus each run has its own environment (`Cell.fresh`).
|
|
3888
4864
|
*/
|
|
3889
|
-
|
|
3890
|
-
|
|
3891
|
-
|
|
3892
|
-
|
|
3893
|
-
|
|
4865
|
+
/** This table holds one row for each write of the client. */
|
|
4866
|
+
const TABLE = "writes";
|
|
4867
|
+
/** This time bounds each wait of the setup of the pair, in ms. */
|
|
4868
|
+
const SETUP_MS = 6e4;
|
|
4869
|
+
/** This count is the acknowledged writes before the fault: the stream is steady. */
|
|
4870
|
+
const STEADY$4 = 20;
|
|
4871
|
+
/** This time is the wait after the stop of the primary, in ms: each writer sends its last write in it. */
|
|
4872
|
+
const LAST_WRITE_MS = 2e3;
|
|
4873
|
+
/** This time bounds the wait for the last line of the client, in ms. */
|
|
4874
|
+
const STOP_MS = 5e3;
|
|
4875
|
+
/** This text is the line of the log of a standby that serves reads (src/backend/postmaster/postmaster.c:3740). */
|
|
4876
|
+
const STANDBY_READY = "database system is ready to accept read-only connections";
|
|
4877
|
+
/** This text is the write of the client, with its sequence number. */
|
|
4878
|
+
const INSERT = `INSERT INTO ${TABLE} VALUES ($n)`;
|
|
4879
|
+
const sleep$5 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
4880
|
+
/** This function gives the argv of one statement in psql, on the server in the same container. */
|
|
4881
|
+
const psql = (statement) => [
|
|
4882
|
+
"psql",
|
|
4883
|
+
"-h",
|
|
4884
|
+
"127.0.0.1",
|
|
4885
|
+
"-U",
|
|
4886
|
+
"postgres",
|
|
4887
|
+
"-At",
|
|
4888
|
+
"-c",
|
|
4889
|
+
statement
|
|
4890
|
+
];
|
|
4891
|
+
/** This function gives the command of a standby: a base backup of the primary, then the server in recovery. */
|
|
4892
|
+
const standbyCommand = (primary) => [
|
|
4893
|
+
"sh",
|
|
4894
|
+
"-c",
|
|
4895
|
+
`chown postgres:postgres "$PGDATA"; chmod 0700 "$PGDATA"; until gosu postgres pg_basebackup -h ${primary} -U postgres -D "$PGDATA" -X stream -c fast -R; do gosu postgres sh -c 'rm -rf "$PGDATA"/*'; sleep 0.5; done; exec gosu postgres postgres`
|
|
4896
|
+
];
|
|
4897
|
+
/** This command starts the old primary again, after the stop of each of its processes. Its crash recovery runs before the wait ends. */
|
|
4898
|
+
const RESTART = [
|
|
4899
|
+
"sh",
|
|
4900
|
+
"-c",
|
|
4901
|
+
"exec gosu postgres pg_ctl -D \"$PGDATA\" -l /tmp/restart.log -w -t 60 start"
|
|
4902
|
+
];
|
|
4903
|
+
/** This function reads the output of a query for sequence numbers into a sorted list. */
|
|
4904
|
+
const numbersOf = (text) => text.split("\n").map((line) => line.trim()).filter((line) => line !== "").map(Number).sort((a, b) => a - b);
|
|
4905
|
+
/** This function gives the arguments of the client of a run. */
|
|
4906
|
+
const pairArgs = (cell, lab) => {
|
|
4907
|
+
const pair = cell.pair;
|
|
4908
|
+
const hold = cell.hold === void 0 ? 0 : valueIn(cell.hold, "s");
|
|
4909
|
+
return [
|
|
4910
|
+
lab.targetIp,
|
|
4911
|
+
String(lab.targetPort),
|
|
4912
|
+
"--mode",
|
|
4913
|
+
"stream",
|
|
4914
|
+
"--interval-ms",
|
|
4915
|
+
String(valueIn(pair.interval, "ms")),
|
|
4916
|
+
"--writers",
|
|
4917
|
+
String(pair.writers),
|
|
4918
|
+
"--query",
|
|
4919
|
+
INSERT,
|
|
4920
|
+
"--max-s",
|
|
4921
|
+
String(Math.ceil(hold) + 60),
|
|
4922
|
+
...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
|
|
4923
|
+
];
|
|
4924
|
+
};
|
|
4925
|
+
/** This function gives the peers of the cut of a pair: the standby, or the standby and the peer of the primary in the topology. */
|
|
4926
|
+
const peersOf$1 = (pair, standby, lab) => pair.cuts === "standby" ? [standby] : [standby, lab.topology?.targetPeer ?? lab.clientIp];
|
|
4927
|
+
/** This function gives the instants of the cancels of a hold, in ns from the fault: equal parts of the hold. */
|
|
4928
|
+
const cancelsOf = (pair, holdNs) => {
|
|
4929
|
+
const count = pair.cancels ?? 0;
|
|
4930
|
+
return Array.from({ length: count }, (_, at) => (at + 1) * holdNs / (count + 1));
|
|
4931
|
+
};
|
|
4932
|
+
/** This function asks until the primary reports the standby in a state of replication. */
|
|
4933
|
+
const untilStreaming = async (lab, state) => {
|
|
4934
|
+
const started = Date.now();
|
|
4935
|
+
for (;;) {
|
|
4936
|
+
const seen = await lab.exec(psql("SELECT sync_state FROM pg_stat_replication WHERE state = 'streaming'"));
|
|
4937
|
+
if (seen === state) return;
|
|
4938
|
+
if (Date.now() - started > SETUP_MS) throw new Error(`the standby did not stream as ${state} in ${SETUP_MS} ms: it reports "${seen}"`);
|
|
4939
|
+
await sleep$5(100);
|
|
4940
|
+
}
|
|
4941
|
+
};
|
|
4942
|
+
/**
|
|
4943
|
+
* This function opens the pair of an environment. The table comes first and the standby next: the
|
|
4944
|
+
* base backup has it. A synchronous standby is named last, thus no setup step waits for it.
|
|
4945
|
+
*/
|
|
4946
|
+
const openStandby = async (lab, cell) => {
|
|
4947
|
+
const pair = cell.pair;
|
|
4948
|
+
await lab.exec([
|
|
4949
|
+
"sh",
|
|
4950
|
+
"-c",
|
|
4951
|
+
"echo \"host replication all all trust\" >> \"$PGDATA/pg_hba.conf\""
|
|
4952
|
+
]);
|
|
4953
|
+
await lab.exec(psql("SELECT pg_reload_conf()"));
|
|
4954
|
+
await lab.exec(psql(`CREATE TABLE ${TABLE} (n integer PRIMARY KEY)`));
|
|
4955
|
+
const standby = await lab.sibling(await targetImageOf(cell.target), standbyCommand(lab.serverIp), STANDBY_READY);
|
|
4956
|
+
await untilStreaming(lab, "async");
|
|
4957
|
+
if (pair.synchronous) {
|
|
4958
|
+
await lab.exec(psql("ALTER SYSTEM SET synchronous_standby_names = '*'"));
|
|
4959
|
+
await lab.exec(psql("SELECT pg_reload_conf()"));
|
|
4960
|
+
await untilStreaming(lab, "sync");
|
|
4961
|
+
}
|
|
4962
|
+
return standby;
|
|
4963
|
+
};
|
|
4964
|
+
/** This function waits for a count of acknowledged writes. */
|
|
4965
|
+
const untilSteady = async (probe) => {
|
|
4966
|
+
const started = Date.now();
|
|
4967
|
+
while (probe.events.filter((event) => event.ev === "ack").length < STEADY$4) {
|
|
4968
|
+
if (Date.now() - started > SETUP_MS) throw new Error(`the client did not write ${STEADY$4} times in ${SETUP_MS} ms`);
|
|
4969
|
+
await sleep$5(10);
|
|
4970
|
+
}
|
|
4971
|
+
};
|
|
4972
|
+
/** This function gives the count of lost records in the latest clock line of the prober. */
|
|
4973
|
+
const droppedSoFar$5 = (records) => {
|
|
4974
|
+
for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
|
|
4975
|
+
return 0;
|
|
4976
|
+
};
|
|
4977
|
+
/**
|
|
4978
|
+
* This function does one run of a pair. A dry run writes and stops, with no fault: the warm-up of the
|
|
4979
|
+
* environment. It removes its rows, thus the numbers of the run start on an empty table.
|
|
4980
|
+
*/
|
|
4981
|
+
const pairOnce = async (lab, standby, cell, environment, index, dry = false) => {
|
|
4982
|
+
const pair = cell.pair;
|
|
4983
|
+
await lab.forget();
|
|
4984
|
+
const from = lab.kernel.length;
|
|
4985
|
+
const lostBefore = droppedSoFar$5(lab.kernel);
|
|
4986
|
+
const probe = await lab.probe(pairArgs(cell, lab));
|
|
4987
|
+
const holdNs = cell.hold === void 0 ? 0 : valueIn(cell.hold, "s") * 1e9;
|
|
4988
|
+
let fault = null;
|
|
4989
|
+
let heal = null;
|
|
4990
|
+
try {
|
|
4991
|
+
await probe.until(["ready"], 15e3);
|
|
4992
|
+
await untilSteady(probe);
|
|
4993
|
+
if (!dry) {
|
|
4994
|
+
fault = await lab.fault("blackhole", "target", { peers: peersOf$1(pair, standby.ip, lab) });
|
|
4995
|
+
const started = Date.now();
|
|
4996
|
+
for (const at of cancelsOf(pair, holdNs)) {
|
|
4997
|
+
await sleep$5(Math.max(0, at / 1e6 - (Date.now() - started)));
|
|
4998
|
+
await lab.exec(psql("SELECT count(pg_cancel_backend(pid)) FROM pg_stat_activity WHERE wait_event = 'SyncRep'"));
|
|
4999
|
+
}
|
|
5000
|
+
heal = await lab.signal("KILL", fault.t0 + holdNs);
|
|
5001
|
+
await sleep$5(LAST_WRITE_MS);
|
|
5002
|
+
if (cell.target.kill !== void 0) await lab.exec(cell.target.kill);
|
|
5003
|
+
}
|
|
5004
|
+
await probe.go().catch(() => void 0);
|
|
5005
|
+
await probe.until(["done"], STOP_MS);
|
|
5006
|
+
} finally {
|
|
5007
|
+
await probe.stop();
|
|
5008
|
+
}
|
|
5009
|
+
if (dry) await lab.exec(psql(`TRUNCATE ${TABLE}`));
|
|
5010
|
+
const settled = await lab.settle();
|
|
5011
|
+
const lost = settled === null ? null : settled - lostBefore;
|
|
5012
|
+
const read = `SELECT n FROM ${TABLE} ORDER BY n`;
|
|
5013
|
+
let holds = null;
|
|
5014
|
+
let oldHolds = [];
|
|
5015
|
+
if (!dry) {
|
|
5016
|
+
await standby.exec(psql("SELECT pg_promote(true, 60)"));
|
|
5017
|
+
holds = numbersOf(await standby.exec(psql(read)));
|
|
5018
|
+
await lab.exec([...RESTART]);
|
|
5019
|
+
oldHolds = numbersOf(await lab.exec(psql(read)));
|
|
5020
|
+
}
|
|
5021
|
+
return {
|
|
5022
|
+
environment,
|
|
5023
|
+
index,
|
|
5024
|
+
events: [...probe.events],
|
|
5025
|
+
fault,
|
|
5026
|
+
...heal === null ? {} : { heal },
|
|
5027
|
+
kernel: lab.kernel.slice(from),
|
|
5028
|
+
netns: lab.netns.client,
|
|
5029
|
+
targetNetns: lab.netns.target,
|
|
5030
|
+
lost,
|
|
5031
|
+
...lab.topology === null ? {} : {
|
|
5032
|
+
topology: lab.topology,
|
|
5033
|
+
middlebox: []
|
|
5034
|
+
},
|
|
5035
|
+
group: {
|
|
5036
|
+
members: {
|
|
5037
|
+
primary: {
|
|
5038
|
+
ip: lab.serverIp,
|
|
5039
|
+
netns: lab.netns.target
|
|
5040
|
+
},
|
|
5041
|
+
standby: {
|
|
5042
|
+
ip: standby.ip,
|
|
5043
|
+
netns: standby.netns
|
|
5044
|
+
}
|
|
5045
|
+
},
|
|
5046
|
+
sentinels: {},
|
|
5047
|
+
primary: "primary",
|
|
5048
|
+
at: "primary",
|
|
5049
|
+
timeline: [],
|
|
5050
|
+
after: dry ? "primary" : "standby",
|
|
5051
|
+
holds,
|
|
5052
|
+
oldHolds
|
|
5053
|
+
}
|
|
5054
|
+
};
|
|
5055
|
+
};
|
|
5056
|
+
//#endregion
|
|
5057
|
+
//#region src/pins.ts
|
|
5058
|
+
/**
|
|
5059
|
+
* This module tells if a manifest pins each program that a run operated. A record that does not pin
|
|
5060
|
+
* them is not a record. A later run cannot operate the same programs, thus nobody can reproduce
|
|
5061
|
+
* its numbers (SYS-101). Each image is pulled by its digest, or the harness builds it. A build
|
|
5062
|
+
* records its bases by digest, the sha256 of each lockfile, and the versions of the packages that
|
|
5063
|
+
* its manifests name. The commit of the code that operated is a field of the record of a cell, and
|
|
5064
|
+
* `commitProblems` examines it.
|
|
5065
|
+
*/
|
|
5066
|
+
/** This pattern matches the id of an image: `sha256:` and 64 hex digits. */
|
|
5067
|
+
const IMAGE_ID = /^sha256:[0-9a-f]{64}$/;
|
|
5068
|
+
/** This pattern matches the tag of a build of the harness: its name, then 12 hex digits of the sha256 of its context. */
|
|
5069
|
+
const BUILD_TAG = /^[a-z0-9][a-z0-9._\-/]*:[0-9a-f]{12}$/;
|
|
5070
|
+
/** This pattern matches a commit of git: 40 hex digits. */
|
|
5071
|
+
const COMMIT = /^[0-9a-f]{40}$/;
|
|
5072
|
+
/** This function gives each problem of the record of a build. A problem is a base that no digest pins, or a tag of another form. */
|
|
5073
|
+
const buildProblems = (what, build) => [...BUILD_TAG.test(build.tag) ? [] : [`${what}: the build has the tag ${build.tag}, which is not a tag of a build`], ...build.bases.filter((base) => !PINNED.test(base)).map((base) => `${what}: the base ${base} is not pinned by digest`)];
|
|
5074
|
+
/**
|
|
5075
|
+
* This function gives each problem of an image of a manifest. A pulled image needs a digest. An
|
|
5076
|
+
* image that the harness builds needs its build record when the manifest can hold one (`build`).
|
|
5077
|
+
* Otherwise, it needs the tag of a build and the id of its image.
|
|
5078
|
+
*/
|
|
5079
|
+
const pinProblems = (what, image, id, build, needsBuild) => {
|
|
5080
|
+
if (PINNED.test(image)) return [];
|
|
5081
|
+
if (build !== void 0) return buildProblems(what, build);
|
|
5082
|
+
if (BUILD_TAG.test(image) && !needsBuild && id !== void 0 && IMAGE_ID.test(id)) return [];
|
|
5083
|
+
return BUILD_TAG.test(image) && needsBuild ? [`${what}: the image ${image} is a build with no build record`] : [`${what}: the image ${image} has no digest`];
|
|
3894
5084
|
};
|
|
3895
5085
|
/**
|
|
3896
5086
|
* This function gives each problem of a manifest that stops it from pinning the programs of its
|
|
@@ -3911,6 +5101,21 @@ const manifestProblems = (manifest) => [
|
|
|
3911
5101
|
];
|
|
3912
5102
|
/** This function gives the problem of the commit of a record: no commit, or a text that is not a commit of git. */
|
|
3913
5103
|
const commitProblems = (commit) => commit === void 0 ? ["the record has no commit of the code that operated"] : COMMIT.test(commit) ? [] : [`the record has the commit ${commit}, which is not 40 hex digits`];
|
|
5104
|
+
/** This problem is the problem of a checkout with changes: its commit does not hold the code that operated. */
|
|
5105
|
+
const DIRTY = "the checkout has changes to tracked files that no commit holds";
|
|
5106
|
+
/** This problem is the problem of a record of a local debug run. */
|
|
5107
|
+
const DEBUG = "the record is of a local debug run, and no page reads it";
|
|
5108
|
+
/**
|
|
5109
|
+
* This function gives each problem that stops a record from pinning the programs of its runs. A
|
|
5110
|
+
* record with a problem is not a record (SYS-101). The writer of the cells refuses it, and the read
|
|
5111
|
+
* of a schedule refuses it. A commit with changes to tracked files is not a pin.
|
|
5112
|
+
*/
|
|
5113
|
+
const recordProblems = (record) => [.../* @__PURE__ */ new Set([
|
|
5114
|
+
...commitProblems(record.code?.commit),
|
|
5115
|
+
...record.code?.dirty === true ? [DIRTY] : [],
|
|
5116
|
+
...record.manifests.flatMap((manifest) => manifestProblems(manifest)),
|
|
5117
|
+
...record.debug === true ? [DEBUG] : []
|
|
5118
|
+
])];
|
|
3914
5119
|
//#endregion
|
|
3915
5120
|
//#region src/group.ts
|
|
3916
5121
|
/** This function gives the configuration file of a member: its port, its settings, and the primary that a replica follows. */
|
|
@@ -3929,19 +5134,19 @@ const sentinelConfig = (group, primary) => [
|
|
|
3929
5134
|
].join("\n");
|
|
3930
5135
|
/**
|
|
3931
5136
|
* This function gives the peers that a fault on a part cuts it from, by their addresses. `all` is
|
|
3932
|
-
* each other part and the client, `group` is the other parts, and `clients`
|
|
3933
|
-
* and `one-way` are the first other member. The observer is not a peer, thus the timeline of the
|
|
5137
|
+
* each other part and the client, `group` is the other parts, and `clients` and `to-clients` are
|
|
5138
|
+
* the client. `one` and `one-way` are the first other member. The observer is not a peer, thus the timeline of the
|
|
3934
5139
|
* group stays whole.
|
|
3935
5140
|
*/
|
|
3936
5141
|
const peersOf = (parts, client, part, cuts) => {
|
|
3937
5142
|
const others = [...Object.entries(parts.members), ...Object.entries(parts.sentinels)].filter(([name]) => name !== part).map(([, at]) => at.ip);
|
|
3938
|
-
if (cuts === "clients") return [client];
|
|
5143
|
+
if (cuts === "clients" || cuts === "to-clients") return [client];
|
|
3939
5144
|
if (cuts === "group") return others;
|
|
3940
5145
|
if (cuts === "one" || cuts === "one-way") return Object.entries(parts.members).filter(([name]) => name !== part).slice(0, 1).map(([, at]) => at.ip);
|
|
3941
5146
|
return [...others, client];
|
|
3942
5147
|
};
|
|
3943
|
-
/** This function gives the direction of the rules of a cut: `one-way`
|
|
3944
|
-
const directionOf = (cuts) => cuts === "one-way" ? ["--direction", "out"] : [];
|
|
5148
|
+
/** This function gives the direction of the rules of a cut: `one-way` and `to-clients` drop only the segments from the part. */
|
|
5149
|
+
const directionOf = (cuts) => cuts === "one-way" || cuts === "to-clients" ? ["--direction", "out"] : [];
|
|
3945
5150
|
/** This function reads the flat reply of a command line client into its fields: name, value, name, value. */
|
|
3946
5151
|
const fieldsOf = (text) => {
|
|
3947
5152
|
const out = [];
|
|
@@ -4034,7 +5239,7 @@ const viewOf = (group, ips, sentinels, members) => {
|
|
|
4034
5239
|
why: "healthy"
|
|
4035
5240
|
};
|
|
4036
5241
|
};
|
|
4037
|
-
const sleep$
|
|
5242
|
+
const sleep$4 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
4038
5243
|
/** This time is the longest that a group takes to converge before a run, or after it. */
|
|
4039
5244
|
const CONVERGE_MS = 24e4;
|
|
4040
5245
|
/**
|
|
@@ -4229,7 +5434,7 @@ const openGroupLab = async (group, options = {}) => {
|
|
|
4229
5434
|
}));
|
|
4230
5435
|
calm = last.healthy ? calm + 1 : 0;
|
|
4231
5436
|
if (calm >= 2 && last.primary !== null) return last.primary;
|
|
4232
|
-
await sleep$
|
|
5437
|
+
await sleep$4(1e3);
|
|
4233
5438
|
}
|
|
4234
5439
|
throw new Error(`the group did not converge in ${within} ms: ${last.why}`);
|
|
4235
5440
|
},
|
|
@@ -4250,11 +5455,16 @@ const openGroupLab = async (group, options = {}) => {
|
|
|
4250
5455
|
]);
|
|
4251
5456
|
},
|
|
4252
5457
|
async holds(primary) {
|
|
4253
|
-
return (await cli(primary, [
|
|
5458
|
+
return holdsOf(await cli(primary, [
|
|
4254
5459
|
"--scan",
|
|
4255
5460
|
"--pattern",
|
|
4256
5461
|
"w:*"
|
|
4257
|
-
])
|
|
5462
|
+
]), await cli(primary, [
|
|
5463
|
+
"LRANGE",
|
|
5464
|
+
"w",
|
|
5465
|
+
"0",
|
|
5466
|
+
"-1"
|
|
5467
|
+
]));
|
|
4258
5468
|
},
|
|
4259
5469
|
partOf: roles,
|
|
4260
5470
|
async client(args) {
|
|
@@ -4309,7 +5519,7 @@ const openGroupLab = async (group, options = {}) => {
|
|
|
4309
5519
|
const started = Date.now();
|
|
4310
5520
|
while (await cli(member, ["PING"]).catch(() => "") !== "PONG") {
|
|
4311
5521
|
if (Date.now() - started > 3e4) throw new Error(`${member} did not start again`);
|
|
4312
|
-
await sleep$
|
|
5522
|
+
await sleep$4(200);
|
|
4313
5523
|
}
|
|
4314
5524
|
},
|
|
4315
5525
|
failover(coordinated) {
|
|
@@ -4425,7 +5635,7 @@ const openGroupLab = async (group, options = {}) => {
|
|
|
4425
5635
|
}
|
|
4426
5636
|
};
|
|
4427
5637
|
/** This count is the number of accepted writes that a run waits for before its fault: the stream is steady then. */
|
|
4428
|
-
const STEADY$
|
|
5638
|
+
const STEADY$3 = 20;
|
|
4429
5639
|
/** This time is how long a run continues after its outcome, so that the record shows what came next. */
|
|
4430
5640
|
const TAIL_MS$3 = 2e3;
|
|
4431
5641
|
/** This function gives the address of a member, with the port of the group. */
|
|
@@ -4435,22 +5645,32 @@ const deadlineOf = (cell) => {
|
|
|
4435
5645
|
const { prediction } = cell;
|
|
4436
5646
|
return (prediction.detects === true ? valueIn(prediction.high, "s") : prediction.detects === false && prediction.recovers !== void 0 ? valueIn(prediction.recovers.high, "s") : valueIn(prediction.window, "s")) * 1e3 + 1e4;
|
|
4437
5647
|
};
|
|
5648
|
+
/**
|
|
5649
|
+
* This function gives the sequence numbers that a group holds, in order, from the output of its
|
|
5650
|
+
* command line. They are the keys `w:<n>` of a stream of SET, and the entries of the list `w` of a
|
|
5651
|
+
* stream of RPUSH. A list keeps each copy, thus a number can repeat.
|
|
5652
|
+
*/
|
|
5653
|
+
const holdsOf = (keys, list) => [...keys.split("\n").map((key) => Number(key.trim().slice(2))), ...list.split("\n").map((entry) => Number(entry.trim()))].filter((n) => Number.isInteger(n) && n > 0).sort((a, b) => a - b);
|
|
4438
5654
|
/** This function gives the arguments of the client of a group cell. */
|
|
4439
5655
|
const clientArgs = (cell, lab) => {
|
|
5656
|
+
const sentinels = cell.group.sentinels.map((name) => `${lab.sentinels[name].ip}:${cell.group.sentinelPort}`);
|
|
5657
|
+
const driver = cell.probe.driver?.name === "ioredis" ? cell.probe.driver : void 0;
|
|
4440
5658
|
return [
|
|
4441
5659
|
"--sentinels",
|
|
4442
|
-
|
|
5660
|
+
sentinels.join(","),
|
|
4443
5661
|
"--name",
|
|
4444
5662
|
cell.group.name,
|
|
4445
5663
|
"--interval-ms",
|
|
4446
5664
|
String(cell.probe.interval === void 0 ? 50 : valueIn(cell.probe.interval, "ms")),
|
|
4447
5665
|
"--max-s",
|
|
4448
5666
|
String(Math.ceil(deadlineOf(cell) / 1e3 + 120)),
|
|
4449
|
-
...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
|
|
5667
|
+
...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)],
|
|
5668
|
+
...driver?.op === void 0 ? [] : ["--op", driver.op],
|
|
5669
|
+
...driver?.window === void 0 ? [] : ["--window", String(driver.window)]
|
|
4450
5670
|
];
|
|
4451
5671
|
};
|
|
4452
5672
|
/** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
|
|
4453
|
-
const waitFor$
|
|
5673
|
+
const waitFor$3 = (client, match, within) => new Promise((resolve) => {
|
|
4454
5674
|
const started = Date.now();
|
|
4455
5675
|
const look = () => {
|
|
4456
5676
|
const found = client.events.find(match);
|
|
@@ -4466,7 +5686,7 @@ const subscriptionsOf = (timeline) => {
|
|
|
4466
5686
|
return [...latest.values()];
|
|
4467
5687
|
};
|
|
4468
5688
|
/** This function gives the count of lost records in the latest clock line of the prober. */
|
|
4469
|
-
const droppedSoFar$
|
|
5689
|
+
const droppedSoFar$4 = (records) => {
|
|
4470
5690
|
for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
|
|
4471
5691
|
return 0;
|
|
4472
5692
|
};
|
|
@@ -4488,7 +5708,7 @@ const groupOnce = async (lab, cell, environment, index) => {
|
|
|
4488
5708
|
await lab.forget();
|
|
4489
5709
|
const from = lab.kernel.length;
|
|
4490
5710
|
const timelineFrom = lab.timeline.length;
|
|
4491
|
-
const lostBefore = droppedSoFar$
|
|
5711
|
+
const lostBefore = droppedSoFar$4(lab.kernel);
|
|
4492
5712
|
const client = await lab.client(clientArgs(cell, lab));
|
|
4493
5713
|
const lands = cell.prediction.group?.lands ?? "new";
|
|
4494
5714
|
const place = cell.faultAt ?? { role: "primary" };
|
|
@@ -4498,7 +5718,7 @@ const groupOnce = async (lab, cell, environment, index) => {
|
|
|
4498
5718
|
const undo = [];
|
|
4499
5719
|
try {
|
|
4500
5720
|
await client.until(["ready"], 6e4);
|
|
4501
|
-
if (await waitFor$
|
|
5721
|
+
if (await waitFor$3(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$3, 3e4) === void 0) throw new Error(`the client did not write ${STEADY$3} times in 30 s`);
|
|
4502
5722
|
let healing = null;
|
|
4503
5723
|
switch (cell.fault) {
|
|
4504
5724
|
case "blackhole":
|
|
@@ -4512,7 +5732,7 @@ const groupOnce = async (lab, cell, environment, index) => {
|
|
|
4512
5732
|
undo.push(() => lab.lift(kind, part, cuts));
|
|
4513
5733
|
if (cell.stall !== void 0) {
|
|
4514
5734
|
const t1 = fault.t1;
|
|
4515
|
-
const sent = await waitFor$
|
|
5735
|
+
const sent = await waitFor$3(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
|
|
4516
5736
|
if (sent !== void 0) healing = lab.lift(kind, part, cuts, sent.t + valueIn(cell.stall, "s") * 1e9);
|
|
4517
5737
|
} else if (cell.hold !== void 0) healing = lab.lift(kind, part, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
|
|
4518
5738
|
break;
|
|
@@ -4546,11 +5766,11 @@ const groupOnce = async (lab, cell, environment, index) => {
|
|
|
4546
5766
|
const t0 = fault.t0;
|
|
4547
5767
|
const old = addressOf(lab, primary);
|
|
4548
5768
|
const deadline = deadlineOf(cell);
|
|
4549
|
-
if ((lands === "new" ? await waitFor$
|
|
5769
|
+
if ((lands === "new" ? await waitFor$3(client, (event) => event.ev === "ack" && event.t > t0 && typeof event["at"] === "string" && event["at"] !== old, deadline) : await (async () => {
|
|
4550
5770
|
await healed;
|
|
4551
5771
|
const after = heal?.t0 ?? t0;
|
|
4552
|
-
return waitFor$
|
|
4553
|
-
})()) !== void 0) await sleep$
|
|
5772
|
+
return waitFor$3(client, (event) => event.ev === "ack" && event.t > after, deadline);
|
|
5773
|
+
})()) !== void 0) await sleep$4(TAIL_MS$3);
|
|
4554
5774
|
await healed;
|
|
4555
5775
|
} finally {
|
|
4556
5776
|
for (const step of undo.reverse()) await step().catch(() => void 0);
|
|
@@ -4643,6 +5863,10 @@ const cutOf = (set, members, client, part, cuts) => {
|
|
|
4643
5863
|
peers: others.slice(0, 1),
|
|
4644
5864
|
direction: "out"
|
|
4645
5865
|
};
|
|
5866
|
+
if (cuts === "to-clients") return {
|
|
5867
|
+
peers: [client],
|
|
5868
|
+
direction: "out"
|
|
5869
|
+
};
|
|
4646
5870
|
return { peers: [...others, client] };
|
|
4647
5871
|
};
|
|
4648
5872
|
/** This function gives the command of the server of each member. The server is the child of a shell, thus a fin can stop it and the container stays up. */
|
|
@@ -4714,7 +5938,7 @@ const statusViewOf = (set, ips, status) => {
|
|
|
4714
5938
|
};
|
|
4715
5939
|
};
|
|
4716
5940
|
/** This function waits for a time, in ms. */
|
|
4717
|
-
const sleep$
|
|
5941
|
+
const sleep$3 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
4718
5942
|
/** These are the arguments of replSetStepDown: the time that the old primary cannot stand again, and the wait for a secondary to catch up. */
|
|
4719
5943
|
const STEP_DOWN = {
|
|
4720
5944
|
seconds: 60,
|
|
@@ -4862,7 +6086,7 @@ const openReplicaSetLab = async (set, backups = false) => {
|
|
|
4862
6086
|
calm = last.healthy ? calm + 1 : 0;
|
|
4863
6087
|
if (last.primary !== null) known = last.primary;
|
|
4864
6088
|
if (calm >= 2 && last.primary !== null) return last.primary;
|
|
4865
|
-
await sleep$
|
|
6089
|
+
await sleep$3(1e3);
|
|
4866
6090
|
}
|
|
4867
6091
|
throw new Error(`the set did not converge in ${within} ms: ${last.why}`);
|
|
4868
6092
|
};
|
|
@@ -4968,7 +6192,7 @@ const openReplicaSetLab = async (set, backups = false) => {
|
|
|
4968
6192
|
const started = Date.now();
|
|
4969
6193
|
while (!(await shell(member, "print(db.hello().ok)").catch(() => "")).includes("1")) {
|
|
4970
6194
|
if (Date.now() - started > 6e4) throw new Error(`${member} did not start again`);
|
|
4971
|
-
await sleep$
|
|
6195
|
+
await sleep$3(500);
|
|
4972
6196
|
}
|
|
4973
6197
|
},
|
|
4974
6198
|
exec(member, argv) {
|
|
@@ -5071,6 +6295,8 @@ const openReplicaSetLab = async (set, backups = false) => {
|
|
|
5071
6295
|
};
|
|
5072
6296
|
/** This time is how long a run continues after its outcome, so that the record shows what came next. */
|
|
5073
6297
|
const TAIL_MS$2 = 2e3;
|
|
6298
|
+
/** This function gives how long a run of a cell continues after its lift, in ms: the tail, and the watch of the cell. */
|
|
6299
|
+
const afterLiftOf = (cell) => TAIL_MS$2 + (cell.watch === void 0 ? 0 : valueIn(cell.watch, "s") * 1e3);
|
|
5074
6300
|
/** This function gives the arguments of the client of a replica set cell: each member, and the stream of writes. */
|
|
5075
6301
|
const replicaSetArgs = (cell, members) => {
|
|
5076
6302
|
const set = cell.replicaSet;
|
|
@@ -5094,7 +6320,7 @@ const replicaSetArgs = (cell, members) => {
|
|
|
5094
6320
|
];
|
|
5095
6321
|
};
|
|
5096
6322
|
/** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
|
|
5097
|
-
const waitFor$
|
|
6323
|
+
const waitFor$2 = (client, match, within) => new Promise((resolve) => {
|
|
5098
6324
|
const started = Date.now();
|
|
5099
6325
|
const look = () => {
|
|
5100
6326
|
const found = client.events.find(match);
|
|
@@ -5104,7 +6330,7 @@ const waitFor$1 = (client, match, within) => new Promise((resolve) => {
|
|
|
5104
6330
|
look();
|
|
5105
6331
|
});
|
|
5106
6332
|
/** This function gives the count of lost records in the latest clock line of the prober. */
|
|
5107
|
-
const droppedSoFar$
|
|
6333
|
+
const droppedSoFar$3 = (records) => {
|
|
5108
6334
|
for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
|
|
5109
6335
|
return 0;
|
|
5110
6336
|
};
|
|
@@ -5129,7 +6355,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
|
|
|
5129
6355
|
await lab.forget();
|
|
5130
6356
|
const from = lab.kernel.length;
|
|
5131
6357
|
const logFrom = lab.log.length;
|
|
5132
|
-
const lostBefore = droppedSoFar$
|
|
6358
|
+
const lostBefore = droppedSoFar$3(lab.kernel);
|
|
5133
6359
|
const client = await lab.client(replicaSetArgs(cell, lab.members));
|
|
5134
6360
|
const lands = cell.prediction.group?.lands ?? "new";
|
|
5135
6361
|
const place = cell.faultAt ?? { role: "primary" };
|
|
@@ -5139,7 +6365,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
|
|
|
5139
6365
|
const undo = [];
|
|
5140
6366
|
try {
|
|
5141
6367
|
await client.until(["ready"], 6e4);
|
|
5142
|
-
if (await waitFor$
|
|
6368
|
+
if (await waitFor$2(client, (event) => event.ev === "ack" && Number(event["n"]) >= 20, 3e4) === void 0) throw new Error(`the client did not write 20 times in 30 s`);
|
|
5143
6369
|
let healing = null;
|
|
5144
6370
|
switch (cell.fault) {
|
|
5145
6371
|
case "blackhole":
|
|
@@ -5153,7 +6379,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
|
|
|
5153
6379
|
undo.push(() => lab.lift(kind, part, cuts));
|
|
5154
6380
|
if (cell.stall !== void 0) {
|
|
5155
6381
|
const t1 = fault.t1;
|
|
5156
|
-
const sent = await waitFor$
|
|
6382
|
+
const sent = await waitFor$2(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
|
|
5157
6383
|
healing = lab.lift(kind, part, cuts, (sent?.t ?? t1) + valueIn(cell.stall, "s") * 1e9);
|
|
5158
6384
|
} else if (cell.hold !== void 0) healing = lab.lift(kind, part, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
|
|
5159
6385
|
break;
|
|
@@ -5188,10 +6414,10 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
|
|
|
5188
6414
|
const t1 = fault.t1;
|
|
5189
6415
|
const old = `${lab.members[primary].ip}:${set.port}`;
|
|
5190
6416
|
const sentAt = (n) => client.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
|
|
5191
|
-
if (await waitFor$
|
|
6417
|
+
if (await waitFor$2(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1 && (lands === "same" || typeof event["at"] === "string" && event["at"] !== old), deadlineOf(cell)) !== void 0) await sleep$3(TAIL_MS$2);
|
|
5192
6418
|
if (healing !== null) {
|
|
5193
6419
|
await healed.catch(() => void 0);
|
|
5194
|
-
await sleep$
|
|
6420
|
+
await sleep$3(afterLiftOf(cell));
|
|
5195
6421
|
}
|
|
5196
6422
|
} finally {
|
|
5197
6423
|
for (const step of undo.reverse()) await step().catch(() => void 0);
|
|
@@ -5249,7 +6475,7 @@ const until = async (what, check, within) => {
|
|
|
5249
6475
|
const started = Date.now();
|
|
5250
6476
|
while (!await check().catch(() => false)) {
|
|
5251
6477
|
if (Date.now() - started > within) throw new Error(`${what} did not come in ${within} ms`);
|
|
5252
|
-
await sleep$
|
|
6478
|
+
await sleep$3(500);
|
|
5253
6479
|
}
|
|
5254
6480
|
};
|
|
5255
6481
|
/** This function gives the time that the client sent a write. Zero is no write. */
|
|
@@ -5307,7 +6533,7 @@ const backupsOf = async (lab, drill, primary, writer) => {
|
|
|
5307
6533
|
}];
|
|
5308
6534
|
const every = valueIn(drill.sliceEvery, "s") * 1e3;
|
|
5309
6535
|
for (let slice = 1; slice <= drill.slices; slice += 1) {
|
|
5310
|
-
await sleep$
|
|
6536
|
+
await sleep$3(Math.max(0, ended + slice * every - Date.now()));
|
|
5311
6537
|
const out = `${VAULT}/slice-${slice}`;
|
|
5312
6538
|
const begun = Date.now();
|
|
5313
6539
|
await lab.vault([
|
|
@@ -5351,7 +6577,7 @@ const lose = async (lab, drill, primary, last) => {
|
|
|
5351
6577
|
const parts = drill.recovery === "restore" ? [primary, ...lab.set.members.filter((name) => name !== primary)] : [primary];
|
|
5352
6578
|
const age = valueIn(drill.age, "s");
|
|
5353
6579
|
const at = last === null || drill.quiet === true ? void 0 : last.t + age * 1e9;
|
|
5354
|
-
if (at === void 0) await sleep$
|
|
6580
|
+
if (at === void 0) await sleep$3(age * 1e3);
|
|
5355
6581
|
const stamps = await Promise.all(parts.map((part) => lab.signal(part, "KILL", at)));
|
|
5356
6582
|
for (const part of parts) {
|
|
5357
6583
|
await lab.exec(part, [
|
|
@@ -5442,7 +6668,7 @@ const drillOnce = async (lab, cell, environment, index) => {
|
|
|
5442
6668
|
const tool = (await lab.vault(["mongodump", "--version"])).split("\n")[0].trim();
|
|
5443
6669
|
const from = lab.kernel.length;
|
|
5444
6670
|
const logFrom = lab.log.length;
|
|
5445
|
-
const lostBefore = droppedSoFar$
|
|
6671
|
+
const lostBefore = droppedSoFar$3(lab.kernel);
|
|
5446
6672
|
const writer = await lab.client(replicaSetArgs(cell, lab.members));
|
|
5447
6673
|
const restores = drill.recovery === "restore";
|
|
5448
6674
|
let successor = null;
|
|
@@ -5452,7 +6678,7 @@ const drillOnce = async (lab, cell, environment, index) => {
|
|
|
5452
6678
|
const backups = [];
|
|
5453
6679
|
try {
|
|
5454
6680
|
await writer.until(["ready"], 6e4);
|
|
5455
|
-
if (await waitFor$
|
|
6681
|
+
if (await waitFor$2(writer, (event) => event.ev === "ack" && Number(event["n"]) >= 20, 3e4) === void 0) throw new Error(`the client did not write 20 times in 30 s`);
|
|
5456
6682
|
if (drill.quiet === true) await writer.stop();
|
|
5457
6683
|
if (restores) backups.push(...await backupsOf(lab, drill, primary, writer));
|
|
5458
6684
|
const stamp = await lose(lab, drill, primary, backups.at(-1)?.last ?? null);
|
|
@@ -5466,11 +6692,11 @@ const drillOnce = async (lab, cell, environment, index) => {
|
|
|
5466
6692
|
"--from",
|
|
5467
6693
|
String(second)
|
|
5468
6694
|
]);
|
|
5469
|
-
if (await waitFor$
|
|
6695
|
+
if (await waitFor$2(successor, (event) => event.ev === "ack", deadlineOf(cell)) !== void 0) await sleep$3(TAIL_MS$2);
|
|
5470
6696
|
} else {
|
|
5471
6697
|
const old = `${lab.members[primary].ip}:${set.port}`;
|
|
5472
6698
|
const sentAt = (n) => writer.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
|
|
5473
|
-
if (await waitFor$
|
|
6699
|
+
if (await waitFor$2(writer, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > stamp.t1 && typeof event["at"] === "string" && event["at"] !== old, deadlineOf(cell)) !== void 0) await sleep$3(TAIL_MS$2);
|
|
5474
6700
|
await lab.restart(primary);
|
|
5475
6701
|
}
|
|
5476
6702
|
} finally {
|
|
@@ -5632,6 +6858,7 @@ const clusterViewOf = (cluster, rows) => {
|
|
|
5632
6858
|
* This function gives the peers that a fault on a node cuts it from. `all` is each other node and the
|
|
5633
6859
|
* client, `group` is the other nodes, and `clients` is the client. `one` is the first other node in
|
|
5634
6860
|
* the sequence of the cluster, and `one-way` is the same node for the segments from the part only.
|
|
6861
|
+
* `to-clients` is the client, for the segments from the part only.
|
|
5635
6862
|
*/
|
|
5636
6863
|
const clusterCutOf = (cluster, members, client, part, cuts) => {
|
|
5637
6864
|
const others = cluster.members.filter((name) => name !== part).map((name) => members[name].ip);
|
|
@@ -5642,9 +6869,13 @@ const clusterCutOf = (cluster, members, client, part, cuts) => {
|
|
|
5642
6869
|
peers: others.slice(0, 1),
|
|
5643
6870
|
direction: "out"
|
|
5644
6871
|
};
|
|
6872
|
+
if (cuts === "to-clients") return {
|
|
6873
|
+
peers: [client],
|
|
6874
|
+
direction: "out"
|
|
6875
|
+
};
|
|
5645
6876
|
return { peers: [...others, client] };
|
|
5646
6877
|
};
|
|
5647
|
-
const sleep$
|
|
6878
|
+
const sleep$2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
5648
6879
|
/**
|
|
5649
6880
|
* This function opens one environment of a cluster cell:
|
|
5650
6881
|
* - one new network, with each node, the client and the admin on it.
|
|
@@ -5806,7 +7037,7 @@ const openClusterLab = async (cluster) => {
|
|
|
5806
7037
|
calm = last.healthy ? calm + 1 : 0;
|
|
5807
7038
|
if (last.primary !== null) known = last.primary;
|
|
5808
7039
|
if (calm >= 2 && last.primary !== null) return last.primary;
|
|
5809
|
-
await sleep$
|
|
7040
|
+
await sleep$2(1e3);
|
|
5810
7041
|
}
|
|
5811
7042
|
throw new Error(`the queue did not converge in ${within} ms: ${last.why}`);
|
|
5812
7043
|
};
|
|
@@ -5822,7 +7053,7 @@ const openClusterLab = async (cluster) => {
|
|
|
5822
7053
|
const running = (text.trim() === "" ? {} : toolJsonOf(text)).running_nodes ?? [];
|
|
5823
7054
|
if (cluster.members.every((name) => running.includes(nodeNameOf(name)))) break;
|
|
5824
7055
|
if (Date.now() - started > 12e4) throw new Error(`the cluster did not form in 120 s: ${running.join(", ")} run`);
|
|
5825
|
-
await sleep$
|
|
7056
|
+
await sleep$2(1e3);
|
|
5826
7057
|
}
|
|
5827
7058
|
const declared = lastLine(await output(commander, [
|
|
5828
7059
|
"node",
|
|
@@ -5942,7 +7173,7 @@ const openClusterLab = async (cluster) => {
|
|
|
5942
7173
|
"check_running"
|
|
5943
7174
|
]).then(() => true).catch(() => false)) {
|
|
5944
7175
|
if (Date.now() - begun > 12e4) throw new Error(`${member} did not start again`);
|
|
5945
|
-
await sleep$
|
|
7176
|
+
await sleep$2(1e3);
|
|
5946
7177
|
}
|
|
5947
7178
|
},
|
|
5948
7179
|
stepDown(leader, to) {
|
|
@@ -6044,7 +7275,7 @@ const openClusterLab = async (cluster) => {
|
|
|
6044
7275
|
}
|
|
6045
7276
|
};
|
|
6046
7277
|
/** This count is the number of confirmed publishes that a run waits for before its fault: the stream is steady then. */
|
|
6047
|
-
const STEADY = 20;
|
|
7278
|
+
const STEADY$1 = 20;
|
|
6048
7279
|
/** This time is how long a run continues after its outcome, so that the record shows what came next. */
|
|
6049
7280
|
const TAIL_MS$1 = 2e3;
|
|
6050
7281
|
/**
|
|
@@ -6073,7 +7304,7 @@ const clusterArgs = (cell, members, leader) => [
|
|
|
6073
7304
|
...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
|
|
6074
7305
|
];
|
|
6075
7306
|
/** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
|
|
6076
|
-
const waitFor = (client, match, within) => new Promise((resolve) => {
|
|
7307
|
+
const waitFor$1 = (client, match, within) => new Promise((resolve) => {
|
|
6077
7308
|
const started = Date.now();
|
|
6078
7309
|
const look = () => {
|
|
6079
7310
|
const found = client.events.find(match);
|
|
@@ -6083,7 +7314,7 @@ const waitFor = (client, match, within) => new Promise((resolve) => {
|
|
|
6083
7314
|
look();
|
|
6084
7315
|
});
|
|
6085
7316
|
/** This function gives the count of lost records in the latest clock line of the prober. */
|
|
6086
|
-
const droppedSoFar$
|
|
7317
|
+
const droppedSoFar$2 = (records) => {
|
|
6087
7318
|
for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
|
|
6088
7319
|
return 0;
|
|
6089
7320
|
};
|
|
@@ -6113,7 +7344,7 @@ const stampOf = async (lab, client, logFrom, marks, wall) => {
|
|
|
6113
7344
|
},
|
|
6114
7345
|
line
|
|
6115
7346
|
};
|
|
6116
|
-
await sleep$
|
|
7347
|
+
await sleep$2(100);
|
|
6117
7348
|
}
|
|
6118
7349
|
return {
|
|
6119
7350
|
stamp: monotonicOf(wall, client.events),
|
|
@@ -6150,7 +7381,7 @@ const clusterOnce = async (lab, cell, environment, index) => {
|
|
|
6150
7381
|
await lab.forget();
|
|
6151
7382
|
const from = lab.kernel.length;
|
|
6152
7383
|
const logFrom = lab.log.length;
|
|
6153
|
-
const lostBefore = droppedSoFar$
|
|
7384
|
+
const lostBefore = droppedSoFar$2(lab.kernel);
|
|
6154
7385
|
const client = await lab.client(clusterArgs(cell, lab.members, leader));
|
|
6155
7386
|
const place = cell.faultAt ?? { role: "primary" };
|
|
6156
7387
|
let fault = null;
|
|
@@ -6161,7 +7392,7 @@ const clusterOnce = async (lab, cell, environment, index) => {
|
|
|
6161
7392
|
const undo = [];
|
|
6162
7393
|
try {
|
|
6163
7394
|
await client.until(["ready"], 6e4);
|
|
6164
|
-
if (await waitFor(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY, 3e4) === void 0) throw new Error(`the client did not publish ${STEADY} times in 30 s`);
|
|
7395
|
+
if (await waitFor$1(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$1, 3e4) === void 0) throw new Error(`the client did not publish ${STEADY$1} times in 30 s`);
|
|
6165
7396
|
let healing = null;
|
|
6166
7397
|
switch (cell.fault) {
|
|
6167
7398
|
case "blackhole":
|
|
@@ -6215,10 +7446,10 @@ const clusterOnce = async (lab, cell, environment, index) => {
|
|
|
6215
7446
|
healed.catch(() => void 0);
|
|
6216
7447
|
const t1 = fault.t1;
|
|
6217
7448
|
const sentAt = (n) => client.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
|
|
6218
|
-
if (await waitFor(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1, deadlineOf(cell)) !== void 0) await sleep$
|
|
7449
|
+
if (await waitFor$1(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1, deadlineOf(cell)) !== void 0) await sleep$2(TAIL_MS$1);
|
|
6219
7450
|
if (healing !== null) {
|
|
6220
7451
|
await healed.catch(() => void 0);
|
|
6221
|
-
await sleep$
|
|
7452
|
+
await sleep$2(TAIL_MS$1);
|
|
6222
7453
|
}
|
|
6223
7454
|
} finally {
|
|
6224
7455
|
for (const step of undo.reverse()) await step().catch(() => void 0);
|
|
@@ -6250,129 +7481,767 @@ const clusterOnce = async (lab, cell, environment, index) => {
|
|
|
6250
7481
|
};
|
|
6251
7482
|
};
|
|
6252
7483
|
//#endregion
|
|
6253
|
-
//#region src/
|
|
7484
|
+
//#region src/kafka.ts
|
|
6254
7485
|
/**
|
|
6255
|
-
* This
|
|
6256
|
-
*
|
|
6257
|
-
*
|
|
7486
|
+
* This module is the lab of a group of Kafka brokers in KRaft mode with one controller. It opens one
|
|
7487
|
+
* environment for a Kafka cell, for the runs of the cell. Each node is a container from the stock
|
|
7488
|
+
* image, in its own network namespace, and an injector sidecar shares its network and process
|
|
7489
|
+
* namespaces. The image starts by its own command, and each setting comes from the environment.
|
|
7490
|
+
*
|
|
7491
|
+
* Each run makes one new topic, thus each run starts with an empty log and its client counts from 1.
|
|
7492
|
+
* The client is the producer of kafka-clients. The commander holds the admin, on the network but
|
|
7493
|
+
* not in the namespace that the prober watches. The record of a run is the lines of the client and
|
|
7494
|
+
* the whole partition, read to its high watermark after the group converged.
|
|
6258
7495
|
*/
|
|
6259
|
-
|
|
6260
|
-
|
|
6261
|
-
|
|
6262
|
-
|
|
6263
|
-
|
|
6264
|
-
|
|
6265
|
-
|
|
6266
|
-
|
|
6267
|
-
|
|
6268
|
-
|
|
6269
|
-
return 0;
|
|
6270
|
-
};
|
|
6271
|
-
/** This function gives the argv of the client for a cell, with a deadline far after all that it waits for. */
|
|
6272
|
-
const probeArgs = (cell, lab) => {
|
|
6273
|
-
const { prediction, probe } = cell;
|
|
6274
|
-
const longest = (prediction.detects === true && prediction.driver !== void 0 ? longestOf(cell) : prediction.detects ? latestEndOf(prediction) : valueIn(prediction.window, "s")) + (cell.idleBefore === void 0 ? 0 : valueIn(cell.idleBefore, "s"));
|
|
6275
|
-
const args = [
|
|
6276
|
-
lab.targetIp,
|
|
6277
|
-
String(lab.targetPort),
|
|
6278
|
-
"--mode",
|
|
6279
|
-
probe.mode
|
|
6280
|
-
];
|
|
6281
|
-
if (probe.ws !== void 0) {
|
|
6282
|
-
const { ws } = probe;
|
|
6283
|
-
const waits = (prediction.detects !== false ? prediction.reconnects?.waits ?? [] : []).reduce((sum, wait) => sum + valueIn(wait.high, "s"), 0);
|
|
6284
|
-
args.push("--websocket", ws.socket);
|
|
6285
|
-
args.push("--heartbeat", ws.heartbeat === null ? "off" : "on");
|
|
6286
|
-
if (ws.heartbeat !== null) args.push("--heartbeat-interval-ms", String(valueIn(ws.heartbeat.interval, "ms")), "--heartbeat-timeout-ms", String(valueIn(ws.heartbeat.timeout, "ms")), "--heartbeat-misses", String(ws.heartbeat.misses));
|
|
6287
|
-
args.push("--reconnect", ws.reconnect === null ? "off" : "on");
|
|
6288
|
-
if (ws.reconnect !== null) args.push("--reconnect-initial-ms", String(valueIn(ws.reconnect.initialDelay, "ms")), "--reconnect-multiplier", String(ws.reconnect.multiplier), "--reconnect-max-ms", String(valueIn(ws.reconnect.maxDelay, "ms")), "--reconnect-attempts", String(ws.reconnect.attempts), "--reconnect-jitter", String(ws.reconnect.jitter));
|
|
6289
|
-
args.push("--request-timeout-ms", String(valueIn(ws.requestTimeout, "ms")));
|
|
6290
|
-
args.push("--connection-timeout-ms", String(valueIn(ws.connectionTimeout, "ms")));
|
|
6291
|
-
if (ws.hold !== void 0) args.push("--hold-ms", String(valueIn(ws.hold, "ms")));
|
|
6292
|
-
args.push("--max-s", String(Math.ceil((longest + waits) * 2 + 10)));
|
|
6293
|
-
return args;
|
|
6294
|
-
}
|
|
6295
|
-
if (probe.http !== void 0) {
|
|
6296
|
-
const { http } = probe;
|
|
6297
|
-
const gap = http.gap === void 0 ? 0 : valueIn(http.gap, "s");
|
|
6298
|
-
args.push("--method", http.method);
|
|
6299
|
-
if (http.gap !== void 0) args.push("--gap-ms", String(valueIn(http.gap, "ms")));
|
|
6300
|
-
if (http.hold !== void 0) args.push("--hold-ms", String(valueIn(http.hold, "ms")));
|
|
6301
|
-
if (http.bytes !== void 0) args.push("--bytes", String(http.bytes));
|
|
6302
|
-
for (const [name, value] of Object.entries(http.options ?? {})) args.push(`--${name}`, String(value));
|
|
6303
|
-
args.push("--linger-ms", String(LINGER_MS));
|
|
6304
|
-
args.push("--max-s", String(Math.ceil(longest * 2 + gap + 10)));
|
|
6305
|
-
return args;
|
|
6306
|
-
}
|
|
6307
|
-
if (probe.request !== void 0) args.push("--request", probe.request);
|
|
6308
|
-
if (probe.count !== void 0) args.push("--count", String(probe.count));
|
|
6309
|
-
if (probe.interval !== void 0) args.push("--interval-ms", String(valueIn(probe.interval, "ms")));
|
|
6310
|
-
if (probe.driver?.name === "mongodb") {
|
|
6311
|
-
args.push("--set", probe.driver.set);
|
|
6312
|
-
if (probe.driver.op !== void 0) args.push("--op", probe.driver.op);
|
|
6313
|
-
if (cell.writeConcern !== void 0) args.push("--w", String(cell.writeConcern));
|
|
6314
|
-
if (cell.before === true) args.push("--before");
|
|
6315
|
-
if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
|
|
6316
|
-
args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
|
|
6317
|
-
return args;
|
|
6318
|
-
}
|
|
6319
|
-
if (probe.driver?.name === "pg" || probe.driver?.name === "hikari" || probe.driver?.name === "amqp") {
|
|
6320
|
-
if (probe.driver.name !== "amqp" && probe.driver.query !== void 0) args.push("--query", probe.driver.query);
|
|
6321
|
-
if (probe.driver.name !== "amqp" && probe.driver.inflight !== void 0) args.push("--inflight-query", probe.driver.inflight);
|
|
6322
|
-
if (cell.before === true) args.push("--before");
|
|
6323
|
-
if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
|
|
6324
|
-
args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
|
|
6325
|
-
return args;
|
|
6326
|
-
}
|
|
6327
|
-
if (probe.driver?.name === "grpc") {
|
|
6328
|
-
if (probe.driver.hold !== void 0) args.push("--hold-ms", String(valueIn(probe.driver.hold, "ms")));
|
|
6329
|
-
if (probe.driver.op !== void 0) args.push("--op", probe.driver.op);
|
|
6330
|
-
if (cell.before === true) args.push("--before");
|
|
6331
|
-
if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
|
|
6332
|
-
args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
|
|
6333
|
-
return args;
|
|
6334
|
-
}
|
|
6335
|
-
if (probe.driver !== void 0) {
|
|
6336
|
-
args.push("--keepalive-ms", String(valueIn(probe.driver.keepAlive, "ms")));
|
|
6337
|
-
args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
|
|
6338
|
-
return args;
|
|
6339
|
-
}
|
|
6340
|
-
if (probe.keepalive) args.push("--keepalive");
|
|
6341
|
-
if (probe.userTimeout !== void 0) args.push("--user-timeout-ms", String(valueIn(probe.userTimeout, "ms")));
|
|
6342
|
-
args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
|
|
6343
|
-
return args;
|
|
7496
|
+
/** This path is the class path of the Kafka image: the JVM expands the star itself. */
|
|
7497
|
+
const CLASS_PATH = "/opt/harness/kafka/classes:/opt/harness/kafka/lib/*";
|
|
7498
|
+
/** This option is the heap of the client and of the admin. It has no source: the two hold a producer or a read of one partition. */
|
|
7499
|
+
const CLIENT_HEAP = "-Xmx256m";
|
|
7500
|
+
/** This function gives the node id of a name. A broker has its place in the list from 1, and the controller has the id after them. */
|
|
7501
|
+
const nodeIdOf = (kafka, name) => {
|
|
7502
|
+
const at = kafka.brokers.indexOf(name);
|
|
7503
|
+
if (at !== -1) return at + 1;
|
|
7504
|
+
if (name === kafka.controller) return kafka.brokers.length + 1;
|
|
7505
|
+
throw new Error(`the Kafka group has no node ${name}`);
|
|
6344
7506
|
};
|
|
6345
7507
|
/**
|
|
6346
|
-
* This function
|
|
6347
|
-
*
|
|
6348
|
-
*
|
|
6349
|
-
* itself (`dropped`). The promise resolves after `within` ms at the latest.
|
|
7508
|
+
* This function gives the variable of a server property for the image. The wrapper of the image
|
|
7509
|
+
* reverses this map (KafkaDockerWrapper.scala:188-203). After the prefix `KAFKA_`, `___` is a
|
|
7510
|
+
* hyphen, `__` is an underscore and `_` is a dot.
|
|
6350
7511
|
*/
|
|
6351
|
-
const
|
|
6352
|
-
|
|
6353
|
-
|
|
6354
|
-
|
|
6355
|
-
|
|
6356
|
-
|
|
6357
|
-
|
|
6358
|
-
|
|
6359
|
-
|
|
7512
|
+
const envOf = (property) => `KAFKA_${property.replace(/_/g, "__").replace(/-/g, "___").replace(/\./g, "_").toUpperCase()}`;
|
|
7513
|
+
/** This function gives the variables of each node: the listener names, the voters, the log and the settings of the group. */
|
|
7514
|
+
const commonEnv = (kafka) => ({
|
|
7515
|
+
CLUSTER_ID: kafka.clusterId,
|
|
7516
|
+
KAFKA_HEAP_OPTS: kafka.heap,
|
|
7517
|
+
KAFKA_CONTROLLER_QUORUM_VOTERS: `${nodeIdOf(kafka, kafka.controller)}@${kafka.controller}:${kafka.controllerPort}`,
|
|
7518
|
+
KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER",
|
|
7519
|
+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT",
|
|
7520
|
+
KAFKA_LOG_DIRS: "/var/lib/kafka/data",
|
|
7521
|
+
...Object.fromEntries(Object.entries(kafka.settings).map(([property, value]) => [envOf(property), value]))
|
|
7522
|
+
});
|
|
7523
|
+
/** This function gives the environment of a broker. */
|
|
7524
|
+
const brokerEnv = (kafka, name) => ({
|
|
7525
|
+
...commonEnv(kafka),
|
|
7526
|
+
KAFKA_PROCESS_ROLES: "broker",
|
|
7527
|
+
KAFKA_NODE_ID: String(nodeIdOf(kafka, name)),
|
|
7528
|
+
KAFKA_LISTENERS: `PLAINTEXT://0.0.0.0:${kafka.port}`,
|
|
7529
|
+
KAFKA_ADVERTISED_LISTENERS: `PLAINTEXT://${name}:${kafka.port}`,
|
|
7530
|
+
KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
|
|
7531
|
+
});
|
|
7532
|
+
/** This function gives the environment of the controller. The start script of the image stops on an advertised listener here (configure:43-48). */
|
|
7533
|
+
const controllerEnv = (kafka) => ({
|
|
7534
|
+
...commonEnv(kafka),
|
|
7535
|
+
KAFKA_PROCESS_ROLES: "controller",
|
|
7536
|
+
KAFKA_NODE_ID: String(nodeIdOf(kafka, kafka.controller)),
|
|
7537
|
+
KAFKA_LISTENERS: `CONTROLLER://0.0.0.0:${kafka.controllerPort}`
|
|
6360
7538
|
});
|
|
6361
|
-
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
6362
|
-
/**
|
|
6363
|
-
* This time is the pause between the client's first reply and the fault. It is more than Linux's
|
|
6364
|
-
* TCP_DELACK_MAX of 200 ms. Thus the delayed ACK of the reply is out before the fault.
|
|
6365
|
-
*/
|
|
6366
|
-
const QUIET_MS = 250;
|
|
6367
7539
|
/**
|
|
6368
|
-
* This
|
|
6369
|
-
*
|
|
7540
|
+
* This function judges a view of a partition: its leader, and if the group is healthy. A healthy
|
|
7541
|
+
* group has each broker not fenced, one leader, and each broker in the in-sync replicas.
|
|
6370
7542
|
*/
|
|
6371
|
-
const
|
|
6372
|
-
|
|
6373
|
-
const
|
|
6374
|
-
|
|
6375
|
-
|
|
7543
|
+
const kafkaViewOf = (kafka, view) => {
|
|
7544
|
+
const primary = view.leader === void 0 || view.leader === null ? null : kafka.brokers[view.leader - 1] ?? null;
|
|
7545
|
+
const all = kafka.brokers.length;
|
|
7546
|
+
if (view.brokers.length !== all) return {
|
|
7547
|
+
primary,
|
|
7548
|
+
healthy: false,
|
|
7549
|
+
why: `${view.brokers.length} brokers are not fenced`
|
|
7550
|
+
};
|
|
7551
|
+
if (primary === null) return {
|
|
7552
|
+
primary,
|
|
7553
|
+
healthy: false,
|
|
7554
|
+
why: "the partition has no leader"
|
|
7555
|
+
};
|
|
7556
|
+
if (view.replicas?.length !== all) return {
|
|
7557
|
+
primary,
|
|
7558
|
+
healthy: false,
|
|
7559
|
+
why: `the partition has ${view.replicas?.length ?? 0} replicas`
|
|
7560
|
+
};
|
|
7561
|
+
if (view.isr?.length !== all) return {
|
|
7562
|
+
primary,
|
|
7563
|
+
healthy: false,
|
|
7564
|
+
why: `the partition has ${view.isr?.length ?? 0} in-sync replicas`
|
|
7565
|
+
};
|
|
7566
|
+
return {
|
|
7567
|
+
primary,
|
|
7568
|
+
healthy: true,
|
|
7569
|
+
why: "healthy"
|
|
7570
|
+
};
|
|
7571
|
+
};
|
|
7572
|
+
/**
|
|
7573
|
+
* This function gives the peers that a fault on a broker cuts it from. Only `group` has a law: the
|
|
7574
|
+
* other brokers and the controller, and the client keeps its route to the broker.
|
|
7575
|
+
*/
|
|
7576
|
+
const kafkaCutOf = (kafka, members, controller, part, cuts) => {
|
|
7577
|
+
if (cuts !== "group") throw new Error(`a Kafka cell has no cut ${cuts}`);
|
|
7578
|
+
return { peers: [...kafka.brokers.filter((name) => name !== part).map((name) => members[name].ip), controller.ip] };
|
|
7579
|
+
};
|
|
7580
|
+
/** This function gives the addresses that the client and the admin bootstrap from. */
|
|
7581
|
+
const bootstrapOf = (kafka) => kafka.brokers.map((name) => `${name}:${kafka.port}`).join(",");
|
|
7582
|
+
/** This function gives the name of the topic of a run. A warm-up has the index -1. */
|
|
7583
|
+
const topicOf = (environment, index) => `cells-e${environment}-r${index + 1}`;
|
|
7584
|
+
const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
7585
|
+
/**
|
|
7586
|
+
* This function opens one environment of a Kafka cell:
|
|
7587
|
+
* - one new network, with each node, the client and the commander on it.
|
|
7588
|
+
* - the controller first, then each broker: the stock image with its own command. The brokers
|
|
7589
|
+
* register with the controller, thus they need it first.
|
|
7590
|
+
* - an injector sidecar for each node, in its network and process namespaces.
|
|
7591
|
+
* - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
|
|
7592
|
+
* - the prober, which watches the namespace of the client.
|
|
7593
|
+
*/
|
|
7594
|
+
const openKafkaLab = async (kafka) => {
|
|
7595
|
+
const image = await harnessImage();
|
|
7596
|
+
const clientImage = await kafkaClientImage();
|
|
7597
|
+
const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();
|
|
7598
|
+
const runtime = await getContainerRuntimeClient();
|
|
7599
|
+
const opened = [];
|
|
7600
|
+
let network;
|
|
7601
|
+
const close = async () => {
|
|
7602
|
+
for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
|
|
7603
|
+
await network?.stop().catch(() => void 0);
|
|
7604
|
+
};
|
|
7605
|
+
const ipOf = async (container) => {
|
|
7606
|
+
const ip = (await runtime.container.inspect(runtime.container.getById(container.getId()))).NetworkSettings.Networks[network.getName()]?.IPAddress;
|
|
7607
|
+
if (ip === void 0 || ip === "") throw new Error(`${container.getName()} has no address on ${network.getName()}`);
|
|
7608
|
+
return ip;
|
|
7609
|
+
};
|
|
7610
|
+
try {
|
|
7611
|
+
network = await new Network().start();
|
|
7612
|
+
const started = network;
|
|
7613
|
+
const boxes = /* @__PURE__ */ new Map();
|
|
7614
|
+
const injectors = /* @__PURE__ */ new Map();
|
|
7615
|
+
const parts = {};
|
|
7616
|
+
const raw = {};
|
|
7617
|
+
const fences = [];
|
|
7618
|
+
const openNode = async (name, environment) => {
|
|
7619
|
+
let partial = "";
|
|
7620
|
+
const box = await new GenericContainer(kafka.image).withNetwork(started).withNetworkAliases(name).withHostname(name).withEnvironment(environment).withWaitStrategy(Wait.forLogMessage(kafka.ready)).withStartupTimeout(18e4).withLogConsumer((stream) => stream.on("data", (chunk) => {
|
|
7621
|
+
partial += chunk.toString();
|
|
7622
|
+
const split = partial.split("\n");
|
|
7623
|
+
partial = split.pop() ?? "";
|
|
7624
|
+
for (const text of split) {
|
|
7625
|
+
const tail = raw[name] ??= [];
|
|
7626
|
+
tail.push(text);
|
|
7627
|
+
if (tail.length > 400) tail.shift();
|
|
7628
|
+
if (isFenceLine(text)) fences.push(text);
|
|
7629
|
+
}
|
|
7630
|
+
})).start();
|
|
7631
|
+
opened.push(box);
|
|
7632
|
+
boxes.set(name, box);
|
|
7633
|
+
const injector = await withPid(new GenericContainer(image).withNetworkMode(`container:${box.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW"), `container:${box.getId()}`).start();
|
|
7634
|
+
opened.push(injector);
|
|
7635
|
+
injectors.set(name, injector);
|
|
7636
|
+
parts[name] = {
|
|
7637
|
+
ip: await ipOf(box),
|
|
7638
|
+
netns: await netnsOf(injector)
|
|
7639
|
+
};
|
|
7640
|
+
};
|
|
7641
|
+
await openNode(kafka.controller, controllerEnv(kafka));
|
|
7642
|
+
for (const name of kafka.brokers) await openNode(name, brokerEnv(kafka, name));
|
|
7643
|
+
const members = Object.fromEntries(kafka.brokers.map((name) => [name, parts[name]]));
|
|
7644
|
+
const controller = parts[kafka.controller];
|
|
7645
|
+
const holder = await new GenericContainer(image).withNetwork(network).start();
|
|
7646
|
+
opened.push(holder);
|
|
7647
|
+
const admin = await new GenericContainer(image).withNetworkMode(`container:${holder.getId()}`).withPrivilegedMode().start();
|
|
7648
|
+
opened.push(admin);
|
|
7649
|
+
const clientIp = await ipOf(holder);
|
|
7650
|
+
const commander = await new GenericContainer(clientImage).withNetwork(network).withCommand([
|
|
7651
|
+
"sh",
|
|
7652
|
+
"-c",
|
|
7653
|
+
"while :; do sleep 3600; done"
|
|
7654
|
+
]).start();
|
|
7655
|
+
opened.push(commander);
|
|
7656
|
+
const clientNetns = await netnsOf(admin);
|
|
7657
|
+
const records = lines();
|
|
7658
|
+
const prober = await new GenericContainer(image).withPrivilegedMode().withNetworkMode("none").withCommand(["/opt/harness/prober", String(clientNetns)]).withLogConsumer(records.consume).start();
|
|
7659
|
+
opened.push(prober);
|
|
7660
|
+
const begun = await records.until(["begin", "fail"], 3e4, "prober");
|
|
7661
|
+
if (begun.ev === "fail") throw new Error(`the prober failed: ${String(begun["why"])}`);
|
|
7662
|
+
const box = (name) => {
|
|
7663
|
+
const found = boxes.get(name);
|
|
7664
|
+
if (found === void 0) throw new Error(`the group has no node ${name}`);
|
|
7665
|
+
return found;
|
|
7666
|
+
};
|
|
7667
|
+
const injectorOf = (name) => {
|
|
7668
|
+
const found = injectors.get(name);
|
|
7669
|
+
if (found === void 0) throw new Error(`the group has no node ${name}`);
|
|
7670
|
+
return found;
|
|
7671
|
+
};
|
|
7672
|
+
const lastLine = (text) => text.split("\n").filter((line) => line.trim() !== "").at(-1) ?? "";
|
|
7673
|
+
const stamped = async (where, argv) => {
|
|
7674
|
+
const stamp = JSON.parse(lastLine(await output(where, argv)));
|
|
7675
|
+
return {
|
|
7676
|
+
t0: stamp.t0,
|
|
7677
|
+
t1: stamp.t1
|
|
7678
|
+
};
|
|
7679
|
+
};
|
|
7680
|
+
const after = (at) => at === void 0 ? [] : ["--at", String(Math.round(at))];
|
|
7681
|
+
const bootstrap = bootstrapOf(kafka);
|
|
7682
|
+
const tool = async (...argv) => lastLine(await output(commander, [
|
|
7683
|
+
"java",
|
|
7684
|
+
CLIENT_HEAP,
|
|
7685
|
+
"-cp",
|
|
7686
|
+
CLASS_PATH,
|
|
7687
|
+
"AdminKafka",
|
|
7688
|
+
bootstrap,
|
|
7689
|
+
...argv
|
|
7690
|
+
]));
|
|
7691
|
+
const view = async (topic) => JSON.parse(await tool("view", ...topic === void 0 ? [] : [topic]));
|
|
7692
|
+
return {
|
|
7693
|
+
kafka,
|
|
7694
|
+
clientIp,
|
|
7695
|
+
members,
|
|
7696
|
+
controller,
|
|
7697
|
+
netns: { client: clientNetns },
|
|
7698
|
+
kernel: records.seen,
|
|
7699
|
+
raw,
|
|
7700
|
+
fences,
|
|
7701
|
+
async sysctl(values) {
|
|
7702
|
+
const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
|
|
7703
|
+
if (settings.length > 0) await output(admin, [
|
|
7704
|
+
"sysctl",
|
|
7705
|
+
"-q",
|
|
7706
|
+
"-w",
|
|
7707
|
+
...settings
|
|
7708
|
+
]);
|
|
7709
|
+
const read = {};
|
|
7710
|
+
for (const key of Object.keys(values)) read[key] = await output(admin, [
|
|
7711
|
+
"sysctl",
|
|
7712
|
+
"-n",
|
|
7713
|
+
key
|
|
7714
|
+
]);
|
|
7715
|
+
return read;
|
|
7716
|
+
},
|
|
7717
|
+
async brokersUp(within = CONVERGE_MS) {
|
|
7718
|
+
const begin = Date.now();
|
|
7719
|
+
let last = "no view yet";
|
|
7720
|
+
while (Date.now() - begin < within) {
|
|
7721
|
+
const read = await view().catch((error) => error.message.slice(0, 200));
|
|
7722
|
+
if (typeof read !== "string" && read.brokers.length === kafka.brokers.length) return;
|
|
7723
|
+
last = typeof read === "string" ? read : `${read.brokers.length} brokers are not fenced`;
|
|
7724
|
+
await sleep$1(1e3);
|
|
7725
|
+
}
|
|
7726
|
+
throw new Error(`the brokers did not come up in ${within} ms: ${last}`);
|
|
7727
|
+
},
|
|
7728
|
+
async create(topic, partitions = 1) {
|
|
7729
|
+
const ids = kafka.brokers.map((name) => nodeIdOf(kafka, name)).join(",");
|
|
7730
|
+
const reply = await tool("create", topic, String(kafka.minInsync), ids, String(partitions));
|
|
7731
|
+
if (!reply.includes("\"ok\":true")) throw new Error(`the create of the topic failed: ${reply}`);
|
|
7732
|
+
},
|
|
7733
|
+
async fill(topic, partitions, count) {
|
|
7734
|
+
const reply = await tool("fill", topic, String(partitions), String(count));
|
|
7735
|
+
if (!reply.includes("\"ok\":true")) throw new Error(`the fill of the topic failed: ${reply}`);
|
|
7736
|
+
},
|
|
7737
|
+
async consumer(args) {
|
|
7738
|
+
const events = lines();
|
|
7739
|
+
const container = await new GenericContainer(clientImage).withNetworkMode(`container:${holder.getId()}`).withCommand([
|
|
7740
|
+
"sh",
|
|
7741
|
+
"-c",
|
|
7742
|
+
"java \"$@\"; :",
|
|
7743
|
+
"consumer",
|
|
7744
|
+
CLIENT_HEAP,
|
|
7745
|
+
"-cp",
|
|
7746
|
+
CLASS_PATH,
|
|
7747
|
+
"ConsumerKafka",
|
|
7748
|
+
bootstrap,
|
|
7749
|
+
...args
|
|
7750
|
+
]).withLogConsumer(events.consume).start();
|
|
7751
|
+
const signaller = await withPid(new GenericContainer(image).withNetworkMode(`container:${holder.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW"), `container:${container.getId()}`).start();
|
|
7752
|
+
return {
|
|
7753
|
+
events: events.seen,
|
|
7754
|
+
until: (kinds, within) => events.until(kinds, within, "client"),
|
|
7755
|
+
signal: (signal, at) => stamped(signaller, [
|
|
7756
|
+
"/opt/harness/inject",
|
|
7757
|
+
"signal",
|
|
7758
|
+
"java",
|
|
7759
|
+
signal,
|
|
7760
|
+
...after(at)
|
|
7761
|
+
]),
|
|
7762
|
+
stop: async () => {
|
|
7763
|
+
await container.stop({ timeout: 2 }).catch(() => void 0);
|
|
7764
|
+
await signaller.stop(STOP).catch(() => void 0);
|
|
7765
|
+
}
|
|
7766
|
+
};
|
|
7767
|
+
},
|
|
7768
|
+
async converge(topic, within = CONVERGE_MS) {
|
|
7769
|
+
const begin = Date.now();
|
|
7770
|
+
let calm = 0;
|
|
7771
|
+
let last = {
|
|
7772
|
+
primary: null,
|
|
7773
|
+
healthy: false,
|
|
7774
|
+
why: "no view yet"
|
|
7775
|
+
};
|
|
7776
|
+
while (Date.now() - begin < within) {
|
|
7777
|
+
last = await view(topic).then((read) => kafkaViewOf(kafka, read), (error) => ({
|
|
7778
|
+
primary: null,
|
|
7779
|
+
healthy: false,
|
|
7780
|
+
why: error.message.slice(0, 200)
|
|
7781
|
+
}));
|
|
7782
|
+
calm = last.healthy ? calm + 1 : 0;
|
|
7783
|
+
if (calm >= 2 && last.primary !== null) return last.primary;
|
|
7784
|
+
await sleep$1(1e3);
|
|
7785
|
+
}
|
|
7786
|
+
throw new Error(`the partition did not converge in ${within} ms: ${last.why}`);
|
|
7787
|
+
},
|
|
7788
|
+
async holds(topic) {
|
|
7789
|
+
const reply = await tool("holds", topic);
|
|
7790
|
+
const { held } = JSON.parse(reply);
|
|
7791
|
+
if (held === void 0) throw new Error(`the read of the partition failed: ${reply}`);
|
|
7792
|
+
return held.map(Number).filter((n) => Number.isInteger(n) && n > 0);
|
|
7793
|
+
},
|
|
7794
|
+
async client(args) {
|
|
7795
|
+
const events = lines();
|
|
7796
|
+
const container = await new GenericContainer(clientImage).withNetworkMode(`container:${holder.getId()}`).withCommand([
|
|
7797
|
+
"java",
|
|
7798
|
+
CLIENT_HEAP,
|
|
7799
|
+
"-cp",
|
|
7800
|
+
CLASS_PATH,
|
|
7801
|
+
"ClientKafka",
|
|
7802
|
+
bootstrap,
|
|
7803
|
+
...args
|
|
7804
|
+
]).withLogConsumer(events.consume).start();
|
|
7805
|
+
return {
|
|
7806
|
+
events: events.seen,
|
|
7807
|
+
until: (kinds, within) => events.until(kinds, within, "client"),
|
|
7808
|
+
stop: () => container.stop({ timeout: 2 }).then(() => void 0).catch(() => void 0)
|
|
7809
|
+
};
|
|
7810
|
+
},
|
|
7811
|
+
fault(kind, part, cuts, at) {
|
|
7812
|
+
const cut = kafkaCutOf(kafka, members, controller, part, cuts);
|
|
7813
|
+
return stamped(injectorOf(part), [
|
|
7814
|
+
"/opt/harness/inject",
|
|
7815
|
+
kind,
|
|
7816
|
+
cut.peers.join(","),
|
|
7817
|
+
...after(at)
|
|
7818
|
+
]);
|
|
7819
|
+
},
|
|
7820
|
+
lift(kind, part, cuts, at) {
|
|
7821
|
+
const cut = kafkaCutOf(kafka, members, controller, part, cuts);
|
|
7822
|
+
return stamped(injectorOf(part), [
|
|
7823
|
+
"/opt/harness/inject",
|
|
7824
|
+
kind === "rst" ? "clear-rst" : "clear",
|
|
7825
|
+
cut.peers.join(","),
|
|
7826
|
+
...after(at)
|
|
7827
|
+
]);
|
|
7828
|
+
},
|
|
7829
|
+
async forget() {
|
|
7830
|
+
await output(admin, [
|
|
7831
|
+
"ip",
|
|
7832
|
+
"tcp_metrics",
|
|
7833
|
+
"flush",
|
|
7834
|
+
"all"
|
|
7835
|
+
]);
|
|
7836
|
+
},
|
|
7837
|
+
async settle() {
|
|
7838
|
+
const from = records.seen.length;
|
|
7839
|
+
const clock = await new Promise((resolve) => {
|
|
7840
|
+
const begin = Date.now();
|
|
7841
|
+
const look = () => {
|
|
7842
|
+
const line = records.seen.slice(from).find((record) => record.ev === "clock");
|
|
7843
|
+
if (line !== void 0) resolve(line);
|
|
7844
|
+
else if (Date.now() - begin > 5e3) resolve(null);
|
|
7845
|
+
else setTimeout(look, 50);
|
|
7846
|
+
};
|
|
7847
|
+
look();
|
|
7848
|
+
});
|
|
7849
|
+
return clock === null ? null : Number(clock["lost"] ?? 0);
|
|
7850
|
+
},
|
|
7851
|
+
async manifest(sysctls) {
|
|
7852
|
+
const config = await output(admin, ["zcat", "/proc/config.gz"]).catch(() => "");
|
|
7853
|
+
const inspect = async (container) => runtime.container.inspect(runtime.container.getById(container.getId()));
|
|
7854
|
+
const settings = {
|
|
7855
|
+
settings: Object.entries(kafka.settings).map(([property, value]) => `${property}=${value}`).join("; "),
|
|
7856
|
+
heap: kafka.heap,
|
|
7857
|
+
"min.insync.replicas": String(kafka.minInsync)
|
|
7858
|
+
};
|
|
7859
|
+
const list = async (names) => {
|
|
7860
|
+
const out = [];
|
|
7861
|
+
for (const name of names) {
|
|
7862
|
+
const version = lastLine(await output(box(name), ["/opt/kafka/bin/kafka-topics.sh", "--version"]).catch(() => ""));
|
|
7863
|
+
out.push({
|
|
7864
|
+
name,
|
|
7865
|
+
ip: parts[name].ip,
|
|
7866
|
+
id: (await inspect(box(name))).Image,
|
|
7867
|
+
settings: {
|
|
7868
|
+
version,
|
|
7869
|
+
...settings
|
|
7870
|
+
}
|
|
7871
|
+
});
|
|
7872
|
+
}
|
|
7873
|
+
return out;
|
|
7874
|
+
};
|
|
7875
|
+
const brokers = await list(kafka.brokers);
|
|
7876
|
+
return {
|
|
7877
|
+
kernel: await output(holder, ["uname", "-r"]),
|
|
7878
|
+
architecture: await output(holder, ["uname", "-m"]),
|
|
7879
|
+
hz: /^CONFIG_HZ=(\d+)$/m.exec(config)?.[1] ?? null,
|
|
7880
|
+
boot_id: await output(holder, ["cat", "/proc/sys/kernel/random/boot_id"]),
|
|
7881
|
+
docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
|
|
7882
|
+
target: {
|
|
7883
|
+
image: kafka.image,
|
|
7884
|
+
id: brokers[0].id,
|
|
7885
|
+
settings: brokers[0].settings,
|
|
7886
|
+
sysctls: {}
|
|
7887
|
+
},
|
|
7888
|
+
harness: {
|
|
7889
|
+
image,
|
|
7890
|
+
id: (await inspect(holder)).Image,
|
|
7891
|
+
packages: (await output(holder, [
|
|
7892
|
+
"apk",
|
|
7893
|
+
"info",
|
|
7894
|
+
"-v"
|
|
7895
|
+
])).split("\n").sort()
|
|
7896
|
+
},
|
|
7897
|
+
client: {
|
|
7898
|
+
kind: "image",
|
|
7899
|
+
image: clientImage
|
|
7900
|
+
},
|
|
7901
|
+
netns: {
|
|
7902
|
+
client: clientNetns,
|
|
7903
|
+
target: members[kafka.brokers[0]].netns
|
|
7904
|
+
},
|
|
7905
|
+
sysctls,
|
|
7906
|
+
group: {
|
|
7907
|
+
members: brokers,
|
|
7908
|
+
sentinels: await list([kafka.controller]),
|
|
7909
|
+
observer: { image: clientImage }
|
|
7910
|
+
}
|
|
7911
|
+
};
|
|
7912
|
+
},
|
|
7913
|
+
close
|
|
7914
|
+
};
|
|
7915
|
+
} catch (error) {
|
|
7916
|
+
await close();
|
|
7917
|
+
throw error;
|
|
7918
|
+
}
|
|
7919
|
+
};
|
|
7920
|
+
/** This count is the number of acknowledged writes that a run waits for before its fault: the stream is steady then. */
|
|
7921
|
+
const STEADY = 20;
|
|
7922
|
+
/**
|
|
7923
|
+
* This time is how long a run continues after its lift. The record then shows the step-down of the
|
|
7924
|
+
* leader and the writes after it. It has no source. It is above the largest step-down of the
|
|
7925
|
+
* law, which is approximately 2 s after the lift.
|
|
7926
|
+
*/
|
|
7927
|
+
const KAFKA_TAIL_MS = 1e4;
|
|
7928
|
+
/** This function gives the arguments of the client of a Kafka cell: the topic, the writers, the stream of writes and the settings. */
|
|
7929
|
+
const kafkaArgs = (cell, topic) => [
|
|
7930
|
+
"--topic",
|
|
7931
|
+
topic,
|
|
7932
|
+
"--writers",
|
|
7933
|
+
String(cell.kafka.writers),
|
|
7934
|
+
"--interval-ms",
|
|
7935
|
+
String(cell.probe.interval === void 0 ? 50 : valueIn(cell.probe.interval, "ms")),
|
|
7936
|
+
"--max-s",
|
|
7937
|
+
String(Math.ceil(deadlineOf(cell) / 1e3 + (cell.hold === void 0 ? 0 : valueIn(cell.hold, "s")) + 120)),
|
|
7938
|
+
...Object.entries(cell.driverOptions ?? {}).flatMap(([name, value]) => ["--set", `${name}=${String(value)}`])
|
|
7939
|
+
];
|
|
7940
|
+
/** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
|
|
7941
|
+
const waitFor = (client, match, within) => new Promise((resolve) => {
|
|
7942
|
+
const started = Date.now();
|
|
7943
|
+
const look = () => {
|
|
7944
|
+
const found = client.events.find(match);
|
|
7945
|
+
if (found !== void 0 || Date.now() - started > within) resolve(found);
|
|
7946
|
+
else setTimeout(look, 20);
|
|
7947
|
+
};
|
|
7948
|
+
look();
|
|
7949
|
+
});
|
|
7950
|
+
/** This function gives the count of lost records in the latest clock line of the prober. */
|
|
7951
|
+
const droppedSoFar$1 = (records) => {
|
|
7952
|
+
for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
|
|
7953
|
+
return 0;
|
|
7954
|
+
};
|
|
7955
|
+
/**
|
|
7956
|
+
* This function does one run of a Kafka cell:
|
|
7957
|
+
* - Each broker is up, the topic of the run exists, and its partition is whole. The client starts,
|
|
7958
|
+
* and its stream of writes gets steady.
|
|
7959
|
+
* - The fault cuts the leader from the other brokers and the controller, and it lifts at its instant
|
|
7960
|
+
* in the VM. The client keeps its route to the leader, thus only the group knows of the cut.
|
|
7961
|
+
* - The run waits for the lift and for the tail after it.
|
|
7962
|
+
* - Each fault goes, the client stops, and the partition converges again. The admin then reads
|
|
7963
|
+
* the whole partition, thus the record tells which writes the group kept.
|
|
7964
|
+
*/
|
|
7965
|
+
const kafkaOnce = async (lab, cell, environment, index) => {
|
|
7966
|
+
if (cell.fault !== "partition") throw new Error(`a Kafka cell has no fault ${cell.fault}`);
|
|
7967
|
+
if (cell.hold === void 0) throw new Error("a Kafka cell needs a hold: its law reads it");
|
|
7968
|
+
const place = cell.faultAt ?? { role: "primary" };
|
|
7969
|
+
if (place.role !== "primary") throw new Error(`a Kafka cell has no fault on the ${place.role}`);
|
|
7970
|
+
const cuts = place.cuts ?? "group";
|
|
7971
|
+
const topic = topicOf(environment, index);
|
|
7972
|
+
await lab.brokersUp();
|
|
7973
|
+
await lab.create(topic);
|
|
7974
|
+
const leader = await lab.converge(topic);
|
|
7975
|
+
await lab.forget();
|
|
7976
|
+
const from = lab.kernel.length;
|
|
7977
|
+
const lostBefore = droppedSoFar$1(lab.kernel);
|
|
7978
|
+
const client = await lab.client(kafkaArgs(cell, topic));
|
|
7979
|
+
let fault = null;
|
|
7980
|
+
let heal = null;
|
|
7981
|
+
const undo = [];
|
|
7982
|
+
try {
|
|
7983
|
+
await client.until(["ready"], 6e4);
|
|
7984
|
+
if (await waitFor(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY, 3e4) === void 0) throw new Error(`the client did not write ${STEADY} times in 30 s`);
|
|
7985
|
+
fault = await lab.fault("blackhole", leader, cuts);
|
|
7986
|
+
undo.push(() => lab.lift("blackhole", leader, cuts));
|
|
7987
|
+
heal = await lab.lift("blackhole", leader, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
|
|
7988
|
+
undo.length = 0;
|
|
7989
|
+
await sleep$1(KAFKA_TAIL_MS);
|
|
7990
|
+
} finally {
|
|
7991
|
+
for (const step of undo.reverse()) await step().catch(() => void 0);
|
|
7992
|
+
await client.stop();
|
|
7993
|
+
}
|
|
7994
|
+
const settled = await lab.settle();
|
|
7995
|
+
const lost = settled === null ? null : settled - lostBefore;
|
|
7996
|
+
const after = await lab.converge(topic).catch(() => null);
|
|
7997
|
+
const holds = after === null ? null : await lab.holds(topic);
|
|
7998
|
+
return {
|
|
7999
|
+
environment,
|
|
8000
|
+
index,
|
|
8001
|
+
events: [...client.events],
|
|
8002
|
+
fault,
|
|
8003
|
+
...heal === null ? {} : { heal },
|
|
8004
|
+
kernel: lab.kernel.slice(from),
|
|
8005
|
+
netns: lab.netns.client,
|
|
8006
|
+
targetNetns: lab.members[leader].netns,
|
|
8007
|
+
lost,
|
|
8008
|
+
group: {
|
|
8009
|
+
members: lab.members,
|
|
8010
|
+
sentinels: { [lab.kafka.controller]: lab.controller },
|
|
8011
|
+
primary: leader,
|
|
8012
|
+
at: leader,
|
|
8013
|
+
timeline: [],
|
|
8014
|
+
after,
|
|
8015
|
+
holds
|
|
8016
|
+
}
|
|
8017
|
+
};
|
|
8018
|
+
};
|
|
8019
|
+
/** This count is the partitions of the topic of a run of a paused consumer: one for each member. */
|
|
8020
|
+
const PAUSE_PARTITIONS = 2;
|
|
8021
|
+
/** This time bounds the wait for one partition for each member and for two commits, in ms. It has no source: it is far above two batches and one rebalance. */
|
|
8022
|
+
const PAUSE_READY_MS = 9e4;
|
|
8023
|
+
/** This time bounds the wait for the next batch of the first member, in ms. It has no source: it is far above one batch. */
|
|
8024
|
+
const PAUSE_BATCH_MS = 3e4;
|
|
8025
|
+
/**
|
|
8026
|
+
* This function does one run of a paused consumer:
|
|
8027
|
+
* - The topic of the run has two partitions, each with records that a run does not finish.
|
|
8028
|
+
* - Two members join one group, and the run waits until each member holds one partition and the
|
|
8029
|
+
* first member has two commits on its partition.
|
|
8030
|
+
* - A SIGSTOP stops the first member while it holds its next batch, and a SIGCONT ends the stop
|
|
8031
|
+
* after the hold of the cell. The run then waits for the tail.
|
|
8032
|
+
*
|
|
8033
|
+
* The fault is null when the members did not get one partition each. The measure then voids the run.
|
|
8034
|
+
* The law reads no kernel record, thus a lost record does not void this run.
|
|
8035
|
+
*/
|
|
8036
|
+
const kafkaPauseOnce = async (lab, cell, environment, index) => {
|
|
8037
|
+
const pause = cell.kafka?.pause;
|
|
8038
|
+
if (pause === void 0) throw new Error("a run of a paused consumer needs `Kafka.pause`");
|
|
8039
|
+
if (cell.fault !== "freeze-process") throw new Error(`a paused consumer has no fault ${cell.fault}`);
|
|
8040
|
+
if (cell.hold === void 0) throw new Error("a paused consumer needs a hold: its law reads it");
|
|
8041
|
+
const hold = valueIn(cell.hold, "s");
|
|
8042
|
+
const topic = topicOf(environment, index);
|
|
8043
|
+
const group = `pause-e${environment}-r${index + 1}`;
|
|
8044
|
+
await lab.brokersUp();
|
|
8045
|
+
await lab.create(topic, PAUSE_PARTITIONS);
|
|
8046
|
+
const leader = await lab.converge(topic);
|
|
8047
|
+
await lab.fill(topic, PAUSE_PARTITIONS, pause.fill);
|
|
8048
|
+
await lab.forget();
|
|
8049
|
+
const from = lab.kernel.length;
|
|
8050
|
+
const lostBefore = droppedSoFar$1(lab.kernel);
|
|
8051
|
+
const common = [
|
|
8052
|
+
"--topic",
|
|
8053
|
+
topic,
|
|
8054
|
+
"--group",
|
|
8055
|
+
group,
|
|
8056
|
+
"--protocol",
|
|
8057
|
+
pause.protocol,
|
|
8058
|
+
"--max-s",
|
|
8059
|
+
String(Math.ceil(hold + 180)),
|
|
8060
|
+
...Object.entries(cell.driverOptions ?? {}).flatMap(([name, value]) => ["--set", `${name}=${String(value)}`])
|
|
8061
|
+
];
|
|
8062
|
+
const members = [];
|
|
8063
|
+
let fault = null;
|
|
8064
|
+
let heal = null;
|
|
8065
|
+
let otherEvents = [];
|
|
8066
|
+
let firstEvents = [];
|
|
8067
|
+
try {
|
|
8068
|
+
const first = await lab.consumer([
|
|
8069
|
+
...common,
|
|
8070
|
+
"--process-ms",
|
|
8071
|
+
String(valueIn(pause.process, "ms")),
|
|
8072
|
+
"--after-pause",
|
|
8073
|
+
pause.afterPause
|
|
8074
|
+
]);
|
|
8075
|
+
members.push(first);
|
|
8076
|
+
const other = await lab.consumer([
|
|
8077
|
+
...common,
|
|
8078
|
+
"--process-ms",
|
|
8079
|
+
"0",
|
|
8080
|
+
"--after-pause",
|
|
8081
|
+
"commit"
|
|
8082
|
+
]);
|
|
8083
|
+
members.push(other);
|
|
8084
|
+
firstEvents = first.events;
|
|
8085
|
+
otherEvents = other.events;
|
|
8086
|
+
const begun = Date.now();
|
|
8087
|
+
while (Date.now() - begun < PAUSE_READY_MS && !pauseReady(first.events, other.events)) await sleep$1(100);
|
|
8088
|
+
if (pauseReady(first.events, other.events)) {
|
|
8089
|
+
const seen = first.events.at(-1)?.t ?? 0;
|
|
8090
|
+
const batch = await waitFor(first, (event) => event.ev === "processing" && event.t > seen, PAUSE_BATCH_MS);
|
|
8091
|
+
if (batch !== void 0) {
|
|
8092
|
+
fault = await first.signal("STOP", batch.t + valueIn(pause.stopAfter, "s") * 1e9);
|
|
8093
|
+
heal = await first.signal("CONT", fault.t0 + hold * 1e9);
|
|
8094
|
+
await sleep$1(KAFKA_TAIL_MS);
|
|
8095
|
+
}
|
|
8096
|
+
}
|
|
8097
|
+
} finally {
|
|
8098
|
+
for (const member of members) await member.stop();
|
|
8099
|
+
}
|
|
8100
|
+
const settled = await lab.settle();
|
|
8101
|
+
return {
|
|
8102
|
+
environment,
|
|
8103
|
+
index,
|
|
8104
|
+
events: [...firstEvents],
|
|
8105
|
+
fault,
|
|
8106
|
+
...heal === null ? {} : { heal },
|
|
8107
|
+
kernel: lab.kernel.slice(from),
|
|
8108
|
+
netns: lab.netns.client,
|
|
8109
|
+
targetNetns: lab.members[leader].netns,
|
|
8110
|
+
lost: settled === null ? null : settled - lostBefore,
|
|
8111
|
+
pause: {
|
|
8112
|
+
other: [...otherEvents],
|
|
8113
|
+
fences: lab.fences.filter((line) => namesGroup(line, group)),
|
|
8114
|
+
group
|
|
8115
|
+
}
|
|
8116
|
+
};
|
|
8117
|
+
};
|
|
8118
|
+
//#endregion
|
|
8119
|
+
//#region src/run.ts
|
|
8120
|
+
/**
|
|
8121
|
+
* This function measures and judges the raw records of a cell, with the code of the harness at
|
|
8122
|
+
* this time. `runCell` uses it during a cell. A holder of the raw records of a cell can use it
|
|
8123
|
+
* again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.
|
|
8124
|
+
*/
|
|
8125
|
+
const judge = (cell, runs) => {
|
|
8126
|
+
const measured = runs.filter((run) => run.warmup !== true).map((run) => measure(cell, run));
|
|
8127
|
+
return {
|
|
8128
|
+
measured,
|
|
8129
|
+
verdict: verdictOf(cell.prediction, measured, noiseOf(cell))
|
|
8130
|
+
};
|
|
8131
|
+
};
|
|
8132
|
+
/** This function gives the count of lost records in the latest clock line of the prober, since it began. */
|
|
8133
|
+
const droppedSoFar = (records) => {
|
|
8134
|
+
for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
|
|
8135
|
+
return 0;
|
|
8136
|
+
};
|
|
8137
|
+
/** This function gives the argv of the client for a cell, with a deadline far after all that it waits for. */
|
|
8138
|
+
const probeArgs = (cell, lab) => {
|
|
8139
|
+
const { prediction, probe } = cell;
|
|
8140
|
+
const longest = (prediction.detects === true && prediction.driver !== void 0 ? longestOf(cell) : prediction.detects ? latestEndOf(prediction) : valueIn(prediction.window, "s")) + (cell.idleBefore === void 0 ? 0 : valueIn(cell.idleBefore, "s"));
|
|
8141
|
+
const args = [
|
|
8142
|
+
lab.targetIp,
|
|
8143
|
+
String(lab.targetPort),
|
|
8144
|
+
"--mode",
|
|
8145
|
+
probe.mode
|
|
8146
|
+
];
|
|
8147
|
+
if (probe.ws !== void 0) {
|
|
8148
|
+
const { ws } = probe;
|
|
8149
|
+
const waits = (prediction.detects !== false ? prediction.reconnects?.waits ?? [] : []).reduce((sum, wait) => sum + valueIn(wait.high, "s"), 0);
|
|
8150
|
+
args.push("--websocket", ws.socket);
|
|
8151
|
+
args.push("--heartbeat", ws.heartbeat === null ? "off" : "on");
|
|
8152
|
+
if (ws.heartbeat !== null) args.push("--heartbeat-interval-ms", String(valueIn(ws.heartbeat.interval, "ms")), "--heartbeat-timeout-ms", String(valueIn(ws.heartbeat.timeout, "ms")), "--heartbeat-misses", String(ws.heartbeat.misses));
|
|
8153
|
+
args.push("--reconnect", ws.reconnect === null ? "off" : "on");
|
|
8154
|
+
if (ws.reconnect !== null) args.push("--reconnect-initial-ms", String(valueIn(ws.reconnect.initialDelay, "ms")), "--reconnect-multiplier", String(ws.reconnect.multiplier), "--reconnect-max-ms", String(valueIn(ws.reconnect.maxDelay, "ms")), "--reconnect-attempts", String(ws.reconnect.attempts), "--reconnect-jitter", String(ws.reconnect.jitter));
|
|
8155
|
+
args.push("--request-timeout-ms", String(valueIn(ws.requestTimeout, "ms")));
|
|
8156
|
+
args.push("--connection-timeout-ms", String(valueIn(ws.connectionTimeout, "ms")));
|
|
8157
|
+
if (ws.hold !== void 0) args.push("--hold-ms", String(valueIn(ws.hold, "ms")));
|
|
8158
|
+
args.push("--max-s", String(Math.ceil((longest + waits) * 2 + 10)));
|
|
8159
|
+
return args;
|
|
8160
|
+
}
|
|
8161
|
+
if (probe.http !== void 0) {
|
|
8162
|
+
const { http } = probe;
|
|
8163
|
+
const gap = http.gap === void 0 ? 0 : valueIn(http.gap, "s");
|
|
8164
|
+
args.push("--method", http.method);
|
|
8165
|
+
if (http.gap !== void 0) args.push("--gap-ms", String(valueIn(http.gap, "ms")));
|
|
8166
|
+
if (http.hold !== void 0) args.push("--hold-ms", String(valueIn(http.hold, "ms")));
|
|
8167
|
+
if (http.bytes !== void 0) args.push("--bytes", String(http.bytes));
|
|
8168
|
+
for (const [name, value] of Object.entries(http.options ?? {})) args.push(`--${name}`, String(value));
|
|
8169
|
+
args.push("--linger-ms", String(LINGER_MS));
|
|
8170
|
+
args.push("--max-s", String(Math.ceil(longest * 2 + gap + 10)));
|
|
8171
|
+
return args;
|
|
8172
|
+
}
|
|
8173
|
+
if (probe.request !== void 0) args.push("--request", probe.request);
|
|
8174
|
+
if (probe.count !== void 0) args.push("--count", String(probe.count));
|
|
8175
|
+
if (probe.interval !== void 0) args.push("--interval-ms", String(valueIn(probe.interval, "ms")));
|
|
8176
|
+
if (probe.driver?.name === "mongodb") {
|
|
8177
|
+
args.push("--set", probe.driver.set);
|
|
8178
|
+
if (probe.driver.op !== void 0) args.push("--op", probe.driver.op);
|
|
8179
|
+
if (cell.writeConcern !== void 0) args.push("--w", String(cell.writeConcern));
|
|
8180
|
+
if (cell.before === true) args.push("--before");
|
|
8181
|
+
if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
|
|
8182
|
+
args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
|
|
8183
|
+
return args;
|
|
8184
|
+
}
|
|
8185
|
+
if (probe.driver?.name === "pg" || probe.driver?.name === "hikari" || probe.driver?.name === "amqp") {
|
|
8186
|
+
if (probe.driver.name !== "amqp" && probe.driver.query !== void 0) args.push("--query", probe.driver.query);
|
|
8187
|
+
if (probe.driver.name !== "amqp" && probe.driver.inflight !== void 0) args.push("--inflight-query", probe.driver.inflight);
|
|
8188
|
+
if (probe.driver.name === "hikari" && probe.driver.burst !== void 0) args.push("--burst", String(probe.driver.burst.count), "--burst-gap-ms", String(Math.round(valueIn(probe.driver.burst.gap, "ms"))));
|
|
8189
|
+
if (cell.before === true) args.push("--before");
|
|
8190
|
+
if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
|
|
8191
|
+
args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
|
|
8192
|
+
return args;
|
|
8193
|
+
}
|
|
8194
|
+
if (probe.driver?.name === "grpc") {
|
|
8195
|
+
if (probe.driver.hold !== void 0) args.push("--hold-ms", String(valueIn(probe.driver.hold, "ms")));
|
|
8196
|
+
if (probe.driver.op !== void 0) args.push("--op", probe.driver.op);
|
|
8197
|
+
if (cell.before === true) args.push("--before");
|
|
8198
|
+
if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
|
|
8199
|
+
args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
|
|
8200
|
+
return args;
|
|
8201
|
+
}
|
|
8202
|
+
if (probe.driver !== void 0) {
|
|
8203
|
+
args.push("--keepalive-ms", String(valueIn(probe.driver.keepAlive, "ms")));
|
|
8204
|
+
args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
|
|
8205
|
+
return args;
|
|
8206
|
+
}
|
|
8207
|
+
if (probe.keepalive) args.push("--keepalive");
|
|
8208
|
+
if (probe.userTimeout !== void 0) args.push("--user-timeout-ms", String(valueIn(probe.userTimeout, "ms")));
|
|
8209
|
+
if (probe.plan !== void 0) args.push("--schedule", SCHEDULE_PATH);
|
|
8210
|
+
if (probe.streams !== void 0) args.push("--plan", LOAD_PATH);
|
|
8211
|
+
args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
|
|
8212
|
+
return args;
|
|
8213
|
+
};
|
|
8214
|
+
/**
|
|
8215
|
+
* This function waits for an error on the socket of the observer, or for its end of file
|
|
8216
|
+
* (CLOSE_WAIT, state 8). The prober records both. A cell whose observer is the server, or a proxy
|
|
8217
|
+
* of a topology, waits for that. A target with its own detector can also drop the connection
|
|
8218
|
+
* itself (`dropped`). The promise resolves after `within` ms at the latest.
|
|
8219
|
+
*/
|
|
8220
|
+
const observerError = (cell, partial, within, dropped = () => false) => new Promise((resolve) => {
|
|
8221
|
+
const started = Date.now();
|
|
8222
|
+
const look = () => {
|
|
8223
|
+
const run = partial();
|
|
8224
|
+
const at = socketOf(cell, run);
|
|
8225
|
+
if (at !== void 0 && run.kernel.some((record) => record.netns === at.netns && record.cookie === at.cookie && (record.ev === "error" && (record.err ?? 0) !== 0 || record.ev === "state" && record.new === 8)) || dropped() || Date.now() - started > within) resolve();
|
|
8226
|
+
else setTimeout(look, 50);
|
|
8227
|
+
};
|
|
8228
|
+
look();
|
|
8229
|
+
});
|
|
8230
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
8231
|
+
/**
|
|
8232
|
+
* This time is the pause between the client's first reply and the fault. It is more than Linux's
|
|
8233
|
+
* TCP_DELACK_MAX of 200 ms. Thus the delayed ACK of the reply is out before the fault.
|
|
8234
|
+
*/
|
|
8235
|
+
const QUIET_MS = 250;
|
|
8236
|
+
/**
|
|
8237
|
+
* This time is how long an HTTP client waits after its second request, before it exits. The exit
|
|
8238
|
+
* closes its sockets, thus that close comes after the window of the cell.
|
|
8239
|
+
*/
|
|
8240
|
+
const LINGER_MS = 1e3;
|
|
8241
|
+
/** These are the lines that end the workload of an HTTP client: the end of its second request, its deadline, or its failure. */
|
|
8242
|
+
const HTTP_DONE = [
|
|
8243
|
+
"done",
|
|
8244
|
+
"deadline",
|
|
6376
8245
|
"fail"
|
|
6377
8246
|
];
|
|
6378
8247
|
/**
|
|
@@ -6480,6 +8349,19 @@ const dials = (probe, count, within) => new Promise((resolve) => {
|
|
|
6480
8349
|
};
|
|
6481
8350
|
look();
|
|
6482
8351
|
});
|
|
8352
|
+
/** This time is the most that a run waits for the lines of the target after the end of its client. */
|
|
8353
|
+
const LINES_MS = 5e3;
|
|
8354
|
+
/**
|
|
8355
|
+
* This function waits for the lines of a run's counts. The lines of the target come on their own
|
|
8356
|
+
* stream, thus they can come after the last line of the client. A line above a count comes in the
|
|
8357
|
+
* pause after the wait, thus it is in the run that wrote it.
|
|
8358
|
+
*/
|
|
8359
|
+
const linesCounted = async (lab, from, counts) => {
|
|
8360
|
+
const started = Date.now();
|
|
8361
|
+
const short = () => counts.some((each) => countOf(lab.server.slice(from), each.match) < each.count);
|
|
8362
|
+
while (short() && Date.now() - started < LINES_MS) await sleep(20);
|
|
8363
|
+
await sleep(250);
|
|
8364
|
+
};
|
|
6483
8365
|
/** This function gives the netem specification of a cell: its latency, then the netem of its fault when it has one. */
|
|
6484
8366
|
const netemSpec = (cell, fault) => {
|
|
6485
8367
|
const parts = [...cell.latency === void 0 ? [] : [`delay ${valueIn(cell.latency, "ms")}ms`], ...fault === void 0 ? [] : [fault]];
|
|
@@ -6508,7 +8390,16 @@ const once = async (lab, cell, environment, index, dry = false) => {
|
|
|
6508
8390
|
const middleboxFrom = lab.middlebox.length;
|
|
6509
8391
|
const serverFrom = lab.server.length;
|
|
6510
8392
|
const lostBefore = droppedSoFar(lab.kernel);
|
|
6511
|
-
const
|
|
8393
|
+
const plan = cell.probe.plan === void 0 ? void 0 : schedule(cell.probe.plan.profile, cell.probe.plan.options);
|
|
8394
|
+
const load = cell.probe.streams === void 0 ? void 0 : loadPlanOf(cell.probe.streams);
|
|
8395
|
+
const files = [...plan === void 0 ? [] : [{
|
|
8396
|
+
content: JSON.stringify(plan),
|
|
8397
|
+
target: SCHEDULE_PATH
|
|
8398
|
+
}], ...load === void 0 ? [] : [{
|
|
8399
|
+
content: load.content,
|
|
8400
|
+
target: LOAD_PATH
|
|
8401
|
+
}]];
|
|
8402
|
+
const probe = await lab.probe(probeArgs(cell, lab), files.length === 0 ? void 0 : files);
|
|
6512
8403
|
const server = cell.observer === "server";
|
|
6513
8404
|
const peer = server ? {
|
|
6514
8405
|
freeze: () => probe.freeze(),
|
|
@@ -6521,7 +8412,11 @@ const once = async (lab, cell, environment, index, dry = false) => {
|
|
|
6521
8412
|
const last = probe.events.slice(-3).map((event) => JSON.stringify(event));
|
|
6522
8413
|
throw new Error(`${error.message}; the client's last lines: ${last.join(" ")}`);
|
|
6523
8414
|
});
|
|
6524
|
-
const placed =
|
|
8415
|
+
const placed = {
|
|
8416
|
+
...lab.topology === null ? {} : { topology: lab.topology },
|
|
8417
|
+
...plan === void 0 ? {} : { schedule: plan },
|
|
8418
|
+
...load === void 0 ? {} : { streams: load.streams }
|
|
8419
|
+
};
|
|
6525
8420
|
const fabricOf = () => {
|
|
6526
8421
|
if (lab.fabric === void 0) throw new Error(`${cell.name} silences a router of a lab with no fabric`);
|
|
6527
8422
|
return lab.fabric;
|
|
@@ -6666,7 +8561,7 @@ const once = async (lab, cell, environment, index, dry = false) => {
|
|
|
6666
8561
|
fault = await apply();
|
|
6667
8562
|
try {
|
|
6668
8563
|
if (cell.idleBefore !== void 0 && cell.probe.mode !== "idle") await sleep(valueIn(cell.idleBefore, "ms"));
|
|
6669
|
-
if ((cell.probe.mode === "inflight" || cell.probe.mode === "stream") && !replyInFlight) await go();
|
|
8564
|
+
if ((cell.probe.mode === "inflight" || cell.probe.mode === "stream" || cell.probe.mode === "schedule") && !replyInFlight) await go();
|
|
6670
8565
|
const { prediction } = cell;
|
|
6671
8566
|
const http = cell.probe.http !== void 0;
|
|
6672
8567
|
if (prediction.detects !== null && prediction.driver !== void 0) {
|
|
@@ -6680,6 +8575,7 @@ const once = async (lab, cell, environment, index, dry = false) => {
|
|
|
6680
8575
|
const gap = cell.probe.http?.gap === void 0 ? 0 : valueIn(cell.probe.http.gap, "s");
|
|
6681
8576
|
const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
|
|
6682
8577
|
await probe.until(HTTP_DONE, (gap + window) * 2e3 + 1e4).catch(() => void 0);
|
|
8578
|
+
if (prediction.detects === null && prediction.counts !== void 0) await linesCounted(lab, serverFrom, prediction.counts);
|
|
6683
8579
|
} else if (cell.fault === "stall") {
|
|
6684
8580
|
if (cell.stall === void 0) throw new Error(`${cell.name} says no stall's length`);
|
|
6685
8581
|
const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
|
|
@@ -6730,7 +8626,7 @@ const once = async (lab, cell, environment, index, dry = false) => {
|
|
|
6730
8626
|
await lift();
|
|
6731
8627
|
if (toxicFirst) await lab.toxic(null);
|
|
6732
8628
|
}
|
|
6733
|
-
if (cell.fault === "freeze-process" && !server && cell.probe.mode !== "idle") await probe.until(["data"], 5e3).catch(() => void 0);
|
|
8629
|
+
if (cell.fault === "freeze-process" && !server && cell.probe.mode !== "idle" && cell.probe.mode !== "schedule") await probe.until(["data"], 5e3).catch(() => void 0);
|
|
6734
8630
|
const expected = cell.prediction.detects !== false ? cell.prediction.reconnects : void 0;
|
|
6735
8631
|
if (expected !== void 0) {
|
|
6736
8632
|
const waits = expected.waits.reduce((sum, wait) => sum + valueIn(wait.high, "s"), 0);
|
|
@@ -6794,6 +8690,8 @@ const inEnvironment = async (cell, environment, count, say) => {
|
|
|
6794
8690
|
if (cell.group !== void 0) return inGroupEnvironment(cell, environment, count, say);
|
|
6795
8691
|
if (cell.replicaSet !== void 0) return inReplicaSetEnvironment(cell, environment, count, say);
|
|
6796
8692
|
if (cell.cluster !== void 0) return inClusterEnvironment(cell, environment, count, say);
|
|
8693
|
+
if (cell.kafka !== void 0) return inKafkaEnvironment(cell, environment, count, say);
|
|
8694
|
+
if (cell.pair !== void 0) return inPairEnvironment(cell, environment, count, say);
|
|
6797
8695
|
const lab = await openLab(cell.target, cell.client ?? cell.probe.driver?.name ?? "tcp", cell.topology);
|
|
6798
8696
|
try {
|
|
6799
8697
|
const settings = await lab.configure(cell.targetSettings ?? {});
|
|
@@ -6808,9 +8706,11 @@ const inEnvironment = async (cell, environment, count, say) => {
|
|
|
6808
8706
|
if (latency !== null) await lab.netem(latency, "target");
|
|
6809
8707
|
await lab.warm();
|
|
6810
8708
|
const manifest = await lab.manifest(await lab.sysctl(cell.sysctls), settings, targetSysctls);
|
|
8709
|
+
const missing = await lab.namespaceSetup(cell.namespaceSetup ?? []).then(() => lab.hostSysctl(cell.hostSysctls ?? [])).then(() => void 0, (error) => error instanceof Error ? error.message : String(error));
|
|
6811
8710
|
const timed = (run) => ({
|
|
6812
8711
|
...run,
|
|
6813
|
-
jitter: jitterOf(cell, run)
|
|
8712
|
+
jitter: jitterOf(cell, run),
|
|
8713
|
+
...missing === void 0 ? {} : { missing }
|
|
6814
8714
|
});
|
|
6815
8715
|
const warmup = {
|
|
6816
8716
|
...timed(await withSteal(() => once(lab, cell, environment, -1, endsEnvironment(cell)))),
|
|
@@ -6823,8 +8723,12 @@ const inEnvironment = async (cell, environment, count, say) => {
|
|
|
6823
8723
|
runs.push(run);
|
|
6824
8724
|
say(line(cell, run, measure(cell, run)));
|
|
6825
8725
|
}
|
|
8726
|
+
const host = await lab.restoreHost();
|
|
6826
8727
|
return {
|
|
6827
|
-
manifest
|
|
8728
|
+
manifest: host.length === 0 ? manifest : {
|
|
8729
|
+
...manifest,
|
|
8730
|
+
host
|
|
8731
|
+
},
|
|
6828
8732
|
runs,
|
|
6829
8733
|
warmup
|
|
6830
8734
|
};
|
|
@@ -6891,6 +8795,36 @@ const inReplicaSetEnvironment = async (cell, environment, count, say) => {
|
|
|
6891
8795
|
}
|
|
6892
8796
|
};
|
|
6893
8797
|
/**
|
|
8798
|
+
* This function does the part of the runs of a pair cell in one environment. The warm-up is a dry
|
|
8799
|
+
* run. The run promotes the standby, thus a pair cell has an environment for each run.
|
|
8800
|
+
*/
|
|
8801
|
+
const inPairEnvironment = async (cell, environment, count, say) => {
|
|
8802
|
+
const lab = await openLab(cell.target, cell.client ?? cell.probe.driver?.name ?? "tcp", cell.topology);
|
|
8803
|
+
try {
|
|
8804
|
+
await lab.warm();
|
|
8805
|
+
const manifest = await lab.manifest(await lab.sysctl(cell.sysctls));
|
|
8806
|
+
const standby = await openStandby(lab, cell);
|
|
8807
|
+
const warmup = {
|
|
8808
|
+
...await pairOnce(lab, standby, cell, environment, -1, true),
|
|
8809
|
+
warmup: true
|
|
8810
|
+
};
|
|
8811
|
+
say(line(cell, warmup, measure(cell, warmup)));
|
|
8812
|
+
const runs = [];
|
|
8813
|
+
for (let index = 0; index < count; index += 1) {
|
|
8814
|
+
const run = await pairOnce(lab, standby, cell, environment, index);
|
|
8815
|
+
runs.push(run);
|
|
8816
|
+
say(line(cell, run, measure(cell, run)));
|
|
8817
|
+
}
|
|
8818
|
+
return {
|
|
8819
|
+
manifest,
|
|
8820
|
+
runs,
|
|
8821
|
+
warmup
|
|
8822
|
+
};
|
|
8823
|
+
} finally {
|
|
8824
|
+
await lab.close();
|
|
8825
|
+
}
|
|
8826
|
+
};
|
|
8827
|
+
/**
|
|
6894
8828
|
* This function does the part of the runs of a cluster cell in one environment. The warm-up is a
|
|
6895
8829
|
* whole run with its fault, as for a replica set. The queue converges before each run.
|
|
6896
8830
|
*/
|
|
@@ -6919,6 +8853,35 @@ const inClusterEnvironment = async (cell, environment, count, say) => {
|
|
|
6919
8853
|
}
|
|
6920
8854
|
};
|
|
6921
8855
|
/**
|
|
8856
|
+
* This function does the part of the runs of a Kafka cell in one environment. The warm-up is a whole
|
|
8857
|
+
* run with its fault, as for a cluster. Each run makes its own topic.
|
|
8858
|
+
*/
|
|
8859
|
+
const inKafkaEnvironment = async (cell, environment, count, say) => {
|
|
8860
|
+
const lab = await openKafkaLab(cell.kafka);
|
|
8861
|
+
const once = cell.kafka.pause === void 0 ? kafkaOnce : kafkaPauseOnce;
|
|
8862
|
+
try {
|
|
8863
|
+
const manifest = await lab.manifest(await lab.sysctl(cell.sysctls));
|
|
8864
|
+
const warmup = {
|
|
8865
|
+
...await once(lab, cell, environment, -1),
|
|
8866
|
+
warmup: true
|
|
8867
|
+
};
|
|
8868
|
+
say(line(cell, warmup, measure(cell, warmup)));
|
|
8869
|
+
const runs = [];
|
|
8870
|
+
for (let index = 0; index < count; index += 1) {
|
|
8871
|
+
const run = await once(lab, cell, environment, index);
|
|
8872
|
+
runs.push(run);
|
|
8873
|
+
say(line(cell, run, measure(cell, run)));
|
|
8874
|
+
}
|
|
8875
|
+
return {
|
|
8876
|
+
manifest,
|
|
8877
|
+
runs,
|
|
8878
|
+
warmup
|
|
8879
|
+
};
|
|
8880
|
+
} finally {
|
|
8881
|
+
await lab.close();
|
|
8882
|
+
}
|
|
8883
|
+
};
|
|
8884
|
+
/**
|
|
6922
8885
|
* This function does the runs of a cell. They go across new environments, and each environment records
|
|
6923
8886
|
* its manifest. Then the verdict judges the runs. A median across the edge of the band extends
|
|
6924
8887
|
* the cell from 11 runs to 22, and then to 44, each extension in new environments. With a
|
|
@@ -6972,6 +8935,6 @@ const runCell = async (cell, say = () => {}) => {
|
|
|
6972
8935
|
}
|
|
6973
8936
|
};
|
|
6974
8937
|
//#endregion
|
|
6975
|
-
export { AWS_FROM, CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, DAEMONS, DROP_HANDLING, FABRIC_CLIENT, FABRIC_DEVICE, FABRIC_TARGET, FRR_IMAGE, GIVES_AWAY, HASH_SCRIPT, INJECT, KEPT, LINGER_MS, LISTING, LOCK_NAMES, LOSS_MARGIN, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, ROUTERS, SEED_DOCUMENT, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, basesOf, buildOf, buildRecordOf, callOf, cargoVersions, clockOffsetOf, closeOrExit, cloudOf, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, commitProblems, comparable, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, drillOnce, drillSpan, dropFinding, dropOf, endpointAfter, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, frrConf, goHttpImage, goVersions, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, harnessRecord, harnessVersion, hashOf, highestHeld, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isHopPlace, jarVersions, jitterOf, jitterOfLines, judge, lagsOf, laterMove, latestEndOf, leafLacks, leftAfter, locksOf, logNameOf, manifestProblems, matches, measure, measureDriver, measureGroup, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicAt, monotonicOf, nodeClientImage, nodeCommand, nodeHttpImage, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, npmVersions, onMonotonic, openClusterLab, openFabric, openGroupLab, openLab, openReplicaSetLab, pairingOf, pairingsOf, peersOf, pgClientImage, pingsOf, plainOf, planOf, programLines, programOf, quantileOf, readCpu, readHost, readOplog, reconnectFinding, reconnectOf, recoveryFindings, recoveryOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routeLine, routerImage, routesOf, runCell, seedScript, sentinelConfig, sessionsOf, shippedClientRecord, shippedSource, sliceQuery, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, standaloneCommand, statusViewOf, stealShare, stopScript, subjectClient, subjectProblems, subscriptionsOf, substrateHere, substrateKey, substrateOf, substrateOn, targetImageOf, targetSourceOf, templated, toolJsonOf, topologyOf, toxicBody, unlockedManifests, unpinnedBases, unpoolable, verdictOf, versionsOf, viewOf, writesOf };
|
|
8938
|
+
export { AWS_FROM, CLIENT_CAPABILITIES, CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, DAEMONS, DEBUG, DIRTY, DROP_HANDLING, FABRIC_CLIENT, FABRIC_DEVICE, FABRIC_TARGET, FRR_IMAGE, GIVES_AWAY, HASH_SCRIPT, INJECT, INSERT, KAFKA_TAIL_MS, KEPT, LINGER_MS, LISTING, LOAD_FORMAT, LOAD_PATH, LOCK_NAMES, LOSS_MARGIN, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, RESTART, ROUTERS, SCHEDULE_PATH, SEED_DOCUMENT, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STANDBY_READY, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, basesOf, bootstrapOf, brokerEnv, buildOf, buildRecordOf, callOf, cancelsOf, cargoVersions, clockOffsetOf, closeOrExit, cloudOf, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, commitProblems, comparable, controllerEnv, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, drillOnce, drillSpan, dropFinding, dropOf, endpointAfter, envOf, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, frrConf, goHttpImage, goVersions, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, harnessRecord, harnessVersion, hashOf, highestHeld, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isFenceLine, isHopPlace, jarVersions, jitterOf, jitterOfLines, judge, kafkaArgs, kafkaClientImage, kafkaCutOf, kafkaOnce, kafkaPauseOnce, kafkaViewOf, lagsOf, laterMove, latestEndOf, leafLacks, leftAfter, loadOf, loadPlanOf, loadVerdict, locksOf, logNameOf, logTimeOf, manifestProblems, matches, measure, measureConntrack, measureDriver, measureGroup, measurePause, measurePorts, measureQueue, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicAt, monotonicOf, namesGroup, nodeClientImage, nodeCommand, nodeHttpImage, nodeIdOf, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, npmVersions, numbersOf, onMonotonic, openClusterLab, openFabric, openGroupLab, openKafkaLab, openLab, openReplicaSetLab, openStandby, pairArgs, pairOnce, pairingOf, pairingsOf, pauseReady, peersOf, pgClientImage, pingsOf, plainOf, planOf, programLines, programOf, quantileOf, readCpu, readHost, readOplog, reconnectFinding, reconnectOf, recordProblems, recoveryFindings, recoveryOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routeLine, routerImage, routesOf, runCell, seedScript, sentinelConfig, sessionsOf, shippedClientRecord, shippedSource, sliceQuery, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, standaloneCommand, standbyCommand, statusViewOf, stealShare, stopScript, streamsLoadOf, subjectClient, subjectProblems, subscriptionsOf, substrateHere, substrateKey, substrateOf, substrateOn, targetImageOf, targetSourceOf, templated, toolJsonOf, topicOf, topologyOf, toxicBody, unlockedManifests, unpinnedBases, unpoolable, verdictOf, versionsOf, viewOf, writesOf };
|
|
6976
8939
|
|
|
6977
8940
|
//# sourceMappingURL=index.js.map
|