@mark1russell7/harness 0.14.3 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/build/index.js CHANGED
@@ -1,25 +1,251 @@
1
- import { A as unpinnedBases, C as routerImage, D as targetImageOf, E as subjectProblems, F as locksOf, I as npmVersions, L as versionsOf, M as cargoVersions, N as goVersions, O as targetSourceOf, P as jarVersions, S as programOf, T as subjectClient, _ as mongoClientImage, a as SHIPPED_NAMES, b as pgClientImage, c as buildRecordOf, d as harnessImage, f as harnessVersion, g as inContext, h as imageProblems, i as SHIPPED_CLIENTS, j as LOCK_NAMES, k as unlockedManifests, l as goHttpImage, m as imageOf, n as PINNED, o as amqpClientImage, p as hikariClientImage, r as REPOSITORY, s as basesOf, t as CLIENT_NAMES, u as grpcClientImage, v as nodeClientImage, w as shippedSource, x as planOf, y as nodeHttpImage } from "./image-BMdV7yWD.js";
1
+ import { A as unlockedManifests, B as isHopPlace, C as programOf, D as subjectProblems, E as subjectClient, F as jarVersions, H as requestFirst, I as locksOf, L as npmVersions, M as LOCK_NAMES, N as cargoVersions, O as targetImageOf, P as goVersions, R as versionsOf, S as planOf, T as shippedSource, V as latestEndOf, _ as kafkaClientImage, a as SHIPPED_NAMES, b as nodeHttpImage, c as buildRecordOf, d as harnessImage, f as harnessVersion, g as inContext, h as imageProblems, i as SHIPPED_CLIENTS, j as unpinnedBases, k as targetSourceOf, l as goHttpImage, m as imageOf, n as PINNED, o as amqpClientImage, p as hikariClientImage, r as REPOSITORY, s as basesOf, t as CLIENT_NAMES, u as grpcClientImage, v as mongoClientImage, w as routerImage, x as pgClientImage, y as nodeClientImage, z as CLIENT_CAPABILITIES } from "./image-BxVTFZ1o.js";
2
2
  import { quantity, valueIn } from "@mark1russell7/quantity";
3
- import { median, medianInterval as medianInterval$1, quantileSorted } from "@mark1russell7/rigor-statistics";
3
+ import { median, medianInterval as medianInterval$1, quantile, quantileSorted } from "@mark1russell7/rigor-statistics";
4
+ import { rungVerdict } from "@mark1russell7/rigor-bench";
5
+ import { measureLoad, schedule, scheduleHeld } from "@mark1russell7/rigor-chaos";
4
6
  import { createHash, randomBytes } from "node:crypto";
5
7
  import { execFile } from "node:child_process";
6
8
  import { mkdtemp, rm, writeFile } from "node:fs/promises";
7
9
  import { release, tmpdir } from "node:os";
8
10
  import { join } from "node:path";
9
11
  import { existsSync, readFileSync } from "node:fs";
10
- //#region src/cell.ts
11
- /** This function tells if a fault place is a hop of a topology. */
12
- const isHopPlace = (place) => typeof place === "object" && "hop" in place;
12
+ //#region src/load.ts
13
13
  /**
14
- * This function tells if the request of a client goes before the fault of its cell. That is so for a
15
- * hop event that comes while the request is in flight (`HopEvent.inFlight`). Then a request after the fault is void.
14
+ * The measure and the verdict of an open-loop cell (SYS-68). A client on a schedule sends each
15
+ * arrival at its time, thus a stall counts in each arrival that it delayed. The package rigor-chaos
16
+ * counts each latency from the planned time. The package rigor-bench judges a quantile by a
17
+ * distribution-free interval.
16
18
  */
17
- const requestFirst = (cell) => cell.fault === "hop-event" && cell.event?.inFlight === true;
19
+ /** This path is where the client of an open-loop cell finds its schedule. */
20
+ const SCHEDULE_PATH = "/tmp/schedule.json";
21
+ /** This function gives the time of a line of the client, in ms from the start of the schedule. */
22
+ const fromStart = (start, t) => (t - start) / 1e6;
18
23
  /**
19
- * This function gives the latest end of a band in seconds, from the reply of the client. It is
20
- * the high bound, after the latest reference of the band (`startsBy`).
24
+ * This function gives the measure of one run of a schedule from the lines of its client. It gives
25
+ * null when the client did not start the schedule. The first `sent` line of a client is its request
26
+ * before `ready`, and it names no arrival. The end of the run is its last line.
21
27
  */
22
- const latestEndOf = (prediction) => valueIn(prediction.high, "s") + (prediction.startsBy === void 0 ? 0 : valueIn(prediction.startsBy, "s"));
28
+ const loadOf = (events, plan) => {
29
+ const start = events.find((event) => event.ev === "scheduled")?.t;
30
+ if (start === void 0) return null;
31
+ return measureOf(events, start, plan, void 0);
32
+ };
33
+ /** This function tells if a status of a reply is a failure: a number outside 200 to 299. */
34
+ const failedStatus = (status) => typeof status === "number" && (status < 200 || status > 299);
35
+ /** This function gives the measure of the arrivals of one stream: the lines with the key `key`, or each line for no key. */
36
+ const measureOf = (events, start, plan, key) => {
37
+ const sent = /* @__PURE__ */ new Map();
38
+ const done = /* @__PURE__ */ new Map();
39
+ let failed = 0;
40
+ for (const event of events) {
41
+ const index = event["i"];
42
+ if (typeof index !== "number" || key !== void 0 && event["k"] !== key) continue;
43
+ if (event.ev === "sent") sent.set(index, fromStart(start, event.t));
44
+ if (event.ev === "replied") {
45
+ done.set(index, fromStart(start, event.t));
46
+ if (failedStatus(event["status"])) failed += 1;
47
+ }
48
+ }
49
+ const completions = [...sent].map(([index, at]) => ({
50
+ index,
51
+ sent: at,
52
+ done: done.get(index) ?? null
53
+ }));
54
+ const end = events.reduce((latest, event) => Math.max(latest, fromStart(start, event.t)), 0);
55
+ return {
56
+ ...measureLoad(plan, completions, end),
57
+ failed
58
+ };
59
+ };
60
+ /** This path is where the client `load` finds its plan. */
61
+ const LOAD_PATH = "/tmp/load.json";
62
+ /** This text is the format of the plan of the client `load`. */
63
+ const LOAD_FORMAT = "systems-load/1";
64
+ /**
65
+ * This function gives the schedule of each stream of a cell, and the text of the plan of the client
66
+ * `load`. It throws for no stream, for two streams with one key and for a stream with no request.
67
+ */
68
+ const loadPlanOf = (streams) => {
69
+ if (streams.length === 0) throw new RangeError("a plan of the client load has no stream");
70
+ const keys = /* @__PURE__ */ new Set();
71
+ for (const stream of streams) {
72
+ if (keys.has(stream.key)) throw new RangeError(`two streams have the key ${stream.key}`);
73
+ if (stream.requests.length === 0) throw new RangeError(`the stream ${stream.key} has no request`);
74
+ keys.add(stream.key);
75
+ }
76
+ const scheduled = streams.map((stream) => ({
77
+ key: stream.key,
78
+ schedule: schedule(stream.plan.profile, stream.plan.options),
79
+ requests: stream.requests
80
+ }));
81
+ return {
82
+ streams: scheduled.map(({ key, schedule: each }) => ({
83
+ key,
84
+ schedule: each
85
+ })),
86
+ content: JSON.stringify({
87
+ format: LOAD_FORMAT,
88
+ streams: scheduled
89
+ })
90
+ };
91
+ };
92
+ /**
93
+ * This function gives the measure of each stream of a run of the client `load`, and their pool. Each
94
+ * stream counts its arrivals from 0, thus the lines of a stream are the lines with its key. The pool
95
+ * sums the counts and joins the times, in the order of the streams. It gives null when the client did
96
+ * not start its schedule.
97
+ */
98
+ const streamsLoadOf = (events, streams) => {
99
+ const start = events.find((event) => event.ev === "scheduled")?.t;
100
+ if (start === void 0) return null;
101
+ const each = streams.map((stream) => [stream.key, measureOf(events, start, stream.schedule, stream.key)]);
102
+ const measures = each.map(([, measure]) => measure);
103
+ const sum = (count) => measures.reduce((total, measure) => total + count(measure), 0);
104
+ const join = (times) => measures.flatMap(times);
105
+ return {
106
+ pool: {
107
+ planned: sum((measure) => measure.planned),
108
+ unsent: sum((measure) => measure.unsent),
109
+ unfinished: sum((measure) => measure.unfinished),
110
+ failed: sum((measure) => measure.failed ?? 0),
111
+ latency: join((measure) => measure.latency),
112
+ service: join((measure) => measure.service),
113
+ lag: join((measure) => measure.lag),
114
+ censored: join((measure) => measure.censored)
115
+ },
116
+ streams: Object.fromEntries(each)
117
+ };
118
+ };
119
+ /** This function gives the spread of a list of times in ms, or null for no time. */
120
+ const spreadOf$1 = (values) => values.length === 0 ? null : {
121
+ p50: quantity(quantile(values, .5, "nearest-rank"), "ms"),
122
+ p99: quantity(quantile(values, .99, "nearest-rank"), "ms"),
123
+ max: quantity(values.reduce((most, value) => Math.max(most, value), 0), "ms")
124
+ };
125
+ /** This time is the date of the record of a band and the date of its verdict. The record has no age, because it comes from this run. */
126
+ const NOW = "2026-01-01T00:00:00.000Z";
127
+ /**
128
+ * This function judges one band of the latency over the pooled arrivals of the counted runs. The
129
+ * top of the band reads the interval of rigor-bench: the band holds only if the interval is under its top.
130
+ * The latencies of one run can follow each other, thus the width of the interval is an estimate.
131
+ * The bottom of the band reads the quantile itself. It gives the text of the verdict, and if it holds.
132
+ */
133
+ const bandVerdict = (band, values, censored) => {
134
+ const high = valueIn(band.high, "ms");
135
+ const rung = {
136
+ name: "band",
137
+ at: {},
138
+ budgets: [{
139
+ kind: "quantile",
140
+ measure: "latency",
141
+ unit: "ms",
142
+ quantile: band.quantile,
143
+ limit: high
144
+ }],
145
+ test: "the cell"
146
+ };
147
+ const top = rungVerdict(rung, {
148
+ rung: "band",
149
+ commit: "the cell",
150
+ at: NOW,
151
+ real: ["the counted runs"],
152
+ synthetic: [],
153
+ evidence: [{
154
+ kind: "quantile",
155
+ measure: "latency",
156
+ values,
157
+ censored
158
+ }]
159
+ }, { now: NOW });
160
+ const read = values.length + censored.length === 0 ? null : quantile([...values, ...censored], band.quantile, "nearest-rank");
161
+ const low = band.low === void 0 ? null : valueIn(band.low, "ms");
162
+ const above = low === null || read !== null && read >= low;
163
+ const place = low === null ? `under ${high} ms` : `in [${low}, ${high}] ms`;
164
+ const outcome = top.outcome === "within" && !above ? "under the bottom of the band" : top.outcome;
165
+ return {
166
+ holds: top.outcome === "within" && above,
167
+ words: `p${band.quantile * 100} of the latency ${read === null ? "has no value" : `is ${read} ms`}, wanted ${place}: ${outcome}. ${top.reason}`
168
+ };
169
+ };
170
+ /**
171
+ * This function judges the share of failed arrivals over the counted runs against a limit. Each
172
+ * planned arrival is a trial. An arrival fails if the client did not send it. It also fails with no
173
+ * reply at the end of its run, or with the status of a failure. The rate budget of rigor-bench
174
+ * reads the upper end of the Wilson interval at 0.95. Thus 381 arrivals with no failure hold a limit
175
+ * of 1 %, and fewer arrivals cannot decide.
176
+ */
177
+ const failuresVerdict = (limit, failures, trials) => {
178
+ const verdict = rungVerdict({
179
+ name: "failures",
180
+ at: {},
181
+ budgets: [{
182
+ kind: "rate",
183
+ measure: "failures",
184
+ limit
185
+ }],
186
+ test: "the cell"
187
+ }, {
188
+ rung: "failures",
189
+ commit: "the cell",
190
+ at: NOW,
191
+ real: ["the counted runs"],
192
+ synthetic: [],
193
+ evidence: [{
194
+ kind: "rate",
195
+ measure: "failures",
196
+ failures,
197
+ trials
198
+ }]
199
+ }, { now: NOW });
200
+ return {
201
+ holds: verdict.outcome === "within",
202
+ words: `${failures} of ${trials} arrivals failed, wanted a share under ${limit * 100} %: ${verdict.outcome}. ${verdict.reason}`
203
+ };
204
+ };
205
+ /**
206
+ * This function judges the load of an open-loop cell over its counted runs. A run that did not keep
207
+ * its schedule measures its driver, not the target, thus it is a finding of that run. Each
208
+ * band reads all the arrivals of the counted runs.
209
+ */
210
+ const loadVerdict = (prediction, runs) => {
211
+ const limit = valueIn(prediction.lag, "ms");
212
+ const outside = runs.flatMap((run) => {
213
+ if (run.load === void 0) return [{
214
+ run: run.name,
215
+ why: "the client never started its schedule"
216
+ }];
217
+ const check = scheduleHeld(run.load, limit);
218
+ return check.held ? [] : [{
219
+ run: run.name,
220
+ why: check.reason
221
+ }];
222
+ });
223
+ const measures = runs.flatMap((run) => run.load === void 0 ? [] : [run.load]);
224
+ const latency = measures.flatMap((measure) => measure.latency);
225
+ const censored = measures.flatMap((measure) => measure.censored);
226
+ const bands = prediction.bands.map((band) => bandVerdict(band, latency, censored));
227
+ const total = (count) => measures.reduce((sum, measure) => sum + count(measure), 0);
228
+ const planned = total((measure) => measure.planned);
229
+ const unsent = total((measure) => measure.unsent);
230
+ const unfinished = total((measure) => measure.unfinished);
231
+ const failed = total((measure) => measure.failed ?? 0);
232
+ const failures = prediction.failures === void 0 ? void 0 : failuresVerdict(prediction.failures, unsent + unfinished + failed, planned);
233
+ return {
234
+ holds: runs.length > 0 && outside.length === 0 && bands.every((band) => band.holds) && (failures?.holds ?? true),
235
+ outside,
236
+ load: {
237
+ planned,
238
+ unsent,
239
+ unfinished,
240
+ failed,
241
+ latency: spreadOf$1([...latency, ...censored]),
242
+ service: spreadOf$1(measures.flatMap((measure) => measure.service)),
243
+ lag: spreadOf$1(measures.flatMap((measure) => measure.lag.map((lag) => Math.abs(lag)))),
244
+ bands: bands.map((band) => band.words),
245
+ ...failures === void 0 ? {} : { failures: failures.words }
246
+ }
247
+ };
248
+ };
23
249
  //#endregion
24
250
  //#region src/verdict.ts
25
251
  /** This function gives the words of the outcome of a second request: its connection, a retry, and its end. */
@@ -288,6 +514,10 @@ const groupVerdictOf = (prediction, counted, delta = NOISE) => {
288
514
  const errors = /* @__PURE__ */ new Map();
289
515
  for (const run of counted) for (const error of run.group?.errors ?? []) errors.set(error.name, [...errors.get(error.name) ?? [], valueIn(error.first, "s")]);
290
516
  const lags = counted.flatMap((run) => run.group?.lags ?? []);
517
+ const copyCounts = counted.flatMap((run) => {
518
+ const copies = run.group?.writes.duplicates;
519
+ return copies === null || copies === void 0 ? [] : [copies.length];
520
+ });
291
521
  return {
292
522
  stages,
293
523
  medians,
@@ -310,7 +540,13 @@ const groupVerdictOf = (prediction, counted, delta = NOISE) => {
310
540
  runs: firsts.length,
311
541
  first: s(median(firsts.sort((a, b) => a - b)))
312
542
  })),
313
- ...lags.length === 0 ? {} : { lags: spreadOf(lags) }
543
+ ...lags.length === 0 ? {} : { lags: spreadOf(lags) },
544
+ ...prediction.group?.duplicates === void 0 ? {} : { duplicates: {
545
+ runs: copyCounts.length,
546
+ total: copyCounts.reduce((sum, n) => sum + n, 0),
547
+ median: medianCount(copyCounts),
548
+ max: copyCounts.length === 0 ? null : Math.max(...copyCounts)
549
+ } }
314
550
  };
315
551
  };
316
552
  /** This function gives the verdict of the band of a cell: its containment and the equivalence of its median. */
@@ -335,6 +571,17 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
335
571
  why: "no run counted"
336
572
  };
337
573
  if (prediction.detects === null) {
574
+ const law = prediction.queue !== void 0 ? "queue" : prediction.ports !== void 0 ? "port range" : prediction.pause !== void 0 ? "paused consumer" : prediction.conntrack !== void 0 ? "conntrack table" : void 0;
575
+ if (law !== void 0) {
576
+ const off = new Set(outside.map((each) => each.run)).size;
577
+ return {
578
+ ...empty,
579
+ decision: off === 0 ? "holds" : "fails",
580
+ band: null,
581
+ outside,
582
+ why: `${counted.length - off} of ${counted.length} runs hold the law of the ${law}; ${prediction.why}`
583
+ };
584
+ }
338
585
  const crossed = counted.flatMap((run) => run.crossings === void 0 ? [] : [run.crossings]);
339
586
  if (crossed.length > 0) {
340
587
  const hop = crossed[0].hop;
@@ -363,16 +610,30 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
363
610
  }
364
611
  const outcomes = [...kinds].map(([kind, n]) => `${n} ${kind}`).join(", ");
365
612
  const timed = times.length === 0 ? "no time" : `median ${s(median(times)).value} s from ${prediction.from}, range [${s(times[0]).value}, ${s(times.at(-1)).value}] s`;
366
- const counts = prediction.group?.lost;
613
+ const { counts } = prediction;
367
614
  if (counts !== void 0) {
615
+ const exact = counts.map((each) => `${each.what} ${each.count}`).join(", ");
616
+ return {
617
+ ...empty,
618
+ decision: outside.length === 0 ? "holds" : "fails",
619
+ band: null,
620
+ outside,
621
+ median: times.length === 0 ? null : s(median(times)),
622
+ why: `${counted.length - outside.length} of ${counted.length} runs gave the exact counts (${exact}); ${outcomes}; ${timed}. ${prediction.why}`
623
+ };
624
+ }
625
+ const lost = prediction.group?.lost;
626
+ const copies = prediction.group?.duplicates;
627
+ if (lost !== void 0 || copies !== void 0) {
368
628
  const off = new Set(outside.map((each) => each.run)).size;
629
+ const bands = [...lost === void 0 ? [] : [`lost writes [${lost.low}, ${lost.high}]`], ...copies === void 0 ? [] : [`extra copies [${copies.low}, ${copies.high}]`]];
369
630
  return {
370
631
  ...empty,
371
632
  decision: off === 0 ? "holds" : "fails",
372
633
  band: null,
373
634
  outside,
374
635
  median: times.length === 0 ? null : s(median(times)),
375
- why: `${counted.length - off} of ${counted.length} runs inside the band of lost writes [${counts.low}, ${counts.high}]; ${outcomes}; ${timed}. ${prediction.why}`
636
+ why: `${counted.length - off} of ${counted.length} runs inside the band of ${bands.join(" and the band of ")}; ${outcomes}; ${timed}. ${prediction.why}`
376
637
  };
377
638
  }
378
639
  const missed = outside.filter((each) => counted.some((run) => name(run) === each.run && run.reconnect !== void 0));
@@ -386,6 +647,33 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
386
647
  why: `${counted.length} runs measured: ${outcomes}; ${timed}.${late} ${prediction.why}`
387
648
  };
388
649
  }
650
+ if (counted.some((run) => run.load !== void 0)) {
651
+ const judged = loadVerdict(prediction.load ?? {
652
+ lag: quantity(Number.MAX_SAFE_INTEGER, "ms"),
653
+ bands: []
654
+ }, counted.map((run) => ({
655
+ name: name(run),
656
+ load: run.load
657
+ })));
658
+ const middleOfLoad = judged.load.latency?.p50;
659
+ const failed = judged.load.failed === 0 ? "" : `, ${judged.load.failed} with the status of a failure`;
660
+ const read = [
661
+ `${counted.length} runs on a schedule, ${judged.load.planned} arrivals, ${judged.load.unsent} not sent, ${judged.load.unfinished} with no reply${failed}.`,
662
+ ...judged.load.bands,
663
+ ...judged.load.failures === void 0 ? [] : [judged.load.failures],
664
+ prediction.why
665
+ ].join(" ");
666
+ return {
667
+ ...empty,
668
+ decision: prediction.load === void 0 ? "measured" : judged.holds ? "holds" : "fails",
669
+ band: null,
670
+ outside: judged.outside,
671
+ median: middleOfLoad === void 0 ? null : s(valueIn(middleOfLoad, "s")),
672
+ ...judged.load.latency === null ? {} : { slowest: s(valueIn(judged.load.latency.max, "s")) },
673
+ load: judged.load,
674
+ why: read
675
+ };
676
+ }
389
677
  const samples = counted.flatMap((run) => (run.samples ?? []).map((sample) => valueIn(sample, "s"))).sort((a, b) => a - b);
390
678
  const retransmissions = counted.reduce((sum, run) => sum + run.retransmissions, 0);
391
679
  const ended = counted.filter((run) => run.outcome.kind === "error" || run.outcome.kind === "eof");
@@ -529,7 +817,7 @@ const DROP_HANDLING = quantity(1.5, "ms");
529
817
  /** This function gives the window of the act of a drop, in seconds: `DROP_HANDLING`, or the pairing window (`delta`) when it is longer. */
530
818
  const dropWindow = (delta) => Math.max(valueIn(DROP_HANDLING, "s"), valueIn(delta, "s"));
531
819
  /** This function gives the interval between two instants on the one clock, in seconds to the microsecond. */
532
- const between$3 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
820
+ const between$4 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
533
821
  /** This function gives the state of a state record after the change, and before it. TCP_ESTABLISHED is 1, TCP_SYN_SENT 2. */
534
822
  const newState$2 = (record) => record.new;
535
823
  const oldState$2 = (record) => record.old;
@@ -571,7 +859,7 @@ const eventOf = (socket, line, matches, delta = NOISE) => {
571
859
  */
572
860
  const leftAfter = (socket, line) => {
573
861
  const left = socket.find((record) => record.ev === "state" && oldState$2(record) === 1 && (newState$2(record) === 4 || newState$2(record) === 7) && record.mono >= line.t);
574
- return left === void 0 ? null : between$3(line.t, left.mono);
862
+ return left === void 0 ? null : between$4(line.t, left.mono);
575
863
  };
576
864
  /**
577
865
  * This function gives the end of the client process in a run, as the kernel stamps it. The process
@@ -663,19 +951,19 @@ const reconnectOf = (run, detected, cookie, from = detected) => {
663
951
  const waits = starts.flatMap((start, at) => {
664
952
  const next = starts[at + 1]?.t ?? Number.POSITIVE_INFINITY;
665
953
  const dial = dials.find((each) => each.t >= start.t && each.t < next);
666
- return dial === void 0 ? [] : [between$3(start.t, dial.t)];
954
+ return dial === void 0 ? [] : [between$4(start.t, dial.t)];
667
955
  });
668
956
  const fresh = run.kernel.filter((record) => record.netns === run.netns && record.cookie !== cookie && record.ev === "state" && record.mono > detected);
669
957
  const syns = fresh.filter((record) => newState$2(record) === 2);
670
958
  const established = fresh.find((record) => newState$2(record) === 1);
671
959
  const connects = dials.flatMap((dial) => {
672
960
  const syn = syns.find((record) => record.mono >= dial.t);
673
- return syn === void 0 ? [] : [between$3(dial.t, syn.mono)];
961
+ return syn === void 0 ? [] : [between$4(dial.t, syn.mono)];
674
962
  });
675
963
  return {
676
964
  waits,
677
- syn: syns[0] === void 0 ? null : between$3(detected, syns[0].mono),
678
- established: established === void 0 ? null : between$3(detected, established.mono),
965
+ syn: syns[0] === void 0 ? null : between$4(detected, syns[0].mono),
966
+ established: established === void 0 ? null : between$4(detected, established.mono),
679
967
  connects
680
968
  };
681
969
  };
@@ -702,7 +990,7 @@ const callOf = (events, request, detected) => {
702
990
  };
703
991
  return {
704
992
  outcome: end.ev === "data" ? "a reply" : String(end["message"] ?? end["code"] ?? "an error"),
705
- after: detected === void 0 ? null : between$3(detected, end.t)
993
+ after: detected === void 0 ? null : between$4(detected, end.t)
706
994
  };
707
995
  };
708
996
  //#endregion
@@ -717,12 +1005,22 @@ const ERRNO$1 = {
717
1005
  111: "ECONNREFUSED",
718
1006
  113: "EHOSTUNREACH"
719
1007
  };
720
- const between$2 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
1008
+ const between$3 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
721
1009
  const round = (seconds) => Math.round(seconds * 1e6) / 1e6;
722
1010
  /** This function gives the value of a field of a line by its name. A dotted name is a field in a field. */
723
1011
  const fieldOf = (line, path) => path.split(".").reduce((value, key) => value !== null && typeof value === "object" ? value[key] : void 0, line);
724
1012
  /** This function tells if a line of the client matches a line match: its `ev`, and each field that the match names. */
725
1013
  const matches = (line, match) => line.ev === match.ev && Object.entries(match.where ?? {}).every(([path, value]) => fieldOf(line, path) === value);
1014
+ /** This function gives the count of the lines that match. */
1015
+ const countOf = (lines, match) => lines.filter((line) => matches(line, match)).length;
1016
+ /** This function gives the finding of the line counts of a run, or null when each count is the exact one. */
1017
+ const countFinding = (counts, lines) => {
1018
+ const off = counts.flatMap((each) => {
1019
+ const seen = countOf(lines, each.match);
1020
+ return seen === each.count ? [] : [`${each.what}: ${seen} lines, and the law gives ${each.count}`];
1021
+ });
1022
+ return off.length === 0 ? null : off.join("; ");
1023
+ };
726
1024
  /** This function says a line match in words, for a finding. */
727
1025
  const wordsOf = (match) => {
728
1026
  const where = Object.entries(match.where ?? {}).map(([path, value]) => `${path} ${String(value)}`);
@@ -1025,7 +1323,7 @@ const measureDriver = (cell, run) => {
1025
1323
  const until = prediction.detects ? Infinity : after + valueIn(prediction.window, "s") * 1e9;
1026
1324
  for (const match of driver.quiet ?? []) {
1027
1325
  const line = events.find((event) => event.t > after && event.t <= until && matches(event, match));
1028
- if (line !== void 0) findings.push(`${wordsOf(match)} came ${between$2(after, line.t).value} s after the fault, where none should`);
1326
+ if (line !== void 0) findings.push(`${wordsOf(match)} came ${between$3(after, line.t).value} s after the fault, where none should`);
1029
1327
  }
1030
1328
  const expected = prediction.detects ? void 0 : prediction.middlebox?.forgets;
1031
1329
  if (!prediction.detects && (run.middlebox !== void 0 || expected !== void 0)) {
@@ -1035,14 +1333,14 @@ const measureDriver = (cell, run) => {
1035
1333
  for (const socket of socketsOf(events)) {
1036
1334
  const forgot = middlebox.find((line) => line.ev === "ct" && line["type"] === "DESTROY" && String(line["orig"] ?? "").startsWith(`${socket.local}>`) && line.t > socket.opened && (socket.ended === null || line.t < socket.ended) && line.t <= until);
1037
1335
  if (expected === void 0 || socket.local !== own) {
1038
- if (forgot !== void 0) findings.push(`a router forgot the flow of the ${socket.role} socket ${between$2(after, forgot.t).value} s after the fault, where it should keep it`);
1336
+ if (forgot !== void 0) findings.push(`a router forgot the flow of the ${socket.role} socket ${between$3(after, forgot.t).value} s after the fault, where it should keep it`);
1039
1337
  continue;
1040
1338
  }
1041
1339
  const last = referenceOf(run, "last-received", driver.socket, false);
1042
1340
  if (forgot === void 0) findings.push(`no router forgot the flow of the ${socket.role} socket in the window, where it should forget it`);
1043
1341
  else if (last === void 0) findings.push(`the ${socket.role} socket got no segment before the fault`);
1044
1342
  else {
1045
- const at = between$2(last, forgot.t).value;
1343
+ const at = between$3(last, forgot.t).value;
1046
1344
  const [low, high] = [round(valueIn(expected.low, "s") - noise), round(valueIn(expected.high, "s") + noise)];
1047
1345
  if (at < low || at > high) findings.push(`a router forgot the flow of the ${socket.role} socket ${at} s after its last segment, outside [${low}, ${high}] s`);
1048
1346
  }
@@ -1059,7 +1357,7 @@ const measureDriver = (cell, run) => {
1059
1357
  elapsed: null
1060
1358
  };
1061
1359
  }
1062
- const elapsed = between$2(from, line.t);
1360
+ const elapsed = between$3(from, line.t);
1063
1361
  const name = nameOf$1(line, step.ends);
1064
1362
  if (step.name !== "OK" && name !== step.name) findings.push(`${step.what}: it ended with ${name}, not ${step.name}`);
1065
1363
  const [low, high] = [round(valueIn(step.low, "s") - noise), round(valueIn(step.high, "s") + noise)];
@@ -1093,7 +1391,7 @@ const measureDriver = (cell, run) => {
1093
1391
  ...nothing,
1094
1392
  ...stepped,
1095
1393
  outcome: { kind: "answered" },
1096
- elapsed: between$2(from, line.t),
1394
+ elapsed: between$3(from, line.t),
1097
1395
  void: voided,
1098
1396
  finding: findings[0] ?? null
1099
1397
  };
@@ -1136,15 +1434,545 @@ const measureDriver = (cell, run) => {
1136
1434
  ...base,
1137
1435
  ...stepped,
1138
1436
  outcome,
1139
- elapsed: between$2(reference, ended),
1140
- socket: seen === void 0 ? null : between$2(reference, seen.t),
1141
- noticed: report === void 0 || seen === void 0 ? null : between$2(report.at, seen.t),
1142
- steps: (prediction.from === "last-received" ? records.filter((record) => record.ev === "probe" && record.what === "keepalive" && record.mono > reference) : []).map((record) => between$2(reference, record.mono)),
1437
+ elapsed: between$3(reference, ended),
1438
+ socket: seen === void 0 ? null : between$3(reference, seen.t),
1439
+ noticed: report === void 0 || seen === void 0 ? null : between$3(report.at, seen.t),
1440
+ steps: (prediction.from === "last-received" ? records.filter((record) => record.ev === "probe" && record.what === "keepalive" && record.mono > reference) : []).map((record) => between$3(reference, record.mono)),
1143
1441
  void: voided,
1144
1442
  finding: findings[0] ?? null
1145
1443
  };
1146
1444
  };
1147
1445
  //#endregion
1446
+ //#region src/queue.ts
1447
+ /** This bit is the SYN of the flags of an `rx` record of the prober. */
1448
+ const SYN = 2;
1449
+ /** This function gives the time between two stamps of CLOCK_MONOTONIC. */
1450
+ const between$2 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
1451
+ const numberOf$1 = (line, field) => {
1452
+ const value = line[field];
1453
+ return typeof value === "number" ? value : 0;
1454
+ };
1455
+ /** This function says if a time is inside a band. */
1456
+ const inside = (time, band) => {
1457
+ const seconds = valueIn(time, "s");
1458
+ return seconds >= valueIn(band.low, "s") && seconds <= valueIn(band.high, "s");
1459
+ };
1460
+ /** This function gives the words of a band. */
1461
+ const bandWords = (band) => `[${valueIn(band.low, "s")}, ${valueIn(band.high, "s")}] s`;
1462
+ const waitsOf = (from, stamps) => stamps.map((stamp, at) => between$2(at === 0 ? from : stamps[at - 1], stamp));
1463
+ /** This function finds the records of the sockets of the client that had this local address. */
1464
+ const socketsAt = (run, local) => {
1465
+ const cookies = new Set(run.kernel.filter((record) => record.netns === run.netns && record.src === local).map((record) => record.cookie));
1466
+ return run.kernel.filter((record) => record.netns === run.netns && cookies.has(record.cookie));
1467
+ };
1468
+ const readConnection = (name, run, expected) => {
1469
+ const lineOf = (ev) => run.events.find((event) => event.ev === ev && event["name"] === name);
1470
+ const dial = lineOf("dial");
1471
+ const connected = lineOf("connected");
1472
+ const failed = lineOf("failed");
1473
+ const state = lineOf("state");
1474
+ const socket = dial === void 0 ? [] : socketsAt(run, dial["local"]);
1475
+ const sent = socket.filter((record) => record.ev === "retrans").map((record) => record.mono);
1476
+ const rx = socket.filter((record) => record.ev === "rx" && ((record.seg?.flags ?? 0) & SYN) !== 0);
1477
+ const from = dial?.t ?? 0;
1478
+ const start = expected?.ends === null || expected?.ends === void 0 ? void 0 : lineOf(expected.ends.from);
1479
+ const last = sent.at(-1);
1480
+ return {
1481
+ name,
1482
+ connected: connected !== void 0,
1483
+ syns: waitsOf(from, connected === void 0 ? sent : sent.filter((stamp) => stamp < connected.t)),
1484
+ synAcks: waitsOf(from, rx.map((record) => record.mono)),
1485
+ retransmissions: sent.length,
1486
+ errno: failed === void 0 ? null : String(failed["errno"]),
1487
+ ended: failed === void 0 || start === void 0 ? null : between$2(start.t, failed.t),
1488
+ afterLast: failed === void 0 || last === void 0 ? null : between$2(last, failed.t),
1489
+ unacked: state === void 0 ? null : numberOf$1(state, "unacked")
1490
+ };
1491
+ };
1492
+ /** This function reads one run against the band of each wait in a list, and gives the words of each miss. */
1493
+ const waitFindings = (what, seen, bands) => {
1494
+ if (seen.length !== bands.length) return [`${what}: ${seen.length} waits, not ${bands.length}`];
1495
+ return seen.flatMap((wait, at) => inside(wait, bands[at]) ? [] : [`${what}: wait ${at} of ${valueIn(wait, "s")} s, outside ${bandWords(bands[at])}`]);
1496
+ };
1497
+ const connectionFindings = (expected, read, expiredAt, stamps) => {
1498
+ const { name, ends } = expected;
1499
+ const found = [];
1500
+ if (read.connected !== expected.connects) found.push(`${name}: ${read.connected ? "connected" : "did not connect"}, against the prediction`);
1501
+ if (expected.syns !== void 0) found.push(...waitFindings(`${name} SYN`, read.syns, expected.syns));
1502
+ if (expected.synAcks !== void 0) found.push(...waitFindings(`${name} SYN-ACK`, read.synAcks, expected.synAcks));
1503
+ if (expected.unacked !== void 0 && read.unacked !== expected.unacked) found.push(`${name}: ${read.unacked ?? "no"} segments without an ACK, not ${expected.unacked}`);
1504
+ if (ends === null) {
1505
+ if (read.errno !== null) found.push(`${name}: ended with ${read.errno}, and the prediction has no end`);
1506
+ return found;
1507
+ }
1508
+ if (read.errno !== ends.errno) found.push(`${name}: ended with ${read.errno ?? "nothing"}, not ${ends.errno}`);
1509
+ if (read.ended === null) return [...found, `${name}: no time for its end`];
1510
+ if (!inside(read.ended, ends.within)) found.push(`${name}: its end came at ${valueIn(read.ended, "s")} s, outside ${bandWords(ends.within)}`);
1511
+ if (ends.retransmitted === void 0) return found;
1512
+ if (read.afterLast === null || !inside(read.afterLast, ends.retransmitted)) found.push(`${name}: its end came ${read.afterLast === null ? "with no retransmission" : `${valueIn(read.afterLast, "s")} s after its last retransmission`}, outside ${bandWords(ends.retransmitted)}`);
1513
+ if (expiredAt === null) return found;
1514
+ const before = stamps.at(-2);
1515
+ const last = stamps.at(-1);
1516
+ if (last === void 0 || last < expiredAt || before !== void 0 && before > expiredAt) found.push(`${name}: its last two retransmissions do not span the expiry`);
1517
+ return found;
1518
+ };
1519
+ /**
1520
+ * This function reads one run of a queue cell. The client lines give the connections and the
1521
+ * counters at the two ends of the run. The target lines give the count of the queue. The records of
1522
+ * the prober give the waits of each socket.
1523
+ */
1524
+ const measureQueue = (run, expected) => {
1525
+ const { events, kernel } = run;
1526
+ const lineOf = (ev) => events.find((event) => event.ev === ev);
1527
+ const base = {
1528
+ environment: run.environment,
1529
+ index: run.index,
1530
+ outcome: { kind: "answered" },
1531
+ elapsed: null,
1532
+ socket: null,
1533
+ noticed: null,
1534
+ steps: [],
1535
+ probes: 0,
1536
+ retransmissions: kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length,
1537
+ acks: 0
1538
+ };
1539
+ const voided = (why) => ({
1540
+ ...base,
1541
+ void: why,
1542
+ finding: null
1543
+ });
1544
+ const opened = lineOf("opened");
1545
+ const stats = lineOf("stats");
1546
+ if (opened === void 0) return voided("the target did not open its listener");
1547
+ if (stats === void 0 || lineOf("done") === void 0) return voided("the client did not reach its end");
1548
+ if (run.lost === null) return voided("the prober wrote no clock line after the run, so nothing says what it dropped");
1549
+ if (run.lost > 0) return voided(`the prober dropped ${run.lost} records`);
1550
+ const lines = (run.server ?? []).filter((line) => line.ev === "queue" && line.t >= opened.t);
1551
+ if (lines.length === 0) return voided("the target wrote no queue line in the run");
1552
+ const named = [...new Set(events.filter((event) => event.ev === "dial").map((event) => String(event["name"])))];
1553
+ const predicted = new Map(expected.connections.map((each) => [each.name, each]));
1554
+ const connections = named.map((name) => readConnection(name, run, predicted.get(name)));
1555
+ const admitted = expected.admitted === void 0 ? void 0 : connections.find((each) => each.name === expected.admitted);
1556
+ if (expected.admitted !== void 0 && (admitted === void 0 || !admitted.connected || admitted.unacked !== 0)) return voided(`${expected.admitted} did not take the last place in the queue: the burst did not land in its sequence`);
1557
+ const change = (field) => numberOf$1(stats, field) - numberOf$1(opened, field);
1558
+ const burst = events.find((event) => event.ev === "dial" && event["name"] === "in")?.t ?? opened.t;
1559
+ const counted = expected.expiry?.timeouts;
1560
+ const expired = counted === void 0 ? void 0 : lines.find((line) => numberOf$1(line, "timeouts") - numberOf$1(opened, "timeouts") >= counted);
1561
+ const accepted = events.find((event) => event.ev === "accepted");
1562
+ const rise = expected.rise?.count ?? 0;
1563
+ const low = accepted === void 0 ? -1 : lines.findIndex((line) => line.t >= accepted.t && numberOf$1(line, "count") < rise);
1564
+ const risen = low < 0 ? void 0 : lines.slice(low).find((line) => numberOf$1(line, "count") >= rise);
1565
+ const drained = events.filter((event) => event.ev === "accepted").reduce((sum, event) => sum + (Array.isArray(event["peers"]) ? event["peers"].length : 0), 0);
1566
+ const queue = {
1567
+ max: numberOf$1(opened, "max"),
1568
+ held: Math.max(...lines.map((line) => numberOf$1(line, "count"))),
1569
+ connected: connections.filter((each) => each.connected).length,
1570
+ overflows: change("overflows"),
1571
+ drops: change("drops"),
1572
+ cookies: change("cookies"),
1573
+ timeouts: change("timeouts"),
1574
+ resets: change("resets"),
1575
+ drained,
1576
+ expiry: expired === void 0 ? null : between$2(burst, expired.t),
1577
+ rise: risen === void 0 ? null : between$2(burst, risen.t),
1578
+ connections
1579
+ };
1580
+ const found = [];
1581
+ if (queue.max !== expected.max) found.push(`the limit is ${queue.max}, not ${expected.max}`);
1582
+ if (queue.held !== expected.held) found.push(`the queue held ${queue.held}, not ${expected.held}`);
1583
+ if (queue.connected !== expected.connected) found.push(`${queue.connected} connected, not ${expected.connected}`);
1584
+ if (queue.overflows < expected.overflows.low || queue.overflows > expected.overflows.high) found.push(`${queue.overflows} overflows, outside [${expected.overflows.low}, ${expected.overflows.high}]`);
1585
+ if (queue.resets !== expected.resets) found.push(`${queue.resets} resets, not ${expected.resets}`);
1586
+ if (queue.cookies !== 0) found.push(`${queue.cookies} SYN cookies, and the law has none`);
1587
+ if (expected.drained !== void 0 && queue.drained !== expected.drained) found.push(`${queue.drained} accepted, not ${expected.drained}`);
1588
+ if (expected.expiry !== void 0 && (queue.expiry === null || !inside(queue.expiry, expected.expiry.within))) found.push(`the expiry came at ${queue.expiry === null ? "no time" : `${valueIn(queue.expiry, "s")} s`}, outside ${bandWords(expected.expiry.within)}`);
1589
+ if (expected.rise !== void 0 && (queue.rise === null || !inside(queue.rise, expected.rise.within))) found.push(`the count got back to ${expected.rise.count} at ${queue.rise === null ? "no time" : `${valueIn(queue.rise, "s")} s`}, outside ${bandWords(expected.rise.within)}`);
1590
+ for (const each of expected.connections) {
1591
+ const read = connections.find((connection) => connection.name === each.name);
1592
+ if (read === void 0) {
1593
+ found.push(`${each.name}: the client did not dial it`);
1594
+ continue;
1595
+ }
1596
+ const dial = events.find((event) => event.ev === "dial" && event["name"] === each.name);
1597
+ const stamps = socketsAt(run, dial?.["local"]).filter((record) => record.ev === "retrans").map((record) => record.mono);
1598
+ found.push(...connectionFindings(each, read, expired?.t ?? null, stamps));
1599
+ }
1600
+ return {
1601
+ ...base,
1602
+ void: null,
1603
+ finding: found.length === 0 ? null : found.join(". "),
1604
+ queue
1605
+ };
1606
+ };
1607
+ //#endregion
1608
+ //#region src/conntrack.ts
1609
+ /** This errno tells that a connect got a slot. */
1610
+ const NONE = "none";
1611
+ const numberOr = (line, field) => {
1612
+ const value = line?.[field];
1613
+ return typeof value === "number" ? value : null;
1614
+ };
1615
+ /** This function gives the time from one stamp to a stamp of a field of a line, when the line has it. */
1616
+ const untilField = (from, line, field) => {
1617
+ const stamp = numberOr(line, field);
1618
+ return from === void 0 || stamp === null ? null : between$2(from.t, stamp);
1619
+ };
1620
+ const stallOf = (line) => line === void 0 ? null : {
1621
+ errno: String(line["errno"]),
1622
+ time: quantity(Math.round((numberOr(line, "elapsed") ?? 0) / 1e3) / 1e6, "s"),
1623
+ drops: numberOr(line, "drops") ?? 0,
1624
+ retransFail: numberOr(line, "retrans_fail") ?? 0
1625
+ };
1626
+ /** This function tells where a reading differs from the law of its cell. */
1627
+ const findingsOf$1 = (read, failed, expected) => {
1628
+ const found = [];
1629
+ const { max, neighbor, refused, drained, pressure } = expected;
1630
+ if (typeof failed === "string") found.push(`a connect failed with ${failed} before the client held its flows`);
1631
+ if (read.flows !== max) found.push(`the client held ${read.flows} flows, not ${max}`);
1632
+ if (read.count !== max) found.push(`the count read ${read.count} at the fill, not ${max}`);
1633
+ if (neighbor !== void 0 && read.neighbor !== neighbor) found.push(`the namespace of the target read ${read.neighbor ?? "nothing"}, not ${neighbor}`);
1634
+ if (refused !== void 0) {
1635
+ const stall = read.stall;
1636
+ if (stall === null) found.push("no connect after the fill ended");
1637
+ else {
1638
+ if (stall.errno !== refused.errno) found.push(`the connect after the fill ended with ${stall.errno}, not ${refused.errno}`);
1639
+ if (!inside(stall.time, refused.within)) found.push(`the connect after the fill took ${valueIn(stall.time, "s")} s, outside ${bandWords(refused.within)}`);
1640
+ if (stall.drops !== refused.drops) found.push(`the field drop rose by ${stall.drops}, not ${refused.drops}`);
1641
+ if (stall.retransFail !== refused.retransFail) found.push(`TCPRetransFail rose by ${stall.retransFail}, not ${refused.retransFail}`);
1642
+ }
1643
+ }
1644
+ if (drained !== void 0) {
1645
+ for (const [name, time] of [["first fall", read.firstFall], ["zero", read.zero]]) if (time === null || !inside(time, drained)) found.push(`the ${name} of the count came at ${time === null ? "no time" : `${valueIn(time, "s")} s`}, outside ${bandWords(drained)}`);
1646
+ }
1647
+ if (pressure !== void 0) {
1648
+ const stall = read.stall;
1649
+ if (read.closed !== max) found.push(`the count read ${read.closed ?? "nothing"} after each side closed, not ${max}`);
1650
+ const { highest } = pressure;
1651
+ if (highest === void 0) {
1652
+ if (read.lowest !== pressure.lowest) found.push(`the lowest count was ${read.lowest ?? "nothing"}, not ${pressure.lowest}`);
1653
+ if (read.after !== pressure.after) found.push(`the count read ${read.after ?? "nothing"} after the connect, not ${pressure.after}`);
1654
+ } else if (read.lowest === null || read.lowest < pressure.lowest || read.lowest > highest) found.push(`the lowest count was ${read.lowest ?? "nothing"}, outside ${pressure.lowest} to ${highest}`);
1655
+ else if (read.after !== read.lowest + 1) found.push(`the count read ${read.after ?? "nothing"} after the connect, not ${read.lowest + 1}`);
1656
+ if (stall === null) found.push("no connect after the close ended");
1657
+ else {
1658
+ if (stall.errno !== NONE) found.push(`the connect after the close ended with ${stall.errno}`);
1659
+ if (!pressure.points.some((point) => inside(stall.time, point))) found.push(`the connect after the close took ${valueIn(stall.time, "s")} s, on no point of the schedule`);
1660
+ }
1661
+ }
1662
+ return found;
1663
+ };
1664
+ /**
1665
+ * This function reads one run of a conntrack cell from the lines of its client and its target. An
1666
+ * `unsettled` line makes the run void. The entries of the run before it were still in the table, thus
1667
+ * the run did not start from an empty table.
1668
+ */
1669
+ const measureConntrack = (run, expected) => {
1670
+ const { events, kernel } = run;
1671
+ const retransmissions = kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length;
1672
+ const base = {
1673
+ environment: run.environment,
1674
+ index: run.index,
1675
+ outcome: { kind: "answered" },
1676
+ elapsed: null,
1677
+ socket: null,
1678
+ noticed: null,
1679
+ steps: [],
1680
+ probes: 0,
1681
+ retransmissions,
1682
+ acks: 0
1683
+ };
1684
+ const voided = (why) => ({
1685
+ ...base,
1686
+ void: why,
1687
+ finding: null
1688
+ });
1689
+ const unsettled = events.find((event) => event.ev === "unsettled");
1690
+ if (unsettled !== void 0) return voided(`the table held ${String(unsettled["count"])} entries at the end of the wait before the run`);
1691
+ if (events.find((event) => event.ev === "done") === void 0) {
1692
+ const error = events.find((event) => event.ev === "error");
1693
+ return voided(error === void 0 ? "the client did not reach its end" : `the client did not reach its end: ${String(error["what"])}`);
1694
+ }
1695
+ if (run.lost === null) return voided("the prober wrote no clock line after the run, so nothing says what it dropped");
1696
+ if (run.lost > 0) return voided(`the prober dropped ${run.lost} records`);
1697
+ const full = events.find((event) => event.ev === "full");
1698
+ if (full === void 0) return voided("the client wrote no line for its fill");
1699
+ const stall = events.find((event) => event.ev === "stall");
1700
+ const closing = events.find((event) => event.ev === "closing");
1701
+ const drained = events.find((event) => event.ev === "drained");
1702
+ const closed = events.find((event) => event.ev === "closed");
1703
+ const neighbor = (run.server ?? []).filter((line) => line.ev === "count" && line.t >= full.t && (stall === void 0 || line.t <= stall.t)).at(-1);
1704
+ const conntrack = {
1705
+ flows: numberOr(full, "flows") ?? 0,
1706
+ count: numberOr(full, "count") ?? 0,
1707
+ neighbor: numberOr(neighbor, "count"),
1708
+ stall: stallOf(stall),
1709
+ firstFall: untilField(closing, drained, "first_fall"),
1710
+ zero: untilField(closing, drained, "zero"),
1711
+ closed: numberOr(closed, "count"),
1712
+ lowest: numberOr(drained ?? stall, "lowest"),
1713
+ after: numberOr(stall, "after")
1714
+ };
1715
+ const found = findingsOf$1(conntrack, full["errno"], expected);
1716
+ return {
1717
+ ...base,
1718
+ void: null,
1719
+ finding: found.length === 0 ? null : found.join(". "),
1720
+ conntrack
1721
+ };
1722
+ };
1723
+ //#endregion
1724
+ //#region src/pause.ts
1725
+ /** This expression finds the stamp of a log line of a broker, as its console writes it, on UTC. */
1726
+ const LOG_TIME = /\[(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d),(\d{3})\]/;
1727
+ /** This function gives the time of a log line on the monotonic clock of a member, in ns, from the `start` line of that member. It is null for a line with no stamp. */
1728
+ const logTimeOf = (line, start) => {
1729
+ const found = LOG_TIME.exec(line);
1730
+ if (found === null) return null;
1731
+ const wall = Date.UTC(Number(found[1]), Number(found[2]) - 1, Number(found[3]), Number(found[4]), Number(found[5]), Number(found[6]), Number(found[7]));
1732
+ return start.t + (wall - Number(start["wall"])) * 1e6;
1733
+ };
1734
+ /** This function gives the time between two instants of the monotonic clock, in s, to 1 microsecond. */
1735
+ const secondsBetween = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
1736
+ const isInside = (time, band) => {
1737
+ const seconds = valueIn(time, "s");
1738
+ return seconds >= valueIn(band.low, "s") && seconds <= valueIn(band.high, "s");
1739
+ };
1740
+ const bandText = (band) => `[${valueIn(band.low, "s")}, ${valueIn(band.high, "s")}] s`;
1741
+ const isChange = (event) => event.ev === "assigned" || event.ev === "revoked" || event.ev === "lost";
1742
+ /** This function gives the partitions that a line of the rebalance listener names, in its field `partitions` or `held`. */
1743
+ const partitionsOf = (event, field) => event[field] ?? [];
1744
+ /** This function gives the last line of the rebalance listener in a list of lines. */
1745
+ const lastChange = (events) => events.findLast(isChange);
1746
+ /** This function gives the offset that a commit names for a partition, or null. */
1747
+ const offsetOf = (event, partition) => {
1748
+ const value = event["offsets"]?.[String(partition)];
1749
+ return typeof value === "number" ? value : null;
1750
+ };
1751
+ /**
1752
+ * This function tells if the two members can take the fault. Each member holds one partition, the
1753
+ * two partitions differ, and the first member has two acknowledged commits on its partition after its
1754
+ * last change. Thus the first member reads and commits at a steady rate.
1755
+ */
1756
+ const pauseReady = (first, other) => {
1757
+ const mine = lastChange(first);
1758
+ const theirs = lastChange(other);
1759
+ if (mine === void 0 || theirs === void 0) return false;
1760
+ const held = partitionsOf(mine, "held");
1761
+ const others = partitionsOf(theirs, "held");
1762
+ if (held.length !== 1 || others.length !== 1 || held[0] === others[0]) return false;
1763
+ return first.filter((event) => event.ev === "commit" && event.t > mine.t && offsetOf(event, held[0]) !== null).length >= 2;
1764
+ };
1765
+ /** This function tells if a log line of a broker tells that a member left a group, under either protocol. */
1766
+ const isFenceLine = (line) => /Member \S+ fenced from the group because|Member \S+ in group \S+ has failed, removing it from the group/.test(line);
1767
+ /** This function tells if a fence line names a group. The two protocols write the name in different words. */
1768
+ const namesGroup = (line, group) => line.includes(`[GroupId ${group}]`) || line.includes(`in group ${group} has`);
1769
+ /** This function compares a reading to the law, and gives each disagreement. */
1770
+ const findingsOf = (reading, prediction, fences, otherReports) => {
1771
+ const found = [];
1772
+ const { commit, fence, takeover } = reading;
1773
+ if (prediction.fence === null) {
1774
+ if (fences.length > 0) found.push(`the group removed a member: ${fences[0]}`);
1775
+ if (otherReports) found.push("the other member reported the partition");
1776
+ } else {
1777
+ if (fence === null) found.push("no log line tells of a fence, and the other member did not report the partition");
1778
+ else if (!isInside(fence, prediction.fence)) found.push(`the fence is ${valueIn(fence, "s")} s, outside ${bandText(prediction.fence)}`);
1779
+ if (reading.otherFirst !== reading.lastAck) found.push(`the other member started at offset ${reading.otherFirst}, and the last commit is ${reading.lastAck}`);
1780
+ }
1781
+ if (prediction.takeover !== null) {
1782
+ if (takeover === null) found.push("the other member did not report the partition");
1783
+ else if (!isInside(takeover, prediction.takeover)) found.push(`the other member reported the partition at ${valueIn(takeover, "s")} s, outside ${bandText(prediction.takeover)}`);
1784
+ }
1785
+ if (commit === null) {
1786
+ found.push("the member made no commit after the stop");
1787
+ return found;
1788
+ }
1789
+ switch (prediction.thaw) {
1790
+ case "fails":
1791
+ if (commit.name !== "CommitFailedException") found.push(`the commit after the stop gave ${commit.name ?? "an acknowledgment"}, not CommitFailedException`);
1792
+ break;
1793
+ case "acknowledged":
1794
+ if (!commit.acked) found.push(`the commit after the stop failed with ${commit.name}`);
1795
+ if (commit.epoch !== reading.epochBefore) found.push(`the epoch changed from ${reading.epochBefore} to ${commit.epoch}`);
1796
+ break;
1797
+ case "rejoin": {
1798
+ const within = prediction.rejoinWithin;
1799
+ if (reading.rejoin === null) found.push("the member did not get a new epoch");
1800
+ else if (within !== void 0 && valueIn(reading.rejoin, "s") > valueIn(within, "s")) found.push(`the new epoch came at ${valueIn(reading.rejoin, "s")} s, above ${valueIn(within, "s")} s`);
1801
+ if (!commit.acked) found.push(`the stale commit failed with ${commit.name}`);
1802
+ if (reading.reassigned) found.push("the member held the partition again before its commit");
1803
+ if (reading.committed !== reading.stale) found.push(`the committed offset is ${reading.committed}, and the stale commit named ${reading.stale}`);
1804
+ if (reading.stale === null || reading.otherLast === null || reading.stale >= reading.otherLast) found.push(`the stale commit ${reading.stale} is not below the progress ${reading.otherLast} of the other member`);
1805
+ break;
1806
+ }
1807
+ }
1808
+ return found;
1809
+ };
1810
+ /** This function reads one run of a paused consumer against its law. */
1811
+ const measurePause = (run, prediction) => {
1812
+ const retransmissions = run.kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length;
1813
+ const base = {
1814
+ environment: run.environment,
1815
+ index: run.index,
1816
+ outcome: { kind: "answered" },
1817
+ elapsed: null,
1818
+ socket: null,
1819
+ noticed: null,
1820
+ steps: [],
1821
+ probes: 0,
1822
+ retransmissions,
1823
+ acks: 0
1824
+ };
1825
+ const voided = (why) => ({
1826
+ ...base,
1827
+ void: why,
1828
+ finding: null
1829
+ });
1830
+ const { fault, heal, pause } = run;
1831
+ if (fault === null || pause === void 0) return voided("the partitions were not one for each member before the stop");
1832
+ if (heal === void 0 || heal === null) return voided("the stop did not end");
1833
+ const stop = fault.t0;
1834
+ const thaw = heal.t0;
1835
+ const before = run.events.filter((event) => event.t < stop);
1836
+ const after = run.events.filter((event) => event.t >= stop);
1837
+ const start = run.events.find((event) => event.ev === "start");
1838
+ if (start === void 0) return voided("the member wrote no start line");
1839
+ const mine = lastChange(before);
1840
+ const theirs = lastChange(pause.other.filter((event) => event.t < stop));
1841
+ const held = mine === void 0 ? [] : partitionsOf(mine, "held");
1842
+ const others = theirs === void 0 ? [] : partitionsOf(theirs, "held");
1843
+ const partition = held[0];
1844
+ if (partition === void 0 || held.length !== 1 || others.length !== 1 || others[0] === partition) return voided("the partitions were not one for each member before the stop");
1845
+ if (after.find((event) => event.ev === "pause")?.["in"] !== "process") return voided("the stop landed outside the processing of a batch");
1846
+ if (before.findLast((event) => event.ev === "read")?.["partition"] !== partition) return voided("the last read before the stop was on another partition");
1847
+ const lastCommit = before.findLast((event) => event.ev === "commit" && offsetOf(event, partition) !== null);
1848
+ if (lastCommit === void 0) return voided("the member had no acknowledged commit on its partition before the stop");
1849
+ const member = String(mine?.["member"]);
1850
+ const fenceAt = pause.fences.filter((line) => line.includes(member)).map((line) => logTimeOf(line, start)).find((time) => time !== null && time >= stop) ?? null;
1851
+ const taken = pause.other.find((event) => event.ev === "assigned" && event.t >= stop && partitionsOf(event, "partitions").includes(partition));
1852
+ if (prediction.fence !== null && fenceAt === null && taken !== void 0) return voided("the broker log lost the line of the fence");
1853
+ const thawed = after.find((event) => event.ev === "commit" || event.ev === "commit-fail");
1854
+ const rejoined = after.find((event) => event.ev === "rejoined");
1855
+ const ackAt = thawed?.t ?? Number.POSITIVE_INFINITY;
1856
+ const otherCommit = pause.other.findLast((event) => event.ev === "commit" && event.t < ackAt && offsetOf(event, partition) !== null);
1857
+ const committed = after.find((event) => event.ev === "committed");
1858
+ const epochOf = (event) => typeof event?.["epoch"] === "number" ? event["epoch"] : null;
1859
+ const pauseReading = {
1860
+ partition,
1861
+ fence: fenceAt === null ? null : secondsBetween(stop, fenceAt),
1862
+ takeover: taken === void 0 ? null : secondsBetween(stop, taken.t),
1863
+ commit: thawed === void 0 ? null : {
1864
+ acked: thawed.ev === "commit",
1865
+ name: typeof thawed["name"] === "string" ? thawed["name"] : null,
1866
+ message: typeof thawed["message"] === "string" ? thawed["message"] : null,
1867
+ epoch: epochOf(thawed)
1868
+ },
1869
+ epochBefore: epochOf(lastCommit),
1870
+ lastAck: offsetOf(lastCommit, partition),
1871
+ otherFirst: pause.other.find((event) => event.ev === "read" && event.t >= stop && event["partition"] === partition)?.["first"] ?? null,
1872
+ otherLast: otherCommit === void 0 ? null : offsetOf(otherCommit, partition),
1873
+ rejoin: rejoined === void 0 ? null : secondsBetween(thaw, rejoined.t),
1874
+ reassigned: after.some((event) => event.ev === "assigned" && event.t >= thaw && event.t <= ackAt && partitionsOf(event, "partitions").includes(partition)),
1875
+ committed: committed === void 0 ? null : offsetOf(committed, partition),
1876
+ stale: thawed === void 0 ? null : offsetOf(thawed, partition)
1877
+ };
1878
+ const findings = findingsOf(pauseReading, prediction, pause.fences, taken !== void 0);
1879
+ return {
1880
+ ...base,
1881
+ void: null,
1882
+ finding: findings.length === 0 ? null : findings.join(". "),
1883
+ pause: pauseReading
1884
+ };
1885
+ };
1886
+ //#endregion
1887
+ //#region src/ports.ts
1888
+ /** This errno ends a search that found no port. */
1889
+ const NO_PORT = "EADDRNOTAVAIL";
1890
+ const portsOf = (line) => Array.isArray(line["ports"]) ? line["ports"].filter((port) => typeof port === "number") : [];
1891
+ /** This function says if the ports of the parity of the low port come first in one search. */
1892
+ const lowFirst = (ports, low) => {
1893
+ const odd = ports.findIndex((port) => (port - low) % 2 !== 0);
1894
+ return odd < 0 || ports.slice(odd).every((port) => (port - low) % 2 !== 0);
1895
+ };
1896
+ /**
1897
+ * This function reads one run of a port cell. The `round` lines give each search. The `closing` and
1898
+ * `freed` lines give when each held port comes back. The records of the prober give the
1899
+ * retransmissions.
1900
+ */
1901
+ const measurePorts = (run, expected) => {
1902
+ const { events, kernel } = run;
1903
+ const retransmissions = kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length;
1904
+ const base = {
1905
+ environment: run.environment,
1906
+ index: run.index,
1907
+ outcome: { kind: "answered" },
1908
+ elapsed: null,
1909
+ socket: null,
1910
+ noticed: null,
1911
+ steps: [],
1912
+ probes: 0,
1913
+ retransmissions,
1914
+ acks: 0
1915
+ };
1916
+ const voided = (why) => ({
1917
+ ...base,
1918
+ void: why,
1919
+ finding: null
1920
+ });
1921
+ if (events.find((event) => event.ev === "done") === void 0) return voided("the client did not reach its end");
1922
+ if (run.lost === null) return voided("the prober wrote no clock line after the run, so nothing says what it dropped");
1923
+ if (run.lost > 0) return voided(`the prober dropped ${run.lost} records`);
1924
+ const rounds = events.filter((event) => event.ev === "round");
1925
+ const closing = events.find((event) => event.ev === "closing");
1926
+ const freedLines = events.filter((event) => event.ev === "freed");
1927
+ const counts = {};
1928
+ for (const round of rounds) {
1929
+ const count = String(portsOf(round).length);
1930
+ counts[count] = (counts[count] ?? 0) + 1;
1931
+ }
1932
+ const seen = [...new Set(rounds.flatMap((round) => portsOf(round).map((port) => port - expected.low)))].sort((a, b) => a - b);
1933
+ const refusal = Math.max(0, ...rounds.map((round) => typeof round["refusal"] === "number" ? round["refusal"] : 0));
1934
+ const first = freedLines[0];
1935
+ const last = freedLines.at(-1);
1936
+ const ports = {
1937
+ rounds: rounds.length,
1938
+ counts,
1939
+ seen,
1940
+ errnos: [...new Set(rounds.map((round) => String(round["errno"])))],
1941
+ refusal: quantity(Math.round(refusal / 1e3) / 1e6, "s"),
1942
+ firstFreed: closing === void 0 || first === void 0 ? null : between$2(closing.t, first.t),
1943
+ lastFreed: closing === void 0 || last === void 0 ? null : between$2(closing.t, last.t),
1944
+ freed: freedLines.length,
1945
+ tries: last === void 0 || typeof last["tries"] !== "number" ? 0 : last["tries"]
1946
+ };
1947
+ const found = [];
1948
+ const searches = expected.rounds * expected.destinations;
1949
+ if (ports.rounds !== searches) found.push(`${ports.rounds} searches, not ${searches}`);
1950
+ const stray = Object.keys(counts).filter((count) => !expected.counts.includes(Number(count)));
1951
+ if (stray.length > 0) found.push(`a search gave ${stray.join(" or ")} connects, outside ${expected.counts.join(" or ")}`);
1952
+ if (seen.join() !== expected.seen.join()) found.push(`the ports at offsets ${seen.join(" ")} were used, not ${expected.seen.join(" ")}`);
1953
+ if (ports.errnos.join() !== NO_PORT) found.push(`the searches ended with ${ports.errnos.join(" and ")}, not ${NO_PORT}`);
1954
+ if (valueIn(ports.refusal, "s") > valueIn(expected.refusedWithin, "s")) found.push(`a refused connect took ${valueIn(ports.refusal, "s")} s, above ${valueIn(expected.refusedWithin, "s")} s`);
1955
+ if (expected.share !== void 0) {
1956
+ const { count, low, high } = expected.share;
1957
+ const share = (counts[String(count)] ?? 0) / Math.max(1, ports.rounds);
1958
+ if (share < low || share > high) found.push(`the share of ${count} connects is ${share}, outside [${low}, ${high}]`);
1959
+ }
1960
+ if (expected.lowFirst === true && !rounds.every((round) => lowFirst(portsOf(round), expected.low))) found.push("a search got a port of the other parity before each port of the parity of the low port");
1961
+ if (expected.retransmissions !== void 0 && retransmissions !== expected.retransmissions) found.push(`${retransmissions} retransmissions, not ${expected.retransmissions}`);
1962
+ if (expected.freed !== void 0) {
1963
+ const held = Math.max(...expected.counts);
1964
+ if (ports.freed !== held) found.push(`${ports.freed} ports came back, not ${held}`);
1965
+ for (const [name, time] of [["first", ports.firstFreed], ["last", ports.lastFreed]]) if (time === null || !inside(time, expected.freed)) found.push(`the ${name} port came back at ${time === null ? "no time" : `${valueIn(time, "s")} s`}, outside ${bandWords(expected.freed)}`);
1966
+ if (ports.tries === 0) found.push("no connect was refused before the ports came back");
1967
+ }
1968
+ return {
1969
+ ...base,
1970
+ void: null,
1971
+ finding: found.length === 0 ? null : found.join(". "),
1972
+ ports
1973
+ };
1974
+ };
1975
+ //#endregion
1148
1976
  //#region src/recovery.ts
1149
1977
  /**
1150
1978
  * This module holds the pure parts of a recovery drill. These are the scripts of the shell, the reader
@@ -1516,12 +2344,32 @@ const writesOf = (run, after, primary) => {
1516
2344
  held: kept === null ? null : kept.has(n)
1517
2345
  };
1518
2346
  });
2347
+ const duplicates = holds === null ? null : [...holds].sort((a, b) => a - b).filter((n, at, all) => all[at - 1] === n);
2348
+ const oldHolds = run.group?.oldHolds;
1519
2349
  return {
1520
2350
  sent: sent.length,
1521
2351
  acked: acks.size,
1522
2352
  old,
1523
2353
  lost,
1524
- unknown
2354
+ unknown,
2355
+ duplicates,
2356
+ ...oldHolds === void 0 ? {} : { sets: setsOf(run, acks.keys(), unknown, new Set(oldHolds)) }
2357
+ };
2358
+ };
2359
+ /**
2360
+ * This function splits the writes of a pair by its two final reads. A write with no reply is `kept`
2361
+ * on the new primary. It is `stranded` on the old primary alone, or `absent` from each server.
2362
+ */
2363
+ const setsOf = (run, acked, unknown, old) => {
2364
+ const kept = unknown.filter((each) => each.held === true).length;
2365
+ const stranded = unknown.filter((each) => each.held !== true && old.has(each.n)).length;
2366
+ return {
2367
+ unknown: unknown.length,
2368
+ kept,
2369
+ stranded,
2370
+ absent: unknown.length - kept - stranded,
2371
+ missing: [...acked].filter((n) => !old.has(n)).length,
2372
+ warnings: run.events.filter((event) => event.ev === "notice" && event["severity"] === "WARNING").length
1525
2373
  };
1526
2374
  };
1527
2375
  /** This function gives the errors that the client met after an instant: each kind, its first time, and its count. */
@@ -1632,9 +2480,10 @@ const measureGroup = (cell, run) => {
1632
2480
  const heal = run.heal ?? null;
1633
2481
  const lands = expected?.lands ?? "new";
1634
2482
  const cluster = cell.cluster !== void 0;
1635
- const replica = cell.replicaSet !== void 0 || cluster;
2483
+ const kafka = cell.kafka !== void 0;
2484
+ const replica = cell.replicaSet !== void 0 || cluster || kafka;
1636
2485
  const stall = prediction.detects === false && prediction.recovers !== void 0;
1637
- const stages = cluster ? clusterStagesOf(run, t0, run.fault.t1) : replica ? replicaSetStagesOf(run, t0, run.fault.t1, lands) : stagesOf(timeline, t0);
2486
+ const stages = kafka ? {} : cluster ? clusterStagesOf(run, t0, run.fault.t1) : replica ? replicaSetStagesOf(run, t0, run.fault.t1, lands) : stagesOf(timeline, t0);
1638
2487
  const after = lands === "same" && heal !== null && (!replica || stall) ? heal.t0 : t0;
1639
2488
  const sentAt = /* @__PURE__ */ new Map();
1640
2489
  for (const event of events) if (event.ev === "sent") sentAt.set(Number(event["n"]), event.t);
@@ -1666,14 +2515,14 @@ const measureGroup = (cell, run) => {
1666
2515
  if (voided === null && split !== void 0 && (elected === void 0 || elected.t > split.t)) voided = "the election split its votes and runs again after 2 × failover-timeout, which the band does not bound";
1667
2516
  const odown = stages["+odown"];
1668
2517
  if (voided === null && odown !== void 0 && stages["+try-failover"] === void 0) voided = "no Sentinel started a failover after +odown: each was in the hold-off of 2 × failover-timeout after an earlier attempt";
1669
- if (voided === null && cell.hold !== void 0 && prediction.detects !== null) {
2518
+ if (voided === null && cell.hold !== void 0 && (prediction.detects !== null || kafka)) {
1670
2519
  if (heal === null) voided = "the fault never lifted";
1671
2520
  else {
1672
2521
  const late = (heal.t0 - t0) / 1e9 - valueIn(cell.hold, "s");
1673
2522
  if (late > valueIn(INJECT, "s")) voided = `the lift came ${Math.round(late * 1e4) / 10} ms after its hold: the injector was late, not the group`;
1674
2523
  }
1675
2524
  }
1676
- const firstAddress = cluster ? String(events.find((event) => event.ev === "connected")?.["at"] ?? "") : `${run.group?.members[primary]?.ip}:${(cell.group ?? cell.replicaSet)?.port}`;
2525
+ const firstAddress = cluster ? String(events.find((event) => event.ev === "connected")?.["at"] ?? "") : `${run.group?.members[primary]?.ip}:${(cell.group ?? cell.replicaSet ?? cell.kafka)?.port}`;
1677
2526
  const assumed = prediction.detects === null ? void 0 : prediction.rtt;
1678
2527
  const first = kernel.find((record) => record.netns === run.netns && record.dst === firstAddress && record.tcp?.srtt !== void 0);
1679
2528
  const rtt = first?.tcp?.srtt === void 0 ? null : valueIn(first.tcp.srtt, "us") / 1e3;
@@ -1693,11 +2542,13 @@ const measureGroup = (cell, run) => {
1693
2542
  const findings = [];
1694
2543
  const switched = replica ? stages["elected"] : stages["+switch-master"];
1695
2544
  if (expected?.failover === false && switched !== void 0) findings.push(`the group failed over ${switched.value} s after the fault, where it should not`);
2545
+ const leaderAfter = run.group?.after ?? null;
2546
+ if (kafka && expected !== void 0 && leaderAfter !== null && expected.failover !== (leaderAfter !== primary)) findings.push(expected.failover ? `the leader stayed on ${primary}, where it should change` : `the leader moved to ${leaderAfter}, where it should stay on ${primary}`);
1696
2547
  for (const stage of expected?.quiet ?? []) {
1697
2548
  const at = stages[stage];
1698
2549
  if (at !== void 0) findings.push(`${stage} came ${at.value} s after the fault, where it should not`);
1699
2550
  }
1700
- if (accepted !== void 0 && lands === "same" && !cluster && to !== primary) findings.push(`the client wrote to ${to ?? String(accepted["at"])}, where it should stay on ${primary}`);
2551
+ if (accepted !== void 0 && lands === "same" && !cluster && !kafka && to !== primary) findings.push(`the client wrote to ${to ?? String(accepted["at"])}, where it should stay on ${primary}`);
1701
2552
  for (const error of expected?.errors ?? []) {
1702
2553
  const met = group.errors.find((each) => each.name === error.name);
1703
2554
  const noise = valueIn(noiseOf(cell), "s");
@@ -1710,9 +2561,22 @@ const measureGroup = (cell, run) => {
1710
2561
  if (lost === null) findings.push("the group did not converge, thus the count of its lost writes is not known");
1711
2562
  else if (lost.length < low || lost.length > high) findings.push(`the group lost ${lost.length} accepted writes, outside [${low}, ${high}]`);
1712
2563
  }
2564
+ if (expected?.duplicates !== void 0) {
2565
+ const { low, high } = expected.duplicates;
2566
+ const copies = group.writes.duplicates ?? null;
2567
+ if (copies === null) findings.push("the group did not converge, thus the count of its extra copies is not known");
2568
+ else if (run.group?.holds?.length === 0) findings.push("the group holds no write, thus the count of its extra copies says nothing");
2569
+ else if (copies.length < low || copies.length > high) findings.push(`the group holds ${copies.length} extra copies, outside [${low}, ${high}]`);
2570
+ }
2571
+ const { sets } = group.writes;
2572
+ for (const [name, band] of Object.entries(expected?.outcomes ?? {})) {
2573
+ const count = sets?.[name];
2574
+ if (count === void 0) findings.push(`the run has no count of ${name}`);
2575
+ else if (count < band.low || count > band.high) findings.push(`the count of ${name} is ${count}, outside [${band.low}, ${band.high}]`);
2576
+ }
1713
2577
  if (group.recovery !== void 0) findings.push(...recoveryFindings(group.recovery, group.writes));
1714
2578
  if (accepted === void 0) {
1715
- if (prediction.detects !== null) findings.push(lands === "new" ? "the client never wrote to a new primary in the window" : "the client never wrote again in the window");
2579
+ if (prediction.detects !== null || kafka) findings.push(lands === "new" ? "the client never wrote to a new primary in the window" : "the client never wrote again in the window");
1716
2580
  return {
1717
2581
  ...silent,
1718
2582
  void: voided,
@@ -1981,7 +2845,22 @@ const requestOf = (events) => {
1981
2845
  };
1982
2846
  /** This function reads the socket of one run against its cell, as `measure` does without the second request of an HTTP client. */
1983
2847
  const measureSocket = (cell, run) => {
1984
- if (cell.group !== void 0 || cell.replicaSet !== void 0 || cell.cluster !== void 0) {
2848
+ if (run.missing !== void 0) return {
2849
+ environment: run.environment,
2850
+ index: run.index,
2851
+ outcome: { kind: "answered" },
2852
+ elapsed: null,
2853
+ socket: null,
2854
+ noticed: null,
2855
+ steps: [],
2856
+ probes: 0,
2857
+ retransmissions: 0,
2858
+ acks: 0,
2859
+ void: `the environment cannot do this cell: ${run.missing}`,
2860
+ finding: null
2861
+ };
2862
+ if (cell.prediction.detects === null && cell.prediction.pause !== void 0) return measurePause(run, cell.prediction.pause);
2863
+ if (cell.group !== void 0 || cell.replicaSet !== void 0 || cell.cluster !== void 0 || cell.pair !== void 0 || cell.kafka !== void 0) {
1985
2864
  const read = measureGroup(cell, run);
1986
2865
  return {
1987
2866
  environment: run.environment,
@@ -1999,6 +2878,9 @@ const measureSocket = (cell, run) => {
1999
2878
  group: read.group
2000
2879
  };
2001
2880
  }
2881
+ if (cell.prediction.detects === null && cell.prediction.queue !== void 0) return measureQueue(run, cell.prediction.queue);
2882
+ if (cell.prediction.detects === null && cell.prediction.ports !== void 0) return measurePorts(run, cell.prediction.ports);
2883
+ if (cell.prediction.detects === null && cell.prediction.conntrack !== void 0) return measureConntrack(run, cell.prediction.conntrack);
2002
2884
  if (cell.prediction.detects !== null && cell.prediction.driver !== void 0) return measureDriver(cell, run);
2003
2885
  const { events, kernel } = run;
2004
2886
  const ready = events.find((event) => event.ev === "ready");
@@ -2129,7 +3011,13 @@ const measureSocket = (cell, run) => {
2129
3011
  forgot: forgotten === void 0 ? null : between(lastRx, forgotten.t)
2130
3012
  };
2131
3013
  if (prediction.detects === null) {
2132
- const sends = events.filter((event) => event.ev === "sent").slice(1);
3014
+ const load = run.streams !== void 0 ? streamsLoadOf(events, run.streams)?.pool ?? null : run.schedule === void 0 ? void 0 : loadOf(events, run.schedule);
3015
+ if (load === null) return {
3016
+ ...base,
3017
+ ...nothing,
3018
+ void: "the client never started its schedule"
3019
+ };
3020
+ const sends = load === void 0 ? events.filter((event) => event.ev === "sent").slice(1) : [];
2133
3021
  const answers = events.some((event) => event.ev === "replied") ? "replied" : "data";
2134
3022
  const samples = sends.flatMap((send, n) => {
2135
3023
  const next = sends[n + 1]?.t ?? Infinity;
@@ -2155,17 +3043,19 @@ const measureSocket = (cell, run) => {
2155
3043
  }, run)) };
2156
3044
  const reconnect = prediction.reconnects === void 0 || ended === void 0 ? void 0 : reconnectOf(run, ended, cookie, drop?.line.t ?? ended);
2157
3045
  const late = reconnect === void 0 || prediction.reconnects === void 0 ? null : reconnectFinding(prediction.reconnects, reconnect, noiseOf(cell));
3046
+ const counted = prediction.counts === void 0 ? null : countFinding(prediction.counts, run.server ?? []);
2158
3047
  return {
2159
3048
  ...base,
2160
3049
  ...nothing,
2161
3050
  ...chain,
2162
3051
  ...reconnect === void 0 ? {} : { reconnect },
2163
- finding: late,
3052
+ finding: late ?? counted,
2164
3053
  outcome: ended !== void 0 ? outcome : { kind: "answered" },
2165
3054
  ...from === void 0 || until === void 0 ? {} : { elapsed: between(from, until) },
2166
3055
  noticed: prediction.from === void 0 ? null : noticed,
2167
3056
  steps: retransmitted.filter((record) => record.mono > request.t).map((record) => between(request.t, record.mono)),
2168
3057
  samples,
3058
+ ...load === void 0 ? {} : { load },
2169
3059
  reordering: reorderingOf(socket.filter((record) => record.mono > request.t), peerRecords.filter((record) => record.mono > request.t)),
2170
3060
  ...crossings,
2171
3061
  void: voided
@@ -2918,7 +3808,7 @@ const TAIL_LINES = 60;
2918
3808
  const OUTPUT_LINES = 2e3;
2919
3809
  /** This time is how long a write of a hop waits for the line that confirms it, in milliseconds. */
2920
3810
  const CONFIRM_MS = 1e4;
2921
- const sleep$4 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
3811
+ const sleep$6 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
2922
3812
  /** This function gives the arguments of the injector for the options of a fault. */
2923
3813
  const optionArgs = (options) => [
2924
3814
  ...options.direction === void 0 ? [] : ["--direction", options.direction],
@@ -3112,6 +4002,7 @@ const clientOf = async (kind) => {
3112
4002
  clientImage: await imageOf(client.image),
3113
4003
  command: client.command,
3114
4004
  clientKind: typeof kind === "string" ? kind : "image",
4005
+ ...client.capabilities === void 0 || client.capabilities.length === 0 ? {} : { capabilities: client.capabilities },
3115
4006
  ...typeof client.image === "string" ? {} : { source: client.image }
3116
4007
  };
3117
4008
  };
@@ -3143,18 +4034,21 @@ const deviceOf = (route) => {
3143
4034
  */
3144
4035
  const openLab = async (target, kind = "tcp", topology) => {
3145
4036
  const image = await harnessImage();
3146
- const { clientImage, command, clientKind, source } = await clientOf(kind);
4037
+ const { clientImage, command, clientKind, capabilities, source } = await clientOf(kind);
3147
4038
  const hops = topology?.hops ?? [];
3148
4039
  const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();
3149
4040
  const runtime = await getContainerRuntimeClient();
3150
4041
  const opened = [];
3151
4042
  const networks = [];
3152
4043
  const serverLines = lines();
4044
+ let restoreHost = () => Promise.resolve([]);
3153
4045
  let fabric;
3154
4046
  const close = async () => {
4047
+ const restored = await restoreHost().then(() => null, (error) => error);
3155
4048
  await fabric?.close();
3156
4049
  for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
3157
4050
  for (const network of networks.reverse()) await network.stop().catch(() => void 0);
4051
+ if (restored !== null) throw restored;
3158
4052
  };
3159
4053
  /** This function gives the address of a container on a network, as Docker has it at this time. A container can join a network after its start. */
3160
4054
  const ipOn = async (container, network) => {
@@ -3459,7 +4353,35 @@ const openLab = async (target, kind = "tcp", topology) => {
3459
4353
  };
3460
4354
  };
3461
4355
  const proxies = inChain.filter((hop) => hop.hop.kind === "proxy");
4356
+ let hostAdmin;
4357
+ const hostChanges = [];
4358
+ let hostDone;
4359
+ restoreHost = async () => {
4360
+ if (hostDone !== void 0) return hostDone;
4361
+ const restored = [];
4362
+ for (const change of [...hostChanges].reverse()) {
4363
+ await output(hostAdmin, [
4364
+ "sysctl",
4365
+ "-q",
4366
+ "-w",
4367
+ `${change.name}=${change.before}`
4368
+ ]);
4369
+ const read = await output(hostAdmin, [
4370
+ "sysctl",
4371
+ "-n",
4372
+ change.name
4373
+ ]);
4374
+ if (read !== change.before) throw new Error(`${change.name} reads ${read} after the restore, not ${change.before}`);
4375
+ restored.unshift({
4376
+ ...change,
4377
+ restored: read
4378
+ });
4379
+ }
4380
+ hostDone = restored;
4381
+ return restored;
4382
+ };
3462
4383
  return {
4384
+ serverIp,
3463
4385
  targetIp,
3464
4386
  targetPort,
3465
4387
  clientIp,
@@ -3504,6 +4426,53 @@ const openLab = async (target, kind = "tcp", topology) => {
3504
4426
  ]);
3505
4427
  return read;
3506
4428
  },
4429
+ async namespaceSetup(commands) {
4430
+ for (const command of commands) {
4431
+ await output(admin, [...command]);
4432
+ await output(injector, [...command]);
4433
+ }
4434
+ },
4435
+ async hostSysctl(settings) {
4436
+ if (settings.length === 0) return;
4437
+ if (hostAdmin === void 0) {
4438
+ hostAdmin = await new GenericContainer(image).withNetworkMode("host").withPrivilegedMode().start();
4439
+ opened.push(hostAdmin);
4440
+ }
4441
+ for (const { name, value, guard } of settings) {
4442
+ if (guard !== void 0) {
4443
+ const reads = Number(await output(hostAdmin, [
4444
+ "sysctl",
4445
+ "-n",
4446
+ guard.name
4447
+ ]));
4448
+ if (!(reads <= guard.atMost)) throw new Error(`${guard.name} of the host reads ${reads}, above ${guard.atMost}`);
4449
+ }
4450
+ const before = await output(hostAdmin, [
4451
+ "sysctl",
4452
+ "-n",
4453
+ name
4454
+ ]);
4455
+ hostChanges.push({
4456
+ name,
4457
+ before,
4458
+ set: value,
4459
+ restored: null
4460
+ });
4461
+ await output(hostAdmin, [
4462
+ "sysctl",
4463
+ "-q",
4464
+ "-w",
4465
+ `${name}=${value}`
4466
+ ]);
4467
+ const set = await output(hostAdmin, [
4468
+ "sysctl",
4469
+ "-n",
4470
+ name
4471
+ ]);
4472
+ if (set !== value) throw new Error(`${name} of the host reads ${set} after the write, not ${value}`);
4473
+ }
4474
+ },
4475
+ restoreHost: () => restoreHost(),
3507
4476
  async configure(values) {
3508
4477
  const read = {};
3509
4478
  if (Object.keys(values).length === 0) return read;
@@ -3524,7 +4493,7 @@ const openLab = async (target, kind = "tcp", topology) => {
3524
4493
  return stamped(where, [
3525
4494
  "/opt/harness/inject",
3526
4495
  kind,
3527
- peer,
4496
+ options.peers?.join(",") ?? peer,
3528
4497
  ...optionArgs(options),
3529
4498
  ...via
3530
4499
  ]);
@@ -3534,7 +4503,7 @@ const openLab = async (target, kind = "tcp", topology) => {
3534
4503
  await output(where, [
3535
4504
  "/opt/harness/inject",
3536
4505
  LIFT[kind],
3537
- peer,
4506
+ options.peers?.join(",") ?? peer,
3538
4507
  ...optionArgs(options),
3539
4508
  ...via
3540
4509
  ]);
@@ -3586,6 +4555,18 @@ const openLab = async (target, kind = "tcp", topology) => {
3586
4555
  if (target.close === void 0) throw new Error(`${target.image} names no way to close its clients`);
3587
4556
  await output(server, [...target.close]);
3588
4557
  },
4558
+ async exec(argv) {
4559
+ return output(server, [...argv]);
4560
+ },
4561
+ async sibling(siblingImage, argv, ready) {
4562
+ const started = await new GenericContainer(siblingImage).withNetwork(last).withWaitStrategy(Wait.forLogMessage(ready)).withCommand([...argv]).start();
4563
+ opened.push(started);
4564
+ return {
4565
+ ip: await ipOn(started, last),
4566
+ netns: await netnsOf(started),
4567
+ exec: (command) => output(started, [...command])
4568
+ };
4569
+ },
3589
4570
  async agent(agentImage, argv, options = {}) {
3590
4571
  const Shared = shared(GenericContainer);
3591
4572
  const seen = lines();
@@ -3726,7 +4707,7 @@ const openLab = async (target, kind = "tcp", topology) => {
3726
4707
  };
3727
4708
  while (!confirmed()) {
3728
4709
  if (Date.now() - started > CONFIRM_MS) throw new Error(`the hop ${event.hop} did not confirm the write of ${event.write.path}: no line "${confirm}" in ${CONFIRM_MS / 1e3} s`);
3729
- await sleep$4(50);
4710
+ await sleep$6(50);
3730
4711
  }
3731
4712
  }
3732
4713
  return stamp;
@@ -3743,9 +4724,10 @@ const openLab = async (target, kind = "tcp", topology) => {
3743
4724
  query
3744
4725
  ]);
3745
4726
  },
3746
- async probe(args) {
4727
+ async probe(args, files = []) {
3747
4728
  const events = lines();
3748
- const container = await new GenericContainer(clientImage).withNetworkMode(`container:${client.getId()}`).withCommand([...command, ...args]).withLogConsumer(events.consume).start();
4729
+ const prober = new GenericContainer(clientImage).withNetworkMode(`container:${client.getId()}`).withCommand([...command, ...args]).withCopyContentToContainer(files.map((file) => ({ ...file }))).withLogConsumer(events.consume);
4730
+ const container = await (capabilities === void 0 ? prober : prober.withAddedCapabilities(...capabilities)).start();
3749
4731
  return {
3750
4732
  events: events.seen,
3751
4733
  until: (kinds, within) => events.until(kinds, within, "client"),
@@ -3846,6 +4828,7 @@ const openLab = async (target, kind = "tcp", topology) => {
3846
4828
  client: {
3847
4829
  kind: clientKind,
3848
4830
  image: clientImage,
4831
+ ...capabilities === void 0 ? {} : { capabilities },
3849
4832
  ...source === void 0 ? {} : {
3850
4833
  source,
3851
4834
  ...buildOf(source)
@@ -3864,33 +4847,240 @@ const openLab = async (target, kind = "tcp", topology) => {
3864
4847
  }
3865
4848
  };
3866
4849
  //#endregion
3867
- //#region src/pins.ts
3868
- /**
3869
- * This module tells if a manifest pins each program that a run operated. A record that does not pin
3870
- * them is not a record. A later run cannot operate the same programs, thus nobody can reproduce
3871
- * its numbers (SYS-101). Each image is pulled by its digest, or the harness builds it. A build
3872
- * records its bases by digest, the sha256 of each lockfile, and the versions of the packages that
3873
- * its manifests name. The commit of the code that operated is a field of the record of a cell, and
3874
- * `commitProblems` examines it.
3875
- */
3876
- /** This pattern matches the id of an image: `sha256:` and 64 hex digits. */
3877
- const IMAGE_ID = /^sha256:[0-9a-f]{64}$/;
3878
- /** This pattern matches the tag of a build of the harness: its name, then 12 hex digits of the sha256 of its context. */
3879
- const BUILD_TAG = /^[a-z0-9][a-z0-9._\-/]*:[0-9a-f]{12}$/;
3880
- /** This pattern matches a commit of git: 40 hex digits. */
3881
- const COMMIT = /^[0-9a-f]{40}$/;
3882
- /** This function gives each problem of the record of a build. A problem is a base that no digest pins, or a tag of another form. */
3883
- const buildProblems = (what, build) => [...BUILD_TAG.test(build.tag) ? [] : [`${what}: the build has the tag ${build.tag}, which is not a tag of a build`], ...build.bases.filter((base) => !PINNED.test(base)).map((base) => `${what}: the base ${base} is not pinned by digest`)];
4850
+ //#region src/standby.ts
3884
4851
  /**
3885
- * This function gives each problem of an image of a manifest. A pulled image needs a digest. An
3886
- * image that the harness builds needs its build record when the manifest can hold one (`build`).
3887
- * Otherwise, it needs the tag of a build and the id of its image.
4852
+ * This module is the second server of a pair (`Cell.pair`). The lab of the cell has the primary, the
4853
+ * topology in front of it, the injector and the prober. This module adds one standby on the network
4854
+ * of the primary, and it does one run:
4855
+ *
4856
+ * 1. The client writes a stream of sequence numbers through the topology.
4857
+ * 2. The fault cuts the primary from the standby, or from each peer. The cut stays to the end.
4858
+ * 3. At the end of the hold, the injector stops each process of the primary with SIGKILL.
4859
+ * 4. The harness promotes the standby, and reads what it holds.
4860
+ * 5. The harness starts the old primary, and reads what its crash recovery gives.
4861
+ *
4862
+ * The standby has no way to the WAL of the old primary, because the primary stops under the cut. A
4863
+ * promotion cannot be undone, thus each run has its own environment (`Cell.fresh`).
3888
4864
  */
3889
- const pinProblems = (what, image, id, build, needsBuild) => {
3890
- if (PINNED.test(image)) return [];
3891
- if (build !== void 0) return buildProblems(what, build);
3892
- if (BUILD_TAG.test(image) && !needsBuild && id !== void 0 && IMAGE_ID.test(id)) return [];
3893
- return BUILD_TAG.test(image) && needsBuild ? [`${what}: the image ${image} is a build with no build record`] : [`${what}: the image ${image} has no digest`];
4865
+ /** This table holds one row for each write of the client. */
4866
+ const TABLE = "writes";
4867
+ /** This time bounds each wait of the setup of the pair, in ms. */
4868
+ const SETUP_MS = 6e4;
4869
+ /** This count is the acknowledged writes before the fault: the stream is steady. */
4870
+ const STEADY$4 = 20;
4871
+ /** This time is the wait after the stop of the primary, in ms: each writer sends its last write in it. */
4872
+ const LAST_WRITE_MS = 2e3;
4873
+ /** This time bounds the wait for the last line of the client, in ms. */
4874
+ const STOP_MS = 5e3;
4875
+ /** This text is the line of the log of a standby that serves reads (src/backend/postmaster/postmaster.c:3740). */
4876
+ const STANDBY_READY = "database system is ready to accept read-only connections";
4877
+ /** This text is the write of the client, with its sequence number. */
4878
+ const INSERT = `INSERT INTO ${TABLE} VALUES ($n)`;
4879
+ const sleep$5 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
4880
+ /** This function gives the argv of one statement in psql, on the server in the same container. */
4881
+ const psql = (statement) => [
4882
+ "psql",
4883
+ "-h",
4884
+ "127.0.0.1",
4885
+ "-U",
4886
+ "postgres",
4887
+ "-At",
4888
+ "-c",
4889
+ statement
4890
+ ];
4891
+ /** This function gives the command of a standby: a base backup of the primary, then the server in recovery. */
4892
+ const standbyCommand = (primary) => [
4893
+ "sh",
4894
+ "-c",
4895
+ `chown postgres:postgres "$PGDATA"; chmod 0700 "$PGDATA"; until gosu postgres pg_basebackup -h ${primary} -U postgres -D "$PGDATA" -X stream -c fast -R; do gosu postgres sh -c 'rm -rf "$PGDATA"/*'; sleep 0.5; done; exec gosu postgres postgres`
4896
+ ];
4897
+ /** This command starts the old primary again, after the stop of each of its processes. Its crash recovery runs before the wait ends. */
4898
+ const RESTART = [
4899
+ "sh",
4900
+ "-c",
4901
+ "exec gosu postgres pg_ctl -D \"$PGDATA\" -l /tmp/restart.log -w -t 60 start"
4902
+ ];
4903
+ /** This function reads the output of a query for sequence numbers into a sorted list. */
4904
+ const numbersOf = (text) => text.split("\n").map((line) => line.trim()).filter((line) => line !== "").map(Number).sort((a, b) => a - b);
4905
+ /** This function gives the arguments of the client of a run. */
4906
+ const pairArgs = (cell, lab) => {
4907
+ const pair = cell.pair;
4908
+ const hold = cell.hold === void 0 ? 0 : valueIn(cell.hold, "s");
4909
+ return [
4910
+ lab.targetIp,
4911
+ String(lab.targetPort),
4912
+ "--mode",
4913
+ "stream",
4914
+ "--interval-ms",
4915
+ String(valueIn(pair.interval, "ms")),
4916
+ "--writers",
4917
+ String(pair.writers),
4918
+ "--query",
4919
+ INSERT,
4920
+ "--max-s",
4921
+ String(Math.ceil(hold) + 60),
4922
+ ...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
4923
+ ];
4924
+ };
4925
+ /** This function gives the peers of the cut of a pair: the standby, or the standby and the peer of the primary in the topology. */
4926
+ const peersOf$1 = (pair, standby, lab) => pair.cuts === "standby" ? [standby] : [standby, lab.topology?.targetPeer ?? lab.clientIp];
4927
+ /** This function gives the instants of the cancels of a hold, in ns from the fault: equal parts of the hold. */
4928
+ const cancelsOf = (pair, holdNs) => {
4929
+ const count = pair.cancels ?? 0;
4930
+ return Array.from({ length: count }, (_, at) => (at + 1) * holdNs / (count + 1));
4931
+ };
4932
+ /** This function asks until the primary reports the standby in a state of replication. */
4933
+ const untilStreaming = async (lab, state) => {
4934
+ const started = Date.now();
4935
+ for (;;) {
4936
+ const seen = await lab.exec(psql("SELECT sync_state FROM pg_stat_replication WHERE state = 'streaming'"));
4937
+ if (seen === state) return;
4938
+ if (Date.now() - started > SETUP_MS) throw new Error(`the standby did not stream as ${state} in ${SETUP_MS} ms: it reports "${seen}"`);
4939
+ await sleep$5(100);
4940
+ }
4941
+ };
4942
+ /**
4943
+ * This function opens the pair of an environment. The table comes first and the standby next: the
4944
+ * base backup has it. A synchronous standby is named last, thus no setup step waits for it.
4945
+ */
4946
+ const openStandby = async (lab, cell) => {
4947
+ const pair = cell.pair;
4948
+ await lab.exec([
4949
+ "sh",
4950
+ "-c",
4951
+ "echo \"host replication all all trust\" >> \"$PGDATA/pg_hba.conf\""
4952
+ ]);
4953
+ await lab.exec(psql("SELECT pg_reload_conf()"));
4954
+ await lab.exec(psql(`CREATE TABLE ${TABLE} (n integer PRIMARY KEY)`));
4955
+ const standby = await lab.sibling(await targetImageOf(cell.target), standbyCommand(lab.serverIp), STANDBY_READY);
4956
+ await untilStreaming(lab, "async");
4957
+ if (pair.synchronous) {
4958
+ await lab.exec(psql("ALTER SYSTEM SET synchronous_standby_names = '*'"));
4959
+ await lab.exec(psql("SELECT pg_reload_conf()"));
4960
+ await untilStreaming(lab, "sync");
4961
+ }
4962
+ return standby;
4963
+ };
4964
+ /** This function waits for a count of acknowledged writes. */
4965
+ const untilSteady = async (probe) => {
4966
+ const started = Date.now();
4967
+ while (probe.events.filter((event) => event.ev === "ack").length < STEADY$4) {
4968
+ if (Date.now() - started > SETUP_MS) throw new Error(`the client did not write ${STEADY$4} times in ${SETUP_MS} ms`);
4969
+ await sleep$5(10);
4970
+ }
4971
+ };
4972
+ /** This function gives the count of lost records in the latest clock line of the prober. */
4973
+ const droppedSoFar$5 = (records) => {
4974
+ for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
4975
+ return 0;
4976
+ };
4977
+ /**
4978
+ * This function does one run of a pair. A dry run writes and stops, with no fault: the warm-up of the
4979
+ * environment. It removes its rows, thus the numbers of the run start on an empty table.
4980
+ */
4981
+ const pairOnce = async (lab, standby, cell, environment, index, dry = false) => {
4982
+ const pair = cell.pair;
4983
+ await lab.forget();
4984
+ const from = lab.kernel.length;
4985
+ const lostBefore = droppedSoFar$5(lab.kernel);
4986
+ const probe = await lab.probe(pairArgs(cell, lab));
4987
+ const holdNs = cell.hold === void 0 ? 0 : valueIn(cell.hold, "s") * 1e9;
4988
+ let fault = null;
4989
+ let heal = null;
4990
+ try {
4991
+ await probe.until(["ready"], 15e3);
4992
+ await untilSteady(probe);
4993
+ if (!dry) {
4994
+ fault = await lab.fault("blackhole", "target", { peers: peersOf$1(pair, standby.ip, lab) });
4995
+ const started = Date.now();
4996
+ for (const at of cancelsOf(pair, holdNs)) {
4997
+ await sleep$5(Math.max(0, at / 1e6 - (Date.now() - started)));
4998
+ await lab.exec(psql("SELECT count(pg_cancel_backend(pid)) FROM pg_stat_activity WHERE wait_event = 'SyncRep'"));
4999
+ }
5000
+ heal = await lab.signal("KILL", fault.t0 + holdNs);
5001
+ await sleep$5(LAST_WRITE_MS);
5002
+ if (cell.target.kill !== void 0) await lab.exec(cell.target.kill);
5003
+ }
5004
+ await probe.go().catch(() => void 0);
5005
+ await probe.until(["done"], STOP_MS);
5006
+ } finally {
5007
+ await probe.stop();
5008
+ }
5009
+ if (dry) await lab.exec(psql(`TRUNCATE ${TABLE}`));
5010
+ const settled = await lab.settle();
5011
+ const lost = settled === null ? null : settled - lostBefore;
5012
+ const read = `SELECT n FROM ${TABLE} ORDER BY n`;
5013
+ let holds = null;
5014
+ let oldHolds = [];
5015
+ if (!dry) {
5016
+ await standby.exec(psql("SELECT pg_promote(true, 60)"));
5017
+ holds = numbersOf(await standby.exec(psql(read)));
5018
+ await lab.exec([...RESTART]);
5019
+ oldHolds = numbersOf(await lab.exec(psql(read)));
5020
+ }
5021
+ return {
5022
+ environment,
5023
+ index,
5024
+ events: [...probe.events],
5025
+ fault,
5026
+ ...heal === null ? {} : { heal },
5027
+ kernel: lab.kernel.slice(from),
5028
+ netns: lab.netns.client,
5029
+ targetNetns: lab.netns.target,
5030
+ lost,
5031
+ ...lab.topology === null ? {} : {
5032
+ topology: lab.topology,
5033
+ middlebox: []
5034
+ },
5035
+ group: {
5036
+ members: {
5037
+ primary: {
5038
+ ip: lab.serverIp,
5039
+ netns: lab.netns.target
5040
+ },
5041
+ standby: {
5042
+ ip: standby.ip,
5043
+ netns: standby.netns
5044
+ }
5045
+ },
5046
+ sentinels: {},
5047
+ primary: "primary",
5048
+ at: "primary",
5049
+ timeline: [],
5050
+ after: dry ? "primary" : "standby",
5051
+ holds,
5052
+ oldHolds
5053
+ }
5054
+ };
5055
+ };
5056
+ //#endregion
5057
+ //#region src/pins.ts
5058
+ /**
5059
+ * This module tells if a manifest pins each program that a run operated. A record that does not pin
5060
+ * them is not a record. A later run cannot operate the same programs, thus nobody can reproduce
5061
+ * its numbers (SYS-101). Each image is pulled by its digest, or the harness builds it. A build
5062
+ * records its bases by digest, the sha256 of each lockfile, and the versions of the packages that
5063
+ * its manifests name. The commit of the code that operated is a field of the record of a cell, and
5064
+ * `commitProblems` examines it.
5065
+ */
5066
+ /** This pattern matches the id of an image: `sha256:` and 64 hex digits. */
5067
+ const IMAGE_ID = /^sha256:[0-9a-f]{64}$/;
5068
+ /** This pattern matches the tag of a build of the harness: its name, then 12 hex digits of the sha256 of its context. */
5069
+ const BUILD_TAG = /^[a-z0-9][a-z0-9._\-/]*:[0-9a-f]{12}$/;
5070
+ /** This pattern matches a commit of git: 40 hex digits. */
5071
+ const COMMIT = /^[0-9a-f]{40}$/;
5072
+ /** This function gives each problem of the record of a build. A problem is a base that no digest pins, or a tag of another form. */
5073
+ const buildProblems = (what, build) => [...BUILD_TAG.test(build.tag) ? [] : [`${what}: the build has the tag ${build.tag}, which is not a tag of a build`], ...build.bases.filter((base) => !PINNED.test(base)).map((base) => `${what}: the base ${base} is not pinned by digest`)];
5074
+ /**
5075
+ * This function gives each problem of an image of a manifest. A pulled image needs a digest. An
5076
+ * image that the harness builds needs its build record when the manifest can hold one (`build`).
5077
+ * Otherwise, it needs the tag of a build and the id of its image.
5078
+ */
5079
+ const pinProblems = (what, image, id, build, needsBuild) => {
5080
+ if (PINNED.test(image)) return [];
5081
+ if (build !== void 0) return buildProblems(what, build);
5082
+ if (BUILD_TAG.test(image) && !needsBuild && id !== void 0 && IMAGE_ID.test(id)) return [];
5083
+ return BUILD_TAG.test(image) && needsBuild ? [`${what}: the image ${image} is a build with no build record`] : [`${what}: the image ${image} has no digest`];
3894
5084
  };
3895
5085
  /**
3896
5086
  * This function gives each problem of a manifest that stops it from pinning the programs of its
@@ -3911,6 +5101,21 @@ const manifestProblems = (manifest) => [
3911
5101
  ];
3912
5102
  /** This function gives the problem of the commit of a record: no commit, or a text that is not a commit of git. */
3913
5103
  const commitProblems = (commit) => commit === void 0 ? ["the record has no commit of the code that operated"] : COMMIT.test(commit) ? [] : [`the record has the commit ${commit}, which is not 40 hex digits`];
5104
+ /** This problem is the problem of a checkout with changes: its commit does not hold the code that operated. */
5105
+ const DIRTY = "the checkout has changes to tracked files that no commit holds";
5106
+ /** This problem is the problem of a record of a local debug run. */
5107
+ const DEBUG = "the record is of a local debug run, and no page reads it";
5108
+ /**
5109
+ * This function gives each problem that stops a record from pinning the programs of its runs. A
5110
+ * record with a problem is not a record (SYS-101). The writer of the cells refuses it, and the read
5111
+ * of a schedule refuses it. A commit with changes to tracked files is not a pin.
5112
+ */
5113
+ const recordProblems = (record) => [.../* @__PURE__ */ new Set([
5114
+ ...commitProblems(record.code?.commit),
5115
+ ...record.code?.dirty === true ? [DIRTY] : [],
5116
+ ...record.manifests.flatMap((manifest) => manifestProblems(manifest)),
5117
+ ...record.debug === true ? [DEBUG] : []
5118
+ ])];
3914
5119
  //#endregion
3915
5120
  //#region src/group.ts
3916
5121
  /** This function gives the configuration file of a member: its port, its settings, and the primary that a replica follows. */
@@ -3929,19 +5134,19 @@ const sentinelConfig = (group, primary) => [
3929
5134
  ].join("\n");
3930
5135
  /**
3931
5136
  * This function gives the peers that a fault on a part cuts it from, by their addresses. `all` is
3932
- * each other part and the client, `group` is the other parts, and `clients` is the client. `one`
3933
- * and `one-way` are the first other member. The observer is not a peer, thus the timeline of the
5137
+ * each other part and the client, `group` is the other parts, and `clients` and `to-clients` are
5138
+ * the client. `one` and `one-way` are the first other member. The observer is not a peer, thus the timeline of the
3934
5139
  * group stays whole.
3935
5140
  */
3936
5141
  const peersOf = (parts, client, part, cuts) => {
3937
5142
  const others = [...Object.entries(parts.members), ...Object.entries(parts.sentinels)].filter(([name]) => name !== part).map(([, at]) => at.ip);
3938
- if (cuts === "clients") return [client];
5143
+ if (cuts === "clients" || cuts === "to-clients") return [client];
3939
5144
  if (cuts === "group") return others;
3940
5145
  if (cuts === "one" || cuts === "one-way") return Object.entries(parts.members).filter(([name]) => name !== part).slice(0, 1).map(([, at]) => at.ip);
3941
5146
  return [...others, client];
3942
5147
  };
3943
- /** This function gives the direction of the rules of a cut: `one-way` drops only the segments from the part. */
3944
- const directionOf = (cuts) => cuts === "one-way" ? ["--direction", "out"] : [];
5148
+ /** This function gives the direction of the rules of a cut: `one-way` and `to-clients` drop only the segments from the part. */
5149
+ const directionOf = (cuts) => cuts === "one-way" || cuts === "to-clients" ? ["--direction", "out"] : [];
3945
5150
  /** This function reads the flat reply of a command line client into its fields: name, value, name, value. */
3946
5151
  const fieldsOf = (text) => {
3947
5152
  const out = [];
@@ -4034,7 +5239,7 @@ const viewOf = (group, ips, sentinels, members) => {
4034
5239
  why: "healthy"
4035
5240
  };
4036
5241
  };
4037
- const sleep$3 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
5242
+ const sleep$4 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
4038
5243
  /** This time is the longest that a group takes to converge before a run, or after it. */
4039
5244
  const CONVERGE_MS = 24e4;
4040
5245
  /**
@@ -4229,7 +5434,7 @@ const openGroupLab = async (group, options = {}) => {
4229
5434
  }));
4230
5435
  calm = last.healthy ? calm + 1 : 0;
4231
5436
  if (calm >= 2 && last.primary !== null) return last.primary;
4232
- await sleep$3(1e3);
5437
+ await sleep$4(1e3);
4233
5438
  }
4234
5439
  throw new Error(`the group did not converge in ${within} ms: ${last.why}`);
4235
5440
  },
@@ -4250,11 +5455,16 @@ const openGroupLab = async (group, options = {}) => {
4250
5455
  ]);
4251
5456
  },
4252
5457
  async holds(primary) {
4253
- return (await cli(primary, [
5458
+ return holdsOf(await cli(primary, [
4254
5459
  "--scan",
4255
5460
  "--pattern",
4256
5461
  "w:*"
4257
- ])).split("\n").map((key) => Number(key.trim().slice(2))).filter((n) => Number.isInteger(n) && n > 0).sort((a, b) => a - b);
5462
+ ]), await cli(primary, [
5463
+ "LRANGE",
5464
+ "w",
5465
+ "0",
5466
+ "-1"
5467
+ ]));
4258
5468
  },
4259
5469
  partOf: roles,
4260
5470
  async client(args) {
@@ -4309,7 +5519,7 @@ const openGroupLab = async (group, options = {}) => {
4309
5519
  const started = Date.now();
4310
5520
  while (await cli(member, ["PING"]).catch(() => "") !== "PONG") {
4311
5521
  if (Date.now() - started > 3e4) throw new Error(`${member} did not start again`);
4312
- await sleep$3(200);
5522
+ await sleep$4(200);
4313
5523
  }
4314
5524
  },
4315
5525
  failover(coordinated) {
@@ -4425,7 +5635,7 @@ const openGroupLab = async (group, options = {}) => {
4425
5635
  }
4426
5636
  };
4427
5637
  /** This count is the number of accepted writes that a run waits for before its fault: the stream is steady then. */
4428
- const STEADY$2 = 20;
5638
+ const STEADY$3 = 20;
4429
5639
  /** This time is how long a run continues after its outcome, so that the record shows what came next. */
4430
5640
  const TAIL_MS$3 = 2e3;
4431
5641
  /** This function gives the address of a member, with the port of the group. */
@@ -4435,22 +5645,32 @@ const deadlineOf = (cell) => {
4435
5645
  const { prediction } = cell;
4436
5646
  return (prediction.detects === true ? valueIn(prediction.high, "s") : prediction.detects === false && prediction.recovers !== void 0 ? valueIn(prediction.recovers.high, "s") : valueIn(prediction.window, "s")) * 1e3 + 1e4;
4437
5647
  };
5648
+ /**
5649
+ * This function gives the sequence numbers that a group holds, in order, from the output of its
5650
+ * command line. They are the keys `w:<n>` of a stream of SET, and the entries of the list `w` of a
5651
+ * stream of RPUSH. A list keeps each copy, thus a number can repeat.
5652
+ */
5653
+ const holdsOf = (keys, list) => [...keys.split("\n").map((key) => Number(key.trim().slice(2))), ...list.split("\n").map((entry) => Number(entry.trim()))].filter((n) => Number.isInteger(n) && n > 0).sort((a, b) => a - b);
4438
5654
  /** This function gives the arguments of the client of a group cell. */
4439
5655
  const clientArgs = (cell, lab) => {
5656
+ const sentinels = cell.group.sentinels.map((name) => `${lab.sentinels[name].ip}:${cell.group.sentinelPort}`);
5657
+ const driver = cell.probe.driver?.name === "ioredis" ? cell.probe.driver : void 0;
4440
5658
  return [
4441
5659
  "--sentinels",
4442
- cell.group.sentinels.map((name) => `${lab.sentinels[name].ip}:${cell.group.sentinelPort}`).join(","),
5660
+ sentinels.join(","),
4443
5661
  "--name",
4444
5662
  cell.group.name,
4445
5663
  "--interval-ms",
4446
5664
  String(cell.probe.interval === void 0 ? 50 : valueIn(cell.probe.interval, "ms")),
4447
5665
  "--max-s",
4448
5666
  String(Math.ceil(deadlineOf(cell) / 1e3 + 120)),
4449
- ...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
5667
+ ...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)],
5668
+ ...driver?.op === void 0 ? [] : ["--op", driver.op],
5669
+ ...driver?.window === void 0 ? [] : ["--window", String(driver.window)]
4450
5670
  ];
4451
5671
  };
4452
5672
  /** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
4453
- const waitFor$2 = (client, match, within) => new Promise((resolve) => {
5673
+ const waitFor$3 = (client, match, within) => new Promise((resolve) => {
4454
5674
  const started = Date.now();
4455
5675
  const look = () => {
4456
5676
  const found = client.events.find(match);
@@ -4466,7 +5686,7 @@ const subscriptionsOf = (timeline) => {
4466
5686
  return [...latest.values()];
4467
5687
  };
4468
5688
  /** This function gives the count of lost records in the latest clock line of the prober. */
4469
- const droppedSoFar$3 = (records) => {
5689
+ const droppedSoFar$4 = (records) => {
4470
5690
  for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
4471
5691
  return 0;
4472
5692
  };
@@ -4488,7 +5708,7 @@ const groupOnce = async (lab, cell, environment, index) => {
4488
5708
  await lab.forget();
4489
5709
  const from = lab.kernel.length;
4490
5710
  const timelineFrom = lab.timeline.length;
4491
- const lostBefore = droppedSoFar$3(lab.kernel);
5711
+ const lostBefore = droppedSoFar$4(lab.kernel);
4492
5712
  const client = await lab.client(clientArgs(cell, lab));
4493
5713
  const lands = cell.prediction.group?.lands ?? "new";
4494
5714
  const place = cell.faultAt ?? { role: "primary" };
@@ -4498,7 +5718,7 @@ const groupOnce = async (lab, cell, environment, index) => {
4498
5718
  const undo = [];
4499
5719
  try {
4500
5720
  await client.until(["ready"], 6e4);
4501
- if (await waitFor$2(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$2, 3e4) === void 0) throw new Error(`the client did not write ${STEADY$2} times in 30 s`);
5721
+ if (await waitFor$3(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$3, 3e4) === void 0) throw new Error(`the client did not write ${STEADY$3} times in 30 s`);
4502
5722
  let healing = null;
4503
5723
  switch (cell.fault) {
4504
5724
  case "blackhole":
@@ -4512,7 +5732,7 @@ const groupOnce = async (lab, cell, environment, index) => {
4512
5732
  undo.push(() => lab.lift(kind, part, cuts));
4513
5733
  if (cell.stall !== void 0) {
4514
5734
  const t1 = fault.t1;
4515
- const sent = await waitFor$2(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
5735
+ const sent = await waitFor$3(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
4516
5736
  if (sent !== void 0) healing = lab.lift(kind, part, cuts, sent.t + valueIn(cell.stall, "s") * 1e9);
4517
5737
  } else if (cell.hold !== void 0) healing = lab.lift(kind, part, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
4518
5738
  break;
@@ -4546,11 +5766,11 @@ const groupOnce = async (lab, cell, environment, index) => {
4546
5766
  const t0 = fault.t0;
4547
5767
  const old = addressOf(lab, primary);
4548
5768
  const deadline = deadlineOf(cell);
4549
- if ((lands === "new" ? await waitFor$2(client, (event) => event.ev === "ack" && event.t > t0 && typeof event["at"] === "string" && event["at"] !== old, deadline) : await (async () => {
5769
+ if ((lands === "new" ? await waitFor$3(client, (event) => event.ev === "ack" && event.t > t0 && typeof event["at"] === "string" && event["at"] !== old, deadline) : await (async () => {
4550
5770
  await healed;
4551
5771
  const after = heal?.t0 ?? t0;
4552
- return waitFor$2(client, (event) => event.ev === "ack" && event.t > after, deadline);
4553
- })()) !== void 0) await sleep$3(TAIL_MS$3);
5772
+ return waitFor$3(client, (event) => event.ev === "ack" && event.t > after, deadline);
5773
+ })()) !== void 0) await sleep$4(TAIL_MS$3);
4554
5774
  await healed;
4555
5775
  } finally {
4556
5776
  for (const step of undo.reverse()) await step().catch(() => void 0);
@@ -4643,6 +5863,10 @@ const cutOf = (set, members, client, part, cuts) => {
4643
5863
  peers: others.slice(0, 1),
4644
5864
  direction: "out"
4645
5865
  };
5866
+ if (cuts === "to-clients") return {
5867
+ peers: [client],
5868
+ direction: "out"
5869
+ };
4646
5870
  return { peers: [...others, client] };
4647
5871
  };
4648
5872
  /** This function gives the command of the server of each member. The server is the child of a shell, thus a fin can stop it and the container stays up. */
@@ -4714,7 +5938,7 @@ const statusViewOf = (set, ips, status) => {
4714
5938
  };
4715
5939
  };
4716
5940
  /** This function waits for a time, in ms. */
4717
- const sleep$2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
5941
+ const sleep$3 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
4718
5942
  /** These are the arguments of replSetStepDown: the time that the old primary cannot stand again, and the wait for a secondary to catch up. */
4719
5943
  const STEP_DOWN = {
4720
5944
  seconds: 60,
@@ -4862,7 +6086,7 @@ const openReplicaSetLab = async (set, backups = false) => {
4862
6086
  calm = last.healthy ? calm + 1 : 0;
4863
6087
  if (last.primary !== null) known = last.primary;
4864
6088
  if (calm >= 2 && last.primary !== null) return last.primary;
4865
- await sleep$2(1e3);
6089
+ await sleep$3(1e3);
4866
6090
  }
4867
6091
  throw new Error(`the set did not converge in ${within} ms: ${last.why}`);
4868
6092
  };
@@ -4968,7 +6192,7 @@ const openReplicaSetLab = async (set, backups = false) => {
4968
6192
  const started = Date.now();
4969
6193
  while (!(await shell(member, "print(db.hello().ok)").catch(() => "")).includes("1")) {
4970
6194
  if (Date.now() - started > 6e4) throw new Error(`${member} did not start again`);
4971
- await sleep$2(500);
6195
+ await sleep$3(500);
4972
6196
  }
4973
6197
  },
4974
6198
  exec(member, argv) {
@@ -5071,6 +6295,8 @@ const openReplicaSetLab = async (set, backups = false) => {
5071
6295
  };
5072
6296
  /** This time is how long a run continues after its outcome, so that the record shows what came next. */
5073
6297
  const TAIL_MS$2 = 2e3;
6298
+ /** This function gives how long a run of a cell continues after its lift, in ms: the tail, and the watch of the cell. */
6299
+ const afterLiftOf = (cell) => TAIL_MS$2 + (cell.watch === void 0 ? 0 : valueIn(cell.watch, "s") * 1e3);
5074
6300
  /** This function gives the arguments of the client of a replica set cell: each member, and the stream of writes. */
5075
6301
  const replicaSetArgs = (cell, members) => {
5076
6302
  const set = cell.replicaSet;
@@ -5094,7 +6320,7 @@ const replicaSetArgs = (cell, members) => {
5094
6320
  ];
5095
6321
  };
5096
6322
  /** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
5097
- const waitFor$1 = (client, match, within) => new Promise((resolve) => {
6323
+ const waitFor$2 = (client, match, within) => new Promise((resolve) => {
5098
6324
  const started = Date.now();
5099
6325
  const look = () => {
5100
6326
  const found = client.events.find(match);
@@ -5104,7 +6330,7 @@ const waitFor$1 = (client, match, within) => new Promise((resolve) => {
5104
6330
  look();
5105
6331
  });
5106
6332
  /** This function gives the count of lost records in the latest clock line of the prober. */
5107
- const droppedSoFar$2 = (records) => {
6333
+ const droppedSoFar$3 = (records) => {
5108
6334
  for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
5109
6335
  return 0;
5110
6336
  };
@@ -5129,7 +6355,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
5129
6355
  await lab.forget();
5130
6356
  const from = lab.kernel.length;
5131
6357
  const logFrom = lab.log.length;
5132
- const lostBefore = droppedSoFar$2(lab.kernel);
6358
+ const lostBefore = droppedSoFar$3(lab.kernel);
5133
6359
  const client = await lab.client(replicaSetArgs(cell, lab.members));
5134
6360
  const lands = cell.prediction.group?.lands ?? "new";
5135
6361
  const place = cell.faultAt ?? { role: "primary" };
@@ -5139,7 +6365,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
5139
6365
  const undo = [];
5140
6366
  try {
5141
6367
  await client.until(["ready"], 6e4);
5142
- if (await waitFor$1(client, (event) => event.ev === "ack" && Number(event["n"]) >= 20, 3e4) === void 0) throw new Error(`the client did not write 20 times in 30 s`);
6368
+ if (await waitFor$2(client, (event) => event.ev === "ack" && Number(event["n"]) >= 20, 3e4) === void 0) throw new Error(`the client did not write 20 times in 30 s`);
5143
6369
  let healing = null;
5144
6370
  switch (cell.fault) {
5145
6371
  case "blackhole":
@@ -5153,7 +6379,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
5153
6379
  undo.push(() => lab.lift(kind, part, cuts));
5154
6380
  if (cell.stall !== void 0) {
5155
6381
  const t1 = fault.t1;
5156
- const sent = await waitFor$1(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
6382
+ const sent = await waitFor$2(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
5157
6383
  healing = lab.lift(kind, part, cuts, (sent?.t ?? t1) + valueIn(cell.stall, "s") * 1e9);
5158
6384
  } else if (cell.hold !== void 0) healing = lab.lift(kind, part, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
5159
6385
  break;
@@ -5188,10 +6414,10 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
5188
6414
  const t1 = fault.t1;
5189
6415
  const old = `${lab.members[primary].ip}:${set.port}`;
5190
6416
  const sentAt = (n) => client.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
5191
- if (await waitFor$1(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1 && (lands === "same" || typeof event["at"] === "string" && event["at"] !== old), deadlineOf(cell)) !== void 0) await sleep$2(TAIL_MS$2);
6417
+ if (await waitFor$2(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1 && (lands === "same" || typeof event["at"] === "string" && event["at"] !== old), deadlineOf(cell)) !== void 0) await sleep$3(TAIL_MS$2);
5192
6418
  if (healing !== null) {
5193
6419
  await healed.catch(() => void 0);
5194
- await sleep$2(TAIL_MS$2);
6420
+ await sleep$3(afterLiftOf(cell));
5195
6421
  }
5196
6422
  } finally {
5197
6423
  for (const step of undo.reverse()) await step().catch(() => void 0);
@@ -5249,7 +6475,7 @@ const until = async (what, check, within) => {
5249
6475
  const started = Date.now();
5250
6476
  while (!await check().catch(() => false)) {
5251
6477
  if (Date.now() - started > within) throw new Error(`${what} did not come in ${within} ms`);
5252
- await sleep$2(500);
6478
+ await sleep$3(500);
5253
6479
  }
5254
6480
  };
5255
6481
  /** This function gives the time that the client sent a write. Zero is no write. */
@@ -5307,7 +6533,7 @@ const backupsOf = async (lab, drill, primary, writer) => {
5307
6533
  }];
5308
6534
  const every = valueIn(drill.sliceEvery, "s") * 1e3;
5309
6535
  for (let slice = 1; slice <= drill.slices; slice += 1) {
5310
- await sleep$2(Math.max(0, ended + slice * every - Date.now()));
6536
+ await sleep$3(Math.max(0, ended + slice * every - Date.now()));
5311
6537
  const out = `${VAULT}/slice-${slice}`;
5312
6538
  const begun = Date.now();
5313
6539
  await lab.vault([
@@ -5351,7 +6577,7 @@ const lose = async (lab, drill, primary, last) => {
5351
6577
  const parts = drill.recovery === "restore" ? [primary, ...lab.set.members.filter((name) => name !== primary)] : [primary];
5352
6578
  const age = valueIn(drill.age, "s");
5353
6579
  const at = last === null || drill.quiet === true ? void 0 : last.t + age * 1e9;
5354
- if (at === void 0) await sleep$2(age * 1e3);
6580
+ if (at === void 0) await sleep$3(age * 1e3);
5355
6581
  const stamps = await Promise.all(parts.map((part) => lab.signal(part, "KILL", at)));
5356
6582
  for (const part of parts) {
5357
6583
  await lab.exec(part, [
@@ -5442,7 +6668,7 @@ const drillOnce = async (lab, cell, environment, index) => {
5442
6668
  const tool = (await lab.vault(["mongodump", "--version"])).split("\n")[0].trim();
5443
6669
  const from = lab.kernel.length;
5444
6670
  const logFrom = lab.log.length;
5445
- const lostBefore = droppedSoFar$2(lab.kernel);
6671
+ const lostBefore = droppedSoFar$3(lab.kernel);
5446
6672
  const writer = await lab.client(replicaSetArgs(cell, lab.members));
5447
6673
  const restores = drill.recovery === "restore";
5448
6674
  let successor = null;
@@ -5452,7 +6678,7 @@ const drillOnce = async (lab, cell, environment, index) => {
5452
6678
  const backups = [];
5453
6679
  try {
5454
6680
  await writer.until(["ready"], 6e4);
5455
- if (await waitFor$1(writer, (event) => event.ev === "ack" && Number(event["n"]) >= 20, 3e4) === void 0) throw new Error(`the client did not write 20 times in 30 s`);
6681
+ if (await waitFor$2(writer, (event) => event.ev === "ack" && Number(event["n"]) >= 20, 3e4) === void 0) throw new Error(`the client did not write 20 times in 30 s`);
5456
6682
  if (drill.quiet === true) await writer.stop();
5457
6683
  if (restores) backups.push(...await backupsOf(lab, drill, primary, writer));
5458
6684
  const stamp = await lose(lab, drill, primary, backups.at(-1)?.last ?? null);
@@ -5466,11 +6692,11 @@ const drillOnce = async (lab, cell, environment, index) => {
5466
6692
  "--from",
5467
6693
  String(second)
5468
6694
  ]);
5469
- if (await waitFor$1(successor, (event) => event.ev === "ack", deadlineOf(cell)) !== void 0) await sleep$2(TAIL_MS$2);
6695
+ if (await waitFor$2(successor, (event) => event.ev === "ack", deadlineOf(cell)) !== void 0) await sleep$3(TAIL_MS$2);
5470
6696
  } else {
5471
6697
  const old = `${lab.members[primary].ip}:${set.port}`;
5472
6698
  const sentAt = (n) => writer.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
5473
- if (await waitFor$1(writer, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > stamp.t1 && typeof event["at"] === "string" && event["at"] !== old, deadlineOf(cell)) !== void 0) await sleep$2(TAIL_MS$2);
6699
+ if (await waitFor$2(writer, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > stamp.t1 && typeof event["at"] === "string" && event["at"] !== old, deadlineOf(cell)) !== void 0) await sleep$3(TAIL_MS$2);
5474
6700
  await lab.restart(primary);
5475
6701
  }
5476
6702
  } finally {
@@ -5632,6 +6858,7 @@ const clusterViewOf = (cluster, rows) => {
5632
6858
  * This function gives the peers that a fault on a node cuts it from. `all` is each other node and the
5633
6859
  * client, `group` is the other nodes, and `clients` is the client. `one` is the first other node in
5634
6860
  * the sequence of the cluster, and `one-way` is the same node for the segments from the part only.
6861
+ * `to-clients` is the client, for the segments from the part only.
5635
6862
  */
5636
6863
  const clusterCutOf = (cluster, members, client, part, cuts) => {
5637
6864
  const others = cluster.members.filter((name) => name !== part).map((name) => members[name].ip);
@@ -5642,9 +6869,13 @@ const clusterCutOf = (cluster, members, client, part, cuts) => {
5642
6869
  peers: others.slice(0, 1),
5643
6870
  direction: "out"
5644
6871
  };
6872
+ if (cuts === "to-clients") return {
6873
+ peers: [client],
6874
+ direction: "out"
6875
+ };
5645
6876
  return { peers: [...others, client] };
5646
6877
  };
5647
- const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
6878
+ const sleep$2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
5648
6879
  /**
5649
6880
  * This function opens one environment of a cluster cell:
5650
6881
  * - one new network, with each node, the client and the admin on it.
@@ -5806,7 +7037,7 @@ const openClusterLab = async (cluster) => {
5806
7037
  calm = last.healthy ? calm + 1 : 0;
5807
7038
  if (last.primary !== null) known = last.primary;
5808
7039
  if (calm >= 2 && last.primary !== null) return last.primary;
5809
- await sleep$1(1e3);
7040
+ await sleep$2(1e3);
5810
7041
  }
5811
7042
  throw new Error(`the queue did not converge in ${within} ms: ${last.why}`);
5812
7043
  };
@@ -5822,7 +7053,7 @@ const openClusterLab = async (cluster) => {
5822
7053
  const running = (text.trim() === "" ? {} : toolJsonOf(text)).running_nodes ?? [];
5823
7054
  if (cluster.members.every((name) => running.includes(nodeNameOf(name)))) break;
5824
7055
  if (Date.now() - started > 12e4) throw new Error(`the cluster did not form in 120 s: ${running.join(", ")} run`);
5825
- await sleep$1(1e3);
7056
+ await sleep$2(1e3);
5826
7057
  }
5827
7058
  const declared = lastLine(await output(commander, [
5828
7059
  "node",
@@ -5942,7 +7173,7 @@ const openClusterLab = async (cluster) => {
5942
7173
  "check_running"
5943
7174
  ]).then(() => true).catch(() => false)) {
5944
7175
  if (Date.now() - begun > 12e4) throw new Error(`${member} did not start again`);
5945
- await sleep$1(1e3);
7176
+ await sleep$2(1e3);
5946
7177
  }
5947
7178
  },
5948
7179
  stepDown(leader, to) {
@@ -6044,7 +7275,7 @@ const openClusterLab = async (cluster) => {
6044
7275
  }
6045
7276
  };
6046
7277
  /** This count is the number of confirmed publishes that a run waits for before its fault: the stream is steady then. */
6047
- const STEADY = 20;
7278
+ const STEADY$1 = 20;
6048
7279
  /** This time is how long a run continues after its outcome, so that the record shows what came next. */
6049
7280
  const TAIL_MS$1 = 2e3;
6050
7281
  /**
@@ -6073,7 +7304,7 @@ const clusterArgs = (cell, members, leader) => [
6073
7304
  ...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
6074
7305
  ];
6075
7306
  /** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
6076
- const waitFor = (client, match, within) => new Promise((resolve) => {
7307
+ const waitFor$1 = (client, match, within) => new Promise((resolve) => {
6077
7308
  const started = Date.now();
6078
7309
  const look = () => {
6079
7310
  const found = client.events.find(match);
@@ -6083,7 +7314,7 @@ const waitFor = (client, match, within) => new Promise((resolve) => {
6083
7314
  look();
6084
7315
  });
6085
7316
  /** This function gives the count of lost records in the latest clock line of the prober. */
6086
- const droppedSoFar$1 = (records) => {
7317
+ const droppedSoFar$2 = (records) => {
6087
7318
  for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
6088
7319
  return 0;
6089
7320
  };
@@ -6113,7 +7344,7 @@ const stampOf = async (lab, client, logFrom, marks, wall) => {
6113
7344
  },
6114
7345
  line
6115
7346
  };
6116
- await sleep$1(100);
7347
+ await sleep$2(100);
6117
7348
  }
6118
7349
  return {
6119
7350
  stamp: monotonicOf(wall, client.events),
@@ -6150,7 +7381,7 @@ const clusterOnce = async (lab, cell, environment, index) => {
6150
7381
  await lab.forget();
6151
7382
  const from = lab.kernel.length;
6152
7383
  const logFrom = lab.log.length;
6153
- const lostBefore = droppedSoFar$1(lab.kernel);
7384
+ const lostBefore = droppedSoFar$2(lab.kernel);
6154
7385
  const client = await lab.client(clusterArgs(cell, lab.members, leader));
6155
7386
  const place = cell.faultAt ?? { role: "primary" };
6156
7387
  let fault = null;
@@ -6161,7 +7392,7 @@ const clusterOnce = async (lab, cell, environment, index) => {
6161
7392
  const undo = [];
6162
7393
  try {
6163
7394
  await client.until(["ready"], 6e4);
6164
- if (await waitFor(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY, 3e4) === void 0) throw new Error(`the client did not publish ${STEADY} times in 30 s`);
7395
+ if (await waitFor$1(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$1, 3e4) === void 0) throw new Error(`the client did not publish ${STEADY$1} times in 30 s`);
6165
7396
  let healing = null;
6166
7397
  switch (cell.fault) {
6167
7398
  case "blackhole":
@@ -6215,10 +7446,10 @@ const clusterOnce = async (lab, cell, environment, index) => {
6215
7446
  healed.catch(() => void 0);
6216
7447
  const t1 = fault.t1;
6217
7448
  const sentAt = (n) => client.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
6218
- if (await waitFor(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1, deadlineOf(cell)) !== void 0) await sleep$1(TAIL_MS$1);
7449
+ if (await waitFor$1(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1, deadlineOf(cell)) !== void 0) await sleep$2(TAIL_MS$1);
6219
7450
  if (healing !== null) {
6220
7451
  await healed.catch(() => void 0);
6221
- await sleep$1(TAIL_MS$1);
7452
+ await sleep$2(TAIL_MS$1);
6222
7453
  }
6223
7454
  } finally {
6224
7455
  for (const step of undo.reverse()) await step().catch(() => void 0);
@@ -6250,129 +7481,767 @@ const clusterOnce = async (lab, cell, environment, index) => {
6250
7481
  };
6251
7482
  };
6252
7483
  //#endregion
6253
- //#region src/run.ts
7484
+ //#region src/kafka.ts
6254
7485
  /**
6255
- * This function measures and judges the raw records of a cell, with the code of the harness at
6256
- * this time. `runCell` uses it during a cell. A holder of the raw records of a cell can use it
6257
- * again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.
7486
+ * This module is the lab of a group of Kafka brokers in KRaft mode with one controller. It opens one
7487
+ * environment for a Kafka cell, for the runs of the cell. Each node is a container from the stock
7488
+ * image, in its own network namespace, and an injector sidecar shares its network and process
7489
+ * namespaces. The image starts by its own command, and each setting comes from the environment.
7490
+ *
7491
+ * Each run makes one new topic, thus each run starts with an empty log and its client counts from 1.
7492
+ * The client is the producer of kafka-clients. The commander holds the admin, on the network but
7493
+ * not in the namespace that the prober watches. The record of a run is the lines of the client and
7494
+ * the whole partition, read to its high watermark after the group converged.
6258
7495
  */
6259
- const judge = (cell, runs) => {
6260
- const measured = runs.filter((run) => run.warmup !== true).map((run) => measure(cell, run));
6261
- return {
6262
- measured,
6263
- verdict: verdictOf(cell.prediction, measured, noiseOf(cell))
6264
- };
6265
- };
6266
- /** This function gives the count of lost records in the latest clock line of the prober, since it began. */
6267
- const droppedSoFar = (records) => {
6268
- for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
6269
- return 0;
6270
- };
6271
- /** This function gives the argv of the client for a cell, with a deadline far after all that it waits for. */
6272
- const probeArgs = (cell, lab) => {
6273
- const { prediction, probe } = cell;
6274
- const longest = (prediction.detects === true && prediction.driver !== void 0 ? longestOf(cell) : prediction.detects ? latestEndOf(prediction) : valueIn(prediction.window, "s")) + (cell.idleBefore === void 0 ? 0 : valueIn(cell.idleBefore, "s"));
6275
- const args = [
6276
- lab.targetIp,
6277
- String(lab.targetPort),
6278
- "--mode",
6279
- probe.mode
6280
- ];
6281
- if (probe.ws !== void 0) {
6282
- const { ws } = probe;
6283
- const waits = (prediction.detects !== false ? prediction.reconnects?.waits ?? [] : []).reduce((sum, wait) => sum + valueIn(wait.high, "s"), 0);
6284
- args.push("--websocket", ws.socket);
6285
- args.push("--heartbeat", ws.heartbeat === null ? "off" : "on");
6286
- if (ws.heartbeat !== null) args.push("--heartbeat-interval-ms", String(valueIn(ws.heartbeat.interval, "ms")), "--heartbeat-timeout-ms", String(valueIn(ws.heartbeat.timeout, "ms")), "--heartbeat-misses", String(ws.heartbeat.misses));
6287
- args.push("--reconnect", ws.reconnect === null ? "off" : "on");
6288
- if (ws.reconnect !== null) args.push("--reconnect-initial-ms", String(valueIn(ws.reconnect.initialDelay, "ms")), "--reconnect-multiplier", String(ws.reconnect.multiplier), "--reconnect-max-ms", String(valueIn(ws.reconnect.maxDelay, "ms")), "--reconnect-attempts", String(ws.reconnect.attempts), "--reconnect-jitter", String(ws.reconnect.jitter));
6289
- args.push("--request-timeout-ms", String(valueIn(ws.requestTimeout, "ms")));
6290
- args.push("--connection-timeout-ms", String(valueIn(ws.connectionTimeout, "ms")));
6291
- if (ws.hold !== void 0) args.push("--hold-ms", String(valueIn(ws.hold, "ms")));
6292
- args.push("--max-s", String(Math.ceil((longest + waits) * 2 + 10)));
6293
- return args;
6294
- }
6295
- if (probe.http !== void 0) {
6296
- const { http } = probe;
6297
- const gap = http.gap === void 0 ? 0 : valueIn(http.gap, "s");
6298
- args.push("--method", http.method);
6299
- if (http.gap !== void 0) args.push("--gap-ms", String(valueIn(http.gap, "ms")));
6300
- if (http.hold !== void 0) args.push("--hold-ms", String(valueIn(http.hold, "ms")));
6301
- if (http.bytes !== void 0) args.push("--bytes", String(http.bytes));
6302
- for (const [name, value] of Object.entries(http.options ?? {})) args.push(`--${name}`, String(value));
6303
- args.push("--linger-ms", String(LINGER_MS));
6304
- args.push("--max-s", String(Math.ceil(longest * 2 + gap + 10)));
6305
- return args;
6306
- }
6307
- if (probe.request !== void 0) args.push("--request", probe.request);
6308
- if (probe.count !== void 0) args.push("--count", String(probe.count));
6309
- if (probe.interval !== void 0) args.push("--interval-ms", String(valueIn(probe.interval, "ms")));
6310
- if (probe.driver?.name === "mongodb") {
6311
- args.push("--set", probe.driver.set);
6312
- if (probe.driver.op !== void 0) args.push("--op", probe.driver.op);
6313
- if (cell.writeConcern !== void 0) args.push("--w", String(cell.writeConcern));
6314
- if (cell.before === true) args.push("--before");
6315
- if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
6316
- args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
6317
- return args;
6318
- }
6319
- if (probe.driver?.name === "pg" || probe.driver?.name === "hikari" || probe.driver?.name === "amqp") {
6320
- if (probe.driver.name !== "amqp" && probe.driver.query !== void 0) args.push("--query", probe.driver.query);
6321
- if (probe.driver.name !== "amqp" && probe.driver.inflight !== void 0) args.push("--inflight-query", probe.driver.inflight);
6322
- if (cell.before === true) args.push("--before");
6323
- if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
6324
- args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
6325
- return args;
6326
- }
6327
- if (probe.driver?.name === "grpc") {
6328
- if (probe.driver.hold !== void 0) args.push("--hold-ms", String(valueIn(probe.driver.hold, "ms")));
6329
- if (probe.driver.op !== void 0) args.push("--op", probe.driver.op);
6330
- if (cell.before === true) args.push("--before");
6331
- if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
6332
- args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
6333
- return args;
6334
- }
6335
- if (probe.driver !== void 0) {
6336
- args.push("--keepalive-ms", String(valueIn(probe.driver.keepAlive, "ms")));
6337
- args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
6338
- return args;
6339
- }
6340
- if (probe.keepalive) args.push("--keepalive");
6341
- if (probe.userTimeout !== void 0) args.push("--user-timeout-ms", String(valueIn(probe.userTimeout, "ms")));
6342
- args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
6343
- return args;
7496
+ /** This path is the class path of the Kafka image: the JVM expands the star itself. */
7497
+ const CLASS_PATH = "/opt/harness/kafka/classes:/opt/harness/kafka/lib/*";
7498
+ /** This option is the heap of the client and of the admin. It has no source: the two hold a producer or a read of one partition. */
7499
+ const CLIENT_HEAP = "-Xmx256m";
7500
+ /** This function gives the node id of a name. A broker has its place in the list from 1, and the controller has the id after them. */
7501
+ const nodeIdOf = (kafka, name) => {
7502
+ const at = kafka.brokers.indexOf(name);
7503
+ if (at !== -1) return at + 1;
7504
+ if (name === kafka.controller) return kafka.brokers.length + 1;
7505
+ throw new Error(`the Kafka group has no node ${name}`);
6344
7506
  };
6345
7507
  /**
6346
- * This function waits for an error on the socket of the observer, or for its end of file
6347
- * (CLOSE_WAIT, state 8). The prober records both. A cell whose observer is the server, or a proxy
6348
- * of a topology, waits for that. A target with its own detector can also drop the connection
6349
- * itself (`dropped`). The promise resolves after `within` ms at the latest.
7508
+ * This function gives the variable of a server property for the image. The wrapper of the image
7509
+ * reverses this map (KafkaDockerWrapper.scala:188-203). After the prefix `KAFKA_`, `___` is a
7510
+ * hyphen, `__` is an underscore and `_` is a dot.
6350
7511
  */
6351
- const observerError = (cell, partial, within, dropped = () => false) => new Promise((resolve) => {
6352
- const started = Date.now();
6353
- const look = () => {
6354
- const run = partial();
6355
- const at = socketOf(cell, run);
6356
- if (at !== void 0 && run.kernel.some((record) => record.netns === at.netns && record.cookie === at.cookie && (record.ev === "error" && (record.err ?? 0) !== 0 || record.ev === "state" && record.new === 8)) || dropped() || Date.now() - started > within) resolve();
6357
- else setTimeout(look, 50);
6358
- };
6359
- look();
7512
+ const envOf = (property) => `KAFKA_${property.replace(/_/g, "__").replace(/-/g, "___").replace(/\./g, "_").toUpperCase()}`;
7513
+ /** This function gives the variables of each node: the listener names, the voters, the log and the settings of the group. */
7514
+ const commonEnv = (kafka) => ({
7515
+ CLUSTER_ID: kafka.clusterId,
7516
+ KAFKA_HEAP_OPTS: kafka.heap,
7517
+ KAFKA_CONTROLLER_QUORUM_VOTERS: `${nodeIdOf(kafka, kafka.controller)}@${kafka.controller}:${kafka.controllerPort}`,
7518
+ KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER",
7519
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT",
7520
+ KAFKA_LOG_DIRS: "/var/lib/kafka/data",
7521
+ ...Object.fromEntries(Object.entries(kafka.settings).map(([property, value]) => [envOf(property), value]))
7522
+ });
7523
+ /** This function gives the environment of a broker. */
7524
+ const brokerEnv = (kafka, name) => ({
7525
+ ...commonEnv(kafka),
7526
+ KAFKA_PROCESS_ROLES: "broker",
7527
+ KAFKA_NODE_ID: String(nodeIdOf(kafka, name)),
7528
+ KAFKA_LISTENERS: `PLAINTEXT://0.0.0.0:${kafka.port}`,
7529
+ KAFKA_ADVERTISED_LISTENERS: `PLAINTEXT://${name}:${kafka.port}`,
7530
+ KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
7531
+ });
7532
+ /** This function gives the environment of the controller. The start script of the image stops on an advertised listener here (configure:43-48). */
7533
+ const controllerEnv = (kafka) => ({
7534
+ ...commonEnv(kafka),
7535
+ KAFKA_PROCESS_ROLES: "controller",
7536
+ KAFKA_NODE_ID: String(nodeIdOf(kafka, kafka.controller)),
7537
+ KAFKA_LISTENERS: `CONTROLLER://0.0.0.0:${kafka.controllerPort}`
6360
7538
  });
6361
- const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
6362
- /**
6363
- * This time is the pause between the client's first reply and the fault. It is more than Linux's
6364
- * TCP_DELACK_MAX of 200 ms. Thus the delayed ACK of the reply is out before the fault.
6365
- */
6366
- const QUIET_MS = 250;
6367
7539
  /**
6368
- * This time is how long an HTTP client waits after its second request, before it exits. The exit
6369
- * closes its sockets, thus that close comes after the window of the cell.
7540
+ * This function judges a view of a partition: its leader, and if the group is healthy. A healthy
7541
+ * group has each broker not fenced, one leader, and each broker in the in-sync replicas.
6370
7542
  */
6371
- const LINGER_MS = 1e3;
6372
- /** These are the lines that end the workload of an HTTP client: the end of its second request, its deadline, or its failure. */
6373
- const HTTP_DONE = [
6374
- "done",
6375
- "deadline",
7543
+ const kafkaViewOf = (kafka, view) => {
7544
+ const primary = view.leader === void 0 || view.leader === null ? null : kafka.brokers[view.leader - 1] ?? null;
7545
+ const all = kafka.brokers.length;
7546
+ if (view.brokers.length !== all) return {
7547
+ primary,
7548
+ healthy: false,
7549
+ why: `${view.brokers.length} brokers are not fenced`
7550
+ };
7551
+ if (primary === null) return {
7552
+ primary,
7553
+ healthy: false,
7554
+ why: "the partition has no leader"
7555
+ };
7556
+ if (view.replicas?.length !== all) return {
7557
+ primary,
7558
+ healthy: false,
7559
+ why: `the partition has ${view.replicas?.length ?? 0} replicas`
7560
+ };
7561
+ if (view.isr?.length !== all) return {
7562
+ primary,
7563
+ healthy: false,
7564
+ why: `the partition has ${view.isr?.length ?? 0} in-sync replicas`
7565
+ };
7566
+ return {
7567
+ primary,
7568
+ healthy: true,
7569
+ why: "healthy"
7570
+ };
7571
+ };
7572
+ /**
7573
+ * This function gives the peers that a fault on a broker cuts it from. Only `group` has a law: the
7574
+ * other brokers and the controller, and the client keeps its route to the broker.
7575
+ */
7576
+ const kafkaCutOf = (kafka, members, controller, part, cuts) => {
7577
+ if (cuts !== "group") throw new Error(`a Kafka cell has no cut ${cuts}`);
7578
+ return { peers: [...kafka.brokers.filter((name) => name !== part).map((name) => members[name].ip), controller.ip] };
7579
+ };
7580
+ /** This function gives the addresses that the client and the admin bootstrap from. */
7581
+ const bootstrapOf = (kafka) => kafka.brokers.map((name) => `${name}:${kafka.port}`).join(",");
7582
+ /** This function gives the name of the topic of a run. A warm-up has the index -1. */
7583
+ const topicOf = (environment, index) => `cells-e${environment}-r${index + 1}`;
7584
+ const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
7585
+ /**
7586
+ * This function opens one environment of a Kafka cell:
7587
+ * - one new network, with each node, the client and the commander on it.
7588
+ * - the controller first, then each broker: the stock image with its own command. The brokers
7589
+ * register with the controller, thus they need it first.
7590
+ * - an injector sidecar for each node, in its network and process namespaces.
7591
+ * - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
7592
+ * - the prober, which watches the namespace of the client.
7593
+ */
7594
+ const openKafkaLab = async (kafka) => {
7595
+ const image = await harnessImage();
7596
+ const clientImage = await kafkaClientImage();
7597
+ const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();
7598
+ const runtime = await getContainerRuntimeClient();
7599
+ const opened = [];
7600
+ let network;
7601
+ const close = async () => {
7602
+ for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
7603
+ await network?.stop().catch(() => void 0);
7604
+ };
7605
+ const ipOf = async (container) => {
7606
+ const ip = (await runtime.container.inspect(runtime.container.getById(container.getId()))).NetworkSettings.Networks[network.getName()]?.IPAddress;
7607
+ if (ip === void 0 || ip === "") throw new Error(`${container.getName()} has no address on ${network.getName()}`);
7608
+ return ip;
7609
+ };
7610
+ try {
7611
+ network = await new Network().start();
7612
+ const started = network;
7613
+ const boxes = /* @__PURE__ */ new Map();
7614
+ const injectors = /* @__PURE__ */ new Map();
7615
+ const parts = {};
7616
+ const raw = {};
7617
+ const fences = [];
7618
+ const openNode = async (name, environment) => {
7619
+ let partial = "";
7620
+ const box = await new GenericContainer(kafka.image).withNetwork(started).withNetworkAliases(name).withHostname(name).withEnvironment(environment).withWaitStrategy(Wait.forLogMessage(kafka.ready)).withStartupTimeout(18e4).withLogConsumer((stream) => stream.on("data", (chunk) => {
7621
+ partial += chunk.toString();
7622
+ const split = partial.split("\n");
7623
+ partial = split.pop() ?? "";
7624
+ for (const text of split) {
7625
+ const tail = raw[name] ??= [];
7626
+ tail.push(text);
7627
+ if (tail.length > 400) tail.shift();
7628
+ if (isFenceLine(text)) fences.push(text);
7629
+ }
7630
+ })).start();
7631
+ opened.push(box);
7632
+ boxes.set(name, box);
7633
+ const injector = await withPid(new GenericContainer(image).withNetworkMode(`container:${box.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW"), `container:${box.getId()}`).start();
7634
+ opened.push(injector);
7635
+ injectors.set(name, injector);
7636
+ parts[name] = {
7637
+ ip: await ipOf(box),
7638
+ netns: await netnsOf(injector)
7639
+ };
7640
+ };
7641
+ await openNode(kafka.controller, controllerEnv(kafka));
7642
+ for (const name of kafka.brokers) await openNode(name, brokerEnv(kafka, name));
7643
+ const members = Object.fromEntries(kafka.brokers.map((name) => [name, parts[name]]));
7644
+ const controller = parts[kafka.controller];
7645
+ const holder = await new GenericContainer(image).withNetwork(network).start();
7646
+ opened.push(holder);
7647
+ const admin = await new GenericContainer(image).withNetworkMode(`container:${holder.getId()}`).withPrivilegedMode().start();
7648
+ opened.push(admin);
7649
+ const clientIp = await ipOf(holder);
7650
+ const commander = await new GenericContainer(clientImage).withNetwork(network).withCommand([
7651
+ "sh",
7652
+ "-c",
7653
+ "while :; do sleep 3600; done"
7654
+ ]).start();
7655
+ opened.push(commander);
7656
+ const clientNetns = await netnsOf(admin);
7657
+ const records = lines();
7658
+ const prober = await new GenericContainer(image).withPrivilegedMode().withNetworkMode("none").withCommand(["/opt/harness/prober", String(clientNetns)]).withLogConsumer(records.consume).start();
7659
+ opened.push(prober);
7660
+ const begun = await records.until(["begin", "fail"], 3e4, "prober");
7661
+ if (begun.ev === "fail") throw new Error(`the prober failed: ${String(begun["why"])}`);
7662
+ const box = (name) => {
7663
+ const found = boxes.get(name);
7664
+ if (found === void 0) throw new Error(`the group has no node ${name}`);
7665
+ return found;
7666
+ };
7667
+ const injectorOf = (name) => {
7668
+ const found = injectors.get(name);
7669
+ if (found === void 0) throw new Error(`the group has no node ${name}`);
7670
+ return found;
7671
+ };
7672
+ const lastLine = (text) => text.split("\n").filter((line) => line.trim() !== "").at(-1) ?? "";
7673
+ const stamped = async (where, argv) => {
7674
+ const stamp = JSON.parse(lastLine(await output(where, argv)));
7675
+ return {
7676
+ t0: stamp.t0,
7677
+ t1: stamp.t1
7678
+ };
7679
+ };
7680
+ const after = (at) => at === void 0 ? [] : ["--at", String(Math.round(at))];
7681
+ const bootstrap = bootstrapOf(kafka);
7682
+ const tool = async (...argv) => lastLine(await output(commander, [
7683
+ "java",
7684
+ CLIENT_HEAP,
7685
+ "-cp",
7686
+ CLASS_PATH,
7687
+ "AdminKafka",
7688
+ bootstrap,
7689
+ ...argv
7690
+ ]));
7691
+ const view = async (topic) => JSON.parse(await tool("view", ...topic === void 0 ? [] : [topic]));
7692
+ return {
7693
+ kafka,
7694
+ clientIp,
7695
+ members,
7696
+ controller,
7697
+ netns: { client: clientNetns },
7698
+ kernel: records.seen,
7699
+ raw,
7700
+ fences,
7701
+ async sysctl(values) {
7702
+ const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
7703
+ if (settings.length > 0) await output(admin, [
7704
+ "sysctl",
7705
+ "-q",
7706
+ "-w",
7707
+ ...settings
7708
+ ]);
7709
+ const read = {};
7710
+ for (const key of Object.keys(values)) read[key] = await output(admin, [
7711
+ "sysctl",
7712
+ "-n",
7713
+ key
7714
+ ]);
7715
+ return read;
7716
+ },
7717
+ async brokersUp(within = CONVERGE_MS) {
7718
+ const begin = Date.now();
7719
+ let last = "no view yet";
7720
+ while (Date.now() - begin < within) {
7721
+ const read = await view().catch((error) => error.message.slice(0, 200));
7722
+ if (typeof read !== "string" && read.brokers.length === kafka.brokers.length) return;
7723
+ last = typeof read === "string" ? read : `${read.brokers.length} brokers are not fenced`;
7724
+ await sleep$1(1e3);
7725
+ }
7726
+ throw new Error(`the brokers did not come up in ${within} ms: ${last}`);
7727
+ },
7728
+ async create(topic, partitions = 1) {
7729
+ const ids = kafka.brokers.map((name) => nodeIdOf(kafka, name)).join(",");
7730
+ const reply = await tool("create", topic, String(kafka.minInsync), ids, String(partitions));
7731
+ if (!reply.includes("\"ok\":true")) throw new Error(`the create of the topic failed: ${reply}`);
7732
+ },
7733
+ async fill(topic, partitions, count) {
7734
+ const reply = await tool("fill", topic, String(partitions), String(count));
7735
+ if (!reply.includes("\"ok\":true")) throw new Error(`the fill of the topic failed: ${reply}`);
7736
+ },
7737
+ async consumer(args) {
7738
+ const events = lines();
7739
+ const container = await new GenericContainer(clientImage).withNetworkMode(`container:${holder.getId()}`).withCommand([
7740
+ "sh",
7741
+ "-c",
7742
+ "java \"$@\"; :",
7743
+ "consumer",
7744
+ CLIENT_HEAP,
7745
+ "-cp",
7746
+ CLASS_PATH,
7747
+ "ConsumerKafka",
7748
+ bootstrap,
7749
+ ...args
7750
+ ]).withLogConsumer(events.consume).start();
7751
+ const signaller = await withPid(new GenericContainer(image).withNetworkMode(`container:${holder.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW"), `container:${container.getId()}`).start();
7752
+ return {
7753
+ events: events.seen,
7754
+ until: (kinds, within) => events.until(kinds, within, "client"),
7755
+ signal: (signal, at) => stamped(signaller, [
7756
+ "/opt/harness/inject",
7757
+ "signal",
7758
+ "java",
7759
+ signal,
7760
+ ...after(at)
7761
+ ]),
7762
+ stop: async () => {
7763
+ await container.stop({ timeout: 2 }).catch(() => void 0);
7764
+ await signaller.stop(STOP).catch(() => void 0);
7765
+ }
7766
+ };
7767
+ },
7768
+ async converge(topic, within = CONVERGE_MS) {
7769
+ const begin = Date.now();
7770
+ let calm = 0;
7771
+ let last = {
7772
+ primary: null,
7773
+ healthy: false,
7774
+ why: "no view yet"
7775
+ };
7776
+ while (Date.now() - begin < within) {
7777
+ last = await view(topic).then((read) => kafkaViewOf(kafka, read), (error) => ({
7778
+ primary: null,
7779
+ healthy: false,
7780
+ why: error.message.slice(0, 200)
7781
+ }));
7782
+ calm = last.healthy ? calm + 1 : 0;
7783
+ if (calm >= 2 && last.primary !== null) return last.primary;
7784
+ await sleep$1(1e3);
7785
+ }
7786
+ throw new Error(`the partition did not converge in ${within} ms: ${last.why}`);
7787
+ },
7788
+ async holds(topic) {
7789
+ const reply = await tool("holds", topic);
7790
+ const { held } = JSON.parse(reply);
7791
+ if (held === void 0) throw new Error(`the read of the partition failed: ${reply}`);
7792
+ return held.map(Number).filter((n) => Number.isInteger(n) && n > 0);
7793
+ },
7794
+ async client(args) {
7795
+ const events = lines();
7796
+ const container = await new GenericContainer(clientImage).withNetworkMode(`container:${holder.getId()}`).withCommand([
7797
+ "java",
7798
+ CLIENT_HEAP,
7799
+ "-cp",
7800
+ CLASS_PATH,
7801
+ "ClientKafka",
7802
+ bootstrap,
7803
+ ...args
7804
+ ]).withLogConsumer(events.consume).start();
7805
+ return {
7806
+ events: events.seen,
7807
+ until: (kinds, within) => events.until(kinds, within, "client"),
7808
+ stop: () => container.stop({ timeout: 2 }).then(() => void 0).catch(() => void 0)
7809
+ };
7810
+ },
7811
+ fault(kind, part, cuts, at) {
7812
+ const cut = kafkaCutOf(kafka, members, controller, part, cuts);
7813
+ return stamped(injectorOf(part), [
7814
+ "/opt/harness/inject",
7815
+ kind,
7816
+ cut.peers.join(","),
7817
+ ...after(at)
7818
+ ]);
7819
+ },
7820
+ lift(kind, part, cuts, at) {
7821
+ const cut = kafkaCutOf(kafka, members, controller, part, cuts);
7822
+ return stamped(injectorOf(part), [
7823
+ "/opt/harness/inject",
7824
+ kind === "rst" ? "clear-rst" : "clear",
7825
+ cut.peers.join(","),
7826
+ ...after(at)
7827
+ ]);
7828
+ },
7829
+ async forget() {
7830
+ await output(admin, [
7831
+ "ip",
7832
+ "tcp_metrics",
7833
+ "flush",
7834
+ "all"
7835
+ ]);
7836
+ },
7837
+ async settle() {
7838
+ const from = records.seen.length;
7839
+ const clock = await new Promise((resolve) => {
7840
+ const begin = Date.now();
7841
+ const look = () => {
7842
+ const line = records.seen.slice(from).find((record) => record.ev === "clock");
7843
+ if (line !== void 0) resolve(line);
7844
+ else if (Date.now() - begin > 5e3) resolve(null);
7845
+ else setTimeout(look, 50);
7846
+ };
7847
+ look();
7848
+ });
7849
+ return clock === null ? null : Number(clock["lost"] ?? 0);
7850
+ },
7851
+ async manifest(sysctls) {
7852
+ const config = await output(admin, ["zcat", "/proc/config.gz"]).catch(() => "");
7853
+ const inspect = async (container) => runtime.container.inspect(runtime.container.getById(container.getId()));
7854
+ const settings = {
7855
+ settings: Object.entries(kafka.settings).map(([property, value]) => `${property}=${value}`).join("; "),
7856
+ heap: kafka.heap,
7857
+ "min.insync.replicas": String(kafka.minInsync)
7858
+ };
7859
+ const list = async (names) => {
7860
+ const out = [];
7861
+ for (const name of names) {
7862
+ const version = lastLine(await output(box(name), ["/opt/kafka/bin/kafka-topics.sh", "--version"]).catch(() => ""));
7863
+ out.push({
7864
+ name,
7865
+ ip: parts[name].ip,
7866
+ id: (await inspect(box(name))).Image,
7867
+ settings: {
7868
+ version,
7869
+ ...settings
7870
+ }
7871
+ });
7872
+ }
7873
+ return out;
7874
+ };
7875
+ const brokers = await list(kafka.brokers);
7876
+ return {
7877
+ kernel: await output(holder, ["uname", "-r"]),
7878
+ architecture: await output(holder, ["uname", "-m"]),
7879
+ hz: /^CONFIG_HZ=(\d+)$/m.exec(config)?.[1] ?? null,
7880
+ boot_id: await output(holder, ["cat", "/proc/sys/kernel/random/boot_id"]),
7881
+ docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
7882
+ target: {
7883
+ image: kafka.image,
7884
+ id: brokers[0].id,
7885
+ settings: brokers[0].settings,
7886
+ sysctls: {}
7887
+ },
7888
+ harness: {
7889
+ image,
7890
+ id: (await inspect(holder)).Image,
7891
+ packages: (await output(holder, [
7892
+ "apk",
7893
+ "info",
7894
+ "-v"
7895
+ ])).split("\n").sort()
7896
+ },
7897
+ client: {
7898
+ kind: "image",
7899
+ image: clientImage
7900
+ },
7901
+ netns: {
7902
+ client: clientNetns,
7903
+ target: members[kafka.brokers[0]].netns
7904
+ },
7905
+ sysctls,
7906
+ group: {
7907
+ members: brokers,
7908
+ sentinels: await list([kafka.controller]),
7909
+ observer: { image: clientImage }
7910
+ }
7911
+ };
7912
+ },
7913
+ close
7914
+ };
7915
+ } catch (error) {
7916
+ await close();
7917
+ throw error;
7918
+ }
7919
+ };
7920
+ /** This count is the number of acknowledged writes that a run waits for before its fault: the stream is steady then. */
7921
+ const STEADY = 20;
7922
+ /**
7923
+ * This time is how long a run continues after its lift. The record then shows the step-down of the
7924
+ * leader and the writes after it. It has no source. It is above the largest step-down of the
7925
+ * law, which is approximately 2 s after the lift.
7926
+ */
7927
+ const KAFKA_TAIL_MS = 1e4;
7928
+ /** This function gives the arguments of the client of a Kafka cell: the topic, the writers, the stream of writes and the settings. */
7929
+ const kafkaArgs = (cell, topic) => [
7930
+ "--topic",
7931
+ topic,
7932
+ "--writers",
7933
+ String(cell.kafka.writers),
7934
+ "--interval-ms",
7935
+ String(cell.probe.interval === void 0 ? 50 : valueIn(cell.probe.interval, "ms")),
7936
+ "--max-s",
7937
+ String(Math.ceil(deadlineOf(cell) / 1e3 + (cell.hold === void 0 ? 0 : valueIn(cell.hold, "s")) + 120)),
7938
+ ...Object.entries(cell.driverOptions ?? {}).flatMap(([name, value]) => ["--set", `${name}=${String(value)}`])
7939
+ ];
7940
+ /** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
7941
+ const waitFor = (client, match, within) => new Promise((resolve) => {
7942
+ const started = Date.now();
7943
+ const look = () => {
7944
+ const found = client.events.find(match);
7945
+ if (found !== void 0 || Date.now() - started > within) resolve(found);
7946
+ else setTimeout(look, 20);
7947
+ };
7948
+ look();
7949
+ });
7950
+ /** This function gives the count of lost records in the latest clock line of the prober. */
7951
+ const droppedSoFar$1 = (records) => {
7952
+ for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
7953
+ return 0;
7954
+ };
7955
+ /**
7956
+ * This function does one run of a Kafka cell:
7957
+ * - Each broker is up, the topic of the run exists, and its partition is whole. The client starts,
7958
+ * and its stream of writes gets steady.
7959
+ * - The fault cuts the leader from the other brokers and the controller, and it lifts at its instant
7960
+ * in the VM. The client keeps its route to the leader, thus only the group knows of the cut.
7961
+ * - The run waits for the lift and for the tail after it.
7962
+ * - Each fault goes, the client stops, and the partition converges again. The admin then reads
7963
+ * the whole partition, thus the record tells which writes the group kept.
7964
+ */
7965
+ const kafkaOnce = async (lab, cell, environment, index) => {
7966
+ if (cell.fault !== "partition") throw new Error(`a Kafka cell has no fault ${cell.fault}`);
7967
+ if (cell.hold === void 0) throw new Error("a Kafka cell needs a hold: its law reads it");
7968
+ const place = cell.faultAt ?? { role: "primary" };
7969
+ if (place.role !== "primary") throw new Error(`a Kafka cell has no fault on the ${place.role}`);
7970
+ const cuts = place.cuts ?? "group";
7971
+ const topic = topicOf(environment, index);
7972
+ await lab.brokersUp();
7973
+ await lab.create(topic);
7974
+ const leader = await lab.converge(topic);
7975
+ await lab.forget();
7976
+ const from = lab.kernel.length;
7977
+ const lostBefore = droppedSoFar$1(lab.kernel);
7978
+ const client = await lab.client(kafkaArgs(cell, topic));
7979
+ let fault = null;
7980
+ let heal = null;
7981
+ const undo = [];
7982
+ try {
7983
+ await client.until(["ready"], 6e4);
7984
+ if (await waitFor(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY, 3e4) === void 0) throw new Error(`the client did not write ${STEADY} times in 30 s`);
7985
+ fault = await lab.fault("blackhole", leader, cuts);
7986
+ undo.push(() => lab.lift("blackhole", leader, cuts));
7987
+ heal = await lab.lift("blackhole", leader, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
7988
+ undo.length = 0;
7989
+ await sleep$1(KAFKA_TAIL_MS);
7990
+ } finally {
7991
+ for (const step of undo.reverse()) await step().catch(() => void 0);
7992
+ await client.stop();
7993
+ }
7994
+ const settled = await lab.settle();
7995
+ const lost = settled === null ? null : settled - lostBefore;
7996
+ const after = await lab.converge(topic).catch(() => null);
7997
+ const holds = after === null ? null : await lab.holds(topic);
7998
+ return {
7999
+ environment,
8000
+ index,
8001
+ events: [...client.events],
8002
+ fault,
8003
+ ...heal === null ? {} : { heal },
8004
+ kernel: lab.kernel.slice(from),
8005
+ netns: lab.netns.client,
8006
+ targetNetns: lab.members[leader].netns,
8007
+ lost,
8008
+ group: {
8009
+ members: lab.members,
8010
+ sentinels: { [lab.kafka.controller]: lab.controller },
8011
+ primary: leader,
8012
+ at: leader,
8013
+ timeline: [],
8014
+ after,
8015
+ holds
8016
+ }
8017
+ };
8018
+ };
8019
+ /** This count is the partitions of the topic of a run of a paused consumer: one for each member. */
8020
+ const PAUSE_PARTITIONS = 2;
8021
+ /** This time bounds the wait for one partition for each member and for two commits, in ms. It has no source: it is far above two batches and one rebalance. */
8022
+ const PAUSE_READY_MS = 9e4;
8023
+ /** This time bounds the wait for the next batch of the first member, in ms. It has no source: it is far above one batch. */
8024
+ const PAUSE_BATCH_MS = 3e4;
8025
+ /**
8026
+ * This function does one run of a paused consumer:
8027
+ * - The topic of the run has two partitions, each with records that a run does not finish.
8028
+ * - Two members join one group, and the run waits until each member holds one partition and the
8029
+ * first member has two commits on its partition.
8030
+ * - A SIGSTOP stops the first member while it holds its next batch, and a SIGCONT ends the stop
8031
+ * after the hold of the cell. The run then waits for the tail.
8032
+ *
8033
+ * The fault is null when the members did not get one partition each. The measure then voids the run.
8034
+ * The law reads no kernel record, thus a lost record does not void this run.
8035
+ */
8036
+ const kafkaPauseOnce = async (lab, cell, environment, index) => {
8037
+ const pause = cell.kafka?.pause;
8038
+ if (pause === void 0) throw new Error("a run of a paused consumer needs `Kafka.pause`");
8039
+ if (cell.fault !== "freeze-process") throw new Error(`a paused consumer has no fault ${cell.fault}`);
8040
+ if (cell.hold === void 0) throw new Error("a paused consumer needs a hold: its law reads it");
8041
+ const hold = valueIn(cell.hold, "s");
8042
+ const topic = topicOf(environment, index);
8043
+ const group = `pause-e${environment}-r${index + 1}`;
8044
+ await lab.brokersUp();
8045
+ await lab.create(topic, PAUSE_PARTITIONS);
8046
+ const leader = await lab.converge(topic);
8047
+ await lab.fill(topic, PAUSE_PARTITIONS, pause.fill);
8048
+ await lab.forget();
8049
+ const from = lab.kernel.length;
8050
+ const lostBefore = droppedSoFar$1(lab.kernel);
8051
+ const common = [
8052
+ "--topic",
8053
+ topic,
8054
+ "--group",
8055
+ group,
8056
+ "--protocol",
8057
+ pause.protocol,
8058
+ "--max-s",
8059
+ String(Math.ceil(hold + 180)),
8060
+ ...Object.entries(cell.driverOptions ?? {}).flatMap(([name, value]) => ["--set", `${name}=${String(value)}`])
8061
+ ];
8062
+ const members = [];
8063
+ let fault = null;
8064
+ let heal = null;
8065
+ let otherEvents = [];
8066
+ let firstEvents = [];
8067
+ try {
8068
+ const first = await lab.consumer([
8069
+ ...common,
8070
+ "--process-ms",
8071
+ String(valueIn(pause.process, "ms")),
8072
+ "--after-pause",
8073
+ pause.afterPause
8074
+ ]);
8075
+ members.push(first);
8076
+ const other = await lab.consumer([
8077
+ ...common,
8078
+ "--process-ms",
8079
+ "0",
8080
+ "--after-pause",
8081
+ "commit"
8082
+ ]);
8083
+ members.push(other);
8084
+ firstEvents = first.events;
8085
+ otherEvents = other.events;
8086
+ const begun = Date.now();
8087
+ while (Date.now() - begun < PAUSE_READY_MS && !pauseReady(first.events, other.events)) await sleep$1(100);
8088
+ if (pauseReady(first.events, other.events)) {
8089
+ const seen = first.events.at(-1)?.t ?? 0;
8090
+ const batch = await waitFor(first, (event) => event.ev === "processing" && event.t > seen, PAUSE_BATCH_MS);
8091
+ if (batch !== void 0) {
8092
+ fault = await first.signal("STOP", batch.t + valueIn(pause.stopAfter, "s") * 1e9);
8093
+ heal = await first.signal("CONT", fault.t0 + hold * 1e9);
8094
+ await sleep$1(KAFKA_TAIL_MS);
8095
+ }
8096
+ }
8097
+ } finally {
8098
+ for (const member of members) await member.stop();
8099
+ }
8100
+ const settled = await lab.settle();
8101
+ return {
8102
+ environment,
8103
+ index,
8104
+ events: [...firstEvents],
8105
+ fault,
8106
+ ...heal === null ? {} : { heal },
8107
+ kernel: lab.kernel.slice(from),
8108
+ netns: lab.netns.client,
8109
+ targetNetns: lab.members[leader].netns,
8110
+ lost: settled === null ? null : settled - lostBefore,
8111
+ pause: {
8112
+ other: [...otherEvents],
8113
+ fences: lab.fences.filter((line) => namesGroup(line, group)),
8114
+ group
8115
+ }
8116
+ };
8117
+ };
8118
+ //#endregion
8119
+ //#region src/run.ts
8120
+ /**
8121
+ * This function measures and judges the raw records of a cell, with the code of the harness at
8122
+ * this time. `runCell` uses it during a cell. A holder of the raw records of a cell can use it
8123
+ * again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.
8124
+ */
8125
+ const judge = (cell, runs) => {
8126
+ const measured = runs.filter((run) => run.warmup !== true).map((run) => measure(cell, run));
8127
+ return {
8128
+ measured,
8129
+ verdict: verdictOf(cell.prediction, measured, noiseOf(cell))
8130
+ };
8131
+ };
8132
+ /** This function gives the count of lost records in the latest clock line of the prober, since it began. */
8133
+ const droppedSoFar = (records) => {
8134
+ for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
8135
+ return 0;
8136
+ };
8137
+ /** This function gives the argv of the client for a cell, with a deadline far after all that it waits for. */
8138
+ const probeArgs = (cell, lab) => {
8139
+ const { prediction, probe } = cell;
8140
+ const longest = (prediction.detects === true && prediction.driver !== void 0 ? longestOf(cell) : prediction.detects ? latestEndOf(prediction) : valueIn(prediction.window, "s")) + (cell.idleBefore === void 0 ? 0 : valueIn(cell.idleBefore, "s"));
8141
+ const args = [
8142
+ lab.targetIp,
8143
+ String(lab.targetPort),
8144
+ "--mode",
8145
+ probe.mode
8146
+ ];
8147
+ if (probe.ws !== void 0) {
8148
+ const { ws } = probe;
8149
+ const waits = (prediction.detects !== false ? prediction.reconnects?.waits ?? [] : []).reduce((sum, wait) => sum + valueIn(wait.high, "s"), 0);
8150
+ args.push("--websocket", ws.socket);
8151
+ args.push("--heartbeat", ws.heartbeat === null ? "off" : "on");
8152
+ if (ws.heartbeat !== null) args.push("--heartbeat-interval-ms", String(valueIn(ws.heartbeat.interval, "ms")), "--heartbeat-timeout-ms", String(valueIn(ws.heartbeat.timeout, "ms")), "--heartbeat-misses", String(ws.heartbeat.misses));
8153
+ args.push("--reconnect", ws.reconnect === null ? "off" : "on");
8154
+ if (ws.reconnect !== null) args.push("--reconnect-initial-ms", String(valueIn(ws.reconnect.initialDelay, "ms")), "--reconnect-multiplier", String(ws.reconnect.multiplier), "--reconnect-max-ms", String(valueIn(ws.reconnect.maxDelay, "ms")), "--reconnect-attempts", String(ws.reconnect.attempts), "--reconnect-jitter", String(ws.reconnect.jitter));
8155
+ args.push("--request-timeout-ms", String(valueIn(ws.requestTimeout, "ms")));
8156
+ args.push("--connection-timeout-ms", String(valueIn(ws.connectionTimeout, "ms")));
8157
+ if (ws.hold !== void 0) args.push("--hold-ms", String(valueIn(ws.hold, "ms")));
8158
+ args.push("--max-s", String(Math.ceil((longest + waits) * 2 + 10)));
8159
+ return args;
8160
+ }
8161
+ if (probe.http !== void 0) {
8162
+ const { http } = probe;
8163
+ const gap = http.gap === void 0 ? 0 : valueIn(http.gap, "s");
8164
+ args.push("--method", http.method);
8165
+ if (http.gap !== void 0) args.push("--gap-ms", String(valueIn(http.gap, "ms")));
8166
+ if (http.hold !== void 0) args.push("--hold-ms", String(valueIn(http.hold, "ms")));
8167
+ if (http.bytes !== void 0) args.push("--bytes", String(http.bytes));
8168
+ for (const [name, value] of Object.entries(http.options ?? {})) args.push(`--${name}`, String(value));
8169
+ args.push("--linger-ms", String(LINGER_MS));
8170
+ args.push("--max-s", String(Math.ceil(longest * 2 + gap + 10)));
8171
+ return args;
8172
+ }
8173
+ if (probe.request !== void 0) args.push("--request", probe.request);
8174
+ if (probe.count !== void 0) args.push("--count", String(probe.count));
8175
+ if (probe.interval !== void 0) args.push("--interval-ms", String(valueIn(probe.interval, "ms")));
8176
+ if (probe.driver?.name === "mongodb") {
8177
+ args.push("--set", probe.driver.set);
8178
+ if (probe.driver.op !== void 0) args.push("--op", probe.driver.op);
8179
+ if (cell.writeConcern !== void 0) args.push("--w", String(cell.writeConcern));
8180
+ if (cell.before === true) args.push("--before");
8181
+ if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
8182
+ args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
8183
+ return args;
8184
+ }
8185
+ if (probe.driver?.name === "pg" || probe.driver?.name === "hikari" || probe.driver?.name === "amqp") {
8186
+ if (probe.driver.name !== "amqp" && probe.driver.query !== void 0) args.push("--query", probe.driver.query);
8187
+ if (probe.driver.name !== "amqp" && probe.driver.inflight !== void 0) args.push("--inflight-query", probe.driver.inflight);
8188
+ if (probe.driver.name === "hikari" && probe.driver.burst !== void 0) args.push("--burst", String(probe.driver.burst.count), "--burst-gap-ms", String(Math.round(valueIn(probe.driver.burst.gap, "ms"))));
8189
+ if (cell.before === true) args.push("--before");
8190
+ if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
8191
+ args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
8192
+ return args;
8193
+ }
8194
+ if (probe.driver?.name === "grpc") {
8195
+ if (probe.driver.hold !== void 0) args.push("--hold-ms", String(valueIn(probe.driver.hold, "ms")));
8196
+ if (probe.driver.op !== void 0) args.push("--op", probe.driver.op);
8197
+ if (cell.before === true) args.push("--before");
8198
+ if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
8199
+ args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
8200
+ return args;
8201
+ }
8202
+ if (probe.driver !== void 0) {
8203
+ args.push("--keepalive-ms", String(valueIn(probe.driver.keepAlive, "ms")));
8204
+ args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
8205
+ return args;
8206
+ }
8207
+ if (probe.keepalive) args.push("--keepalive");
8208
+ if (probe.userTimeout !== void 0) args.push("--user-timeout-ms", String(valueIn(probe.userTimeout, "ms")));
8209
+ if (probe.plan !== void 0) args.push("--schedule", SCHEDULE_PATH);
8210
+ if (probe.streams !== void 0) args.push("--plan", LOAD_PATH);
8211
+ args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
8212
+ return args;
8213
+ };
8214
+ /**
8215
+ * This function waits for an error on the socket of the observer, or for its end of file
8216
+ * (CLOSE_WAIT, state 8). The prober records both. A cell whose observer is the server, or a proxy
8217
+ * of a topology, waits for that. A target with its own detector can also drop the connection
8218
+ * itself (`dropped`). The promise resolves after `within` ms at the latest.
8219
+ */
8220
+ const observerError = (cell, partial, within, dropped = () => false) => new Promise((resolve) => {
8221
+ const started = Date.now();
8222
+ const look = () => {
8223
+ const run = partial();
8224
+ const at = socketOf(cell, run);
8225
+ if (at !== void 0 && run.kernel.some((record) => record.netns === at.netns && record.cookie === at.cookie && (record.ev === "error" && (record.err ?? 0) !== 0 || record.ev === "state" && record.new === 8)) || dropped() || Date.now() - started > within) resolve();
8226
+ else setTimeout(look, 50);
8227
+ };
8228
+ look();
8229
+ });
8230
+ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
8231
+ /**
8232
+ * This time is the pause between the client's first reply and the fault. It is more than Linux's
8233
+ * TCP_DELACK_MAX of 200 ms. Thus the delayed ACK of the reply is out before the fault.
8234
+ */
8235
+ const QUIET_MS = 250;
8236
+ /**
8237
+ * This time is how long an HTTP client waits after its second request, before it exits. The exit
8238
+ * closes its sockets, thus that close comes after the window of the cell.
8239
+ */
8240
+ const LINGER_MS = 1e3;
8241
+ /** These are the lines that end the workload of an HTTP client: the end of its second request, its deadline, or its failure. */
8242
+ const HTTP_DONE = [
8243
+ "done",
8244
+ "deadline",
6376
8245
  "fail"
6377
8246
  ];
6378
8247
  /**
@@ -6480,6 +8349,19 @@ const dials = (probe, count, within) => new Promise((resolve) => {
6480
8349
  };
6481
8350
  look();
6482
8351
  });
8352
+ /** This time is the most that a run waits for the lines of the target after the end of its client. */
8353
+ const LINES_MS = 5e3;
8354
+ /**
8355
+ * This function waits for the lines of a run's counts. The lines of the target come on their own
8356
+ * stream, thus they can come after the last line of the client. A line above a count comes in the
8357
+ * pause after the wait, thus it is in the run that wrote it.
8358
+ */
8359
+ const linesCounted = async (lab, from, counts) => {
8360
+ const started = Date.now();
8361
+ const short = () => counts.some((each) => countOf(lab.server.slice(from), each.match) < each.count);
8362
+ while (short() && Date.now() - started < LINES_MS) await sleep(20);
8363
+ await sleep(250);
8364
+ };
6483
8365
  /** This function gives the netem specification of a cell: its latency, then the netem of its fault when it has one. */
6484
8366
  const netemSpec = (cell, fault) => {
6485
8367
  const parts = [...cell.latency === void 0 ? [] : [`delay ${valueIn(cell.latency, "ms")}ms`], ...fault === void 0 ? [] : [fault]];
@@ -6508,7 +8390,16 @@ const once = async (lab, cell, environment, index, dry = false) => {
6508
8390
  const middleboxFrom = lab.middlebox.length;
6509
8391
  const serverFrom = lab.server.length;
6510
8392
  const lostBefore = droppedSoFar(lab.kernel);
6511
- const probe = await lab.probe(probeArgs(cell, lab));
8393
+ const plan = cell.probe.plan === void 0 ? void 0 : schedule(cell.probe.plan.profile, cell.probe.plan.options);
8394
+ const load = cell.probe.streams === void 0 ? void 0 : loadPlanOf(cell.probe.streams);
8395
+ const files = [...plan === void 0 ? [] : [{
8396
+ content: JSON.stringify(plan),
8397
+ target: SCHEDULE_PATH
8398
+ }], ...load === void 0 ? [] : [{
8399
+ content: load.content,
8400
+ target: LOAD_PATH
8401
+ }]];
8402
+ const probe = await lab.probe(probeArgs(cell, lab), files.length === 0 ? void 0 : files);
6512
8403
  const server = cell.observer === "server";
6513
8404
  const peer = server ? {
6514
8405
  freeze: () => probe.freeze(),
@@ -6521,7 +8412,11 @@ const once = async (lab, cell, environment, index, dry = false) => {
6521
8412
  const last = probe.events.slice(-3).map((event) => JSON.stringify(event));
6522
8413
  throw new Error(`${error.message}; the client's last lines: ${last.join(" ")}`);
6523
8414
  });
6524
- const placed = lab.topology === null ? {} : { topology: lab.topology };
8415
+ const placed = {
8416
+ ...lab.topology === null ? {} : { topology: lab.topology },
8417
+ ...plan === void 0 ? {} : { schedule: plan },
8418
+ ...load === void 0 ? {} : { streams: load.streams }
8419
+ };
6525
8420
  const fabricOf = () => {
6526
8421
  if (lab.fabric === void 0) throw new Error(`${cell.name} silences a router of a lab with no fabric`);
6527
8422
  return lab.fabric;
@@ -6666,7 +8561,7 @@ const once = async (lab, cell, environment, index, dry = false) => {
6666
8561
  fault = await apply();
6667
8562
  try {
6668
8563
  if (cell.idleBefore !== void 0 && cell.probe.mode !== "idle") await sleep(valueIn(cell.idleBefore, "ms"));
6669
- if ((cell.probe.mode === "inflight" || cell.probe.mode === "stream") && !replyInFlight) await go();
8564
+ if ((cell.probe.mode === "inflight" || cell.probe.mode === "stream" || cell.probe.mode === "schedule") && !replyInFlight) await go();
6670
8565
  const { prediction } = cell;
6671
8566
  const http = cell.probe.http !== void 0;
6672
8567
  if (prediction.detects !== null && prediction.driver !== void 0) {
@@ -6680,6 +8575,7 @@ const once = async (lab, cell, environment, index, dry = false) => {
6680
8575
  const gap = cell.probe.http?.gap === void 0 ? 0 : valueIn(cell.probe.http.gap, "s");
6681
8576
  const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
6682
8577
  await probe.until(HTTP_DONE, (gap + window) * 2e3 + 1e4).catch(() => void 0);
8578
+ if (prediction.detects === null && prediction.counts !== void 0) await linesCounted(lab, serverFrom, prediction.counts);
6683
8579
  } else if (cell.fault === "stall") {
6684
8580
  if (cell.stall === void 0) throw new Error(`${cell.name} says no stall's length`);
6685
8581
  const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
@@ -6730,7 +8626,7 @@ const once = async (lab, cell, environment, index, dry = false) => {
6730
8626
  await lift();
6731
8627
  if (toxicFirst) await lab.toxic(null);
6732
8628
  }
6733
- if (cell.fault === "freeze-process" && !server && cell.probe.mode !== "idle") await probe.until(["data"], 5e3).catch(() => void 0);
8629
+ if (cell.fault === "freeze-process" && !server && cell.probe.mode !== "idle" && cell.probe.mode !== "schedule") await probe.until(["data"], 5e3).catch(() => void 0);
6734
8630
  const expected = cell.prediction.detects !== false ? cell.prediction.reconnects : void 0;
6735
8631
  if (expected !== void 0) {
6736
8632
  const waits = expected.waits.reduce((sum, wait) => sum + valueIn(wait.high, "s"), 0);
@@ -6794,6 +8690,8 @@ const inEnvironment = async (cell, environment, count, say) => {
6794
8690
  if (cell.group !== void 0) return inGroupEnvironment(cell, environment, count, say);
6795
8691
  if (cell.replicaSet !== void 0) return inReplicaSetEnvironment(cell, environment, count, say);
6796
8692
  if (cell.cluster !== void 0) return inClusterEnvironment(cell, environment, count, say);
8693
+ if (cell.kafka !== void 0) return inKafkaEnvironment(cell, environment, count, say);
8694
+ if (cell.pair !== void 0) return inPairEnvironment(cell, environment, count, say);
6797
8695
  const lab = await openLab(cell.target, cell.client ?? cell.probe.driver?.name ?? "tcp", cell.topology);
6798
8696
  try {
6799
8697
  const settings = await lab.configure(cell.targetSettings ?? {});
@@ -6808,9 +8706,11 @@ const inEnvironment = async (cell, environment, count, say) => {
6808
8706
  if (latency !== null) await lab.netem(latency, "target");
6809
8707
  await lab.warm();
6810
8708
  const manifest = await lab.manifest(await lab.sysctl(cell.sysctls), settings, targetSysctls);
8709
+ const missing = await lab.namespaceSetup(cell.namespaceSetup ?? []).then(() => lab.hostSysctl(cell.hostSysctls ?? [])).then(() => void 0, (error) => error instanceof Error ? error.message : String(error));
6811
8710
  const timed = (run) => ({
6812
8711
  ...run,
6813
- jitter: jitterOf(cell, run)
8712
+ jitter: jitterOf(cell, run),
8713
+ ...missing === void 0 ? {} : { missing }
6814
8714
  });
6815
8715
  const warmup = {
6816
8716
  ...timed(await withSteal(() => once(lab, cell, environment, -1, endsEnvironment(cell)))),
@@ -6823,8 +8723,12 @@ const inEnvironment = async (cell, environment, count, say) => {
6823
8723
  runs.push(run);
6824
8724
  say(line(cell, run, measure(cell, run)));
6825
8725
  }
8726
+ const host = await lab.restoreHost();
6826
8727
  return {
6827
- manifest,
8728
+ manifest: host.length === 0 ? manifest : {
8729
+ ...manifest,
8730
+ host
8731
+ },
6828
8732
  runs,
6829
8733
  warmup
6830
8734
  };
@@ -6891,6 +8795,36 @@ const inReplicaSetEnvironment = async (cell, environment, count, say) => {
6891
8795
  }
6892
8796
  };
6893
8797
  /**
8798
+ * This function does the part of the runs of a pair cell in one environment. The warm-up is a dry
8799
+ * run. The run promotes the standby, thus a pair cell has an environment for each run.
8800
+ */
8801
+ const inPairEnvironment = async (cell, environment, count, say) => {
8802
+ const lab = await openLab(cell.target, cell.client ?? cell.probe.driver?.name ?? "tcp", cell.topology);
8803
+ try {
8804
+ await lab.warm();
8805
+ const manifest = await lab.manifest(await lab.sysctl(cell.sysctls));
8806
+ const standby = await openStandby(lab, cell);
8807
+ const warmup = {
8808
+ ...await pairOnce(lab, standby, cell, environment, -1, true),
8809
+ warmup: true
8810
+ };
8811
+ say(line(cell, warmup, measure(cell, warmup)));
8812
+ const runs = [];
8813
+ for (let index = 0; index < count; index += 1) {
8814
+ const run = await pairOnce(lab, standby, cell, environment, index);
8815
+ runs.push(run);
8816
+ say(line(cell, run, measure(cell, run)));
8817
+ }
8818
+ return {
8819
+ manifest,
8820
+ runs,
8821
+ warmup
8822
+ };
8823
+ } finally {
8824
+ await lab.close();
8825
+ }
8826
+ };
8827
+ /**
6894
8828
  * This function does the part of the runs of a cluster cell in one environment. The warm-up is a
6895
8829
  * whole run with its fault, as for a replica set. The queue converges before each run.
6896
8830
  */
@@ -6919,6 +8853,35 @@ const inClusterEnvironment = async (cell, environment, count, say) => {
6919
8853
  }
6920
8854
  };
6921
8855
  /**
8856
+ * This function does the part of the runs of a Kafka cell in one environment. The warm-up is a whole
8857
+ * run with its fault, as for a cluster. Each run makes its own topic.
8858
+ */
8859
+ const inKafkaEnvironment = async (cell, environment, count, say) => {
8860
+ const lab = await openKafkaLab(cell.kafka);
8861
+ const once = cell.kafka.pause === void 0 ? kafkaOnce : kafkaPauseOnce;
8862
+ try {
8863
+ const manifest = await lab.manifest(await lab.sysctl(cell.sysctls));
8864
+ const warmup = {
8865
+ ...await once(lab, cell, environment, -1),
8866
+ warmup: true
8867
+ };
8868
+ say(line(cell, warmup, measure(cell, warmup)));
8869
+ const runs = [];
8870
+ for (let index = 0; index < count; index += 1) {
8871
+ const run = await once(lab, cell, environment, index);
8872
+ runs.push(run);
8873
+ say(line(cell, run, measure(cell, run)));
8874
+ }
8875
+ return {
8876
+ manifest,
8877
+ runs,
8878
+ warmup
8879
+ };
8880
+ } finally {
8881
+ await lab.close();
8882
+ }
8883
+ };
8884
+ /**
6922
8885
  * This function does the runs of a cell. They go across new environments, and each environment records
6923
8886
  * its manifest. Then the verdict judges the runs. A median across the edge of the band extends
6924
8887
  * the cell from 11 runs to 22, and then to 44, each extension in new environments. With a
@@ -6972,6 +8935,6 @@ const runCell = async (cell, say = () => {}) => {
6972
8935
  }
6973
8936
  };
6974
8937
  //#endregion
6975
- export { AWS_FROM, CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, DAEMONS, DROP_HANDLING, FABRIC_CLIENT, FABRIC_DEVICE, FABRIC_TARGET, FRR_IMAGE, GIVES_AWAY, HASH_SCRIPT, INJECT, KEPT, LINGER_MS, LISTING, LOCK_NAMES, LOSS_MARGIN, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, ROUTERS, SEED_DOCUMENT, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, basesOf, buildOf, buildRecordOf, callOf, cargoVersions, clockOffsetOf, closeOrExit, cloudOf, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, commitProblems, comparable, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, drillOnce, drillSpan, dropFinding, dropOf, endpointAfter, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, frrConf, goHttpImage, goVersions, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, harnessRecord, harnessVersion, hashOf, highestHeld, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isHopPlace, jarVersions, jitterOf, jitterOfLines, judge, lagsOf, laterMove, latestEndOf, leafLacks, leftAfter, locksOf, logNameOf, manifestProblems, matches, measure, measureDriver, measureGroup, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicAt, monotonicOf, nodeClientImage, nodeCommand, nodeHttpImage, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, npmVersions, onMonotonic, openClusterLab, openFabric, openGroupLab, openLab, openReplicaSetLab, pairingOf, pairingsOf, peersOf, pgClientImage, pingsOf, plainOf, planOf, programLines, programOf, quantileOf, readCpu, readHost, readOplog, reconnectFinding, reconnectOf, recoveryFindings, recoveryOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routeLine, routerImage, routesOf, runCell, seedScript, sentinelConfig, sessionsOf, shippedClientRecord, shippedSource, sliceQuery, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, standaloneCommand, statusViewOf, stealShare, stopScript, subjectClient, subjectProblems, subscriptionsOf, substrateHere, substrateKey, substrateOf, substrateOn, targetImageOf, targetSourceOf, templated, toolJsonOf, topologyOf, toxicBody, unlockedManifests, unpinnedBases, unpoolable, verdictOf, versionsOf, viewOf, writesOf };
8938
+ export { AWS_FROM, CLIENT_CAPABILITIES, CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, DAEMONS, DEBUG, DIRTY, DROP_HANDLING, FABRIC_CLIENT, FABRIC_DEVICE, FABRIC_TARGET, FRR_IMAGE, GIVES_AWAY, HASH_SCRIPT, INJECT, INSERT, KAFKA_TAIL_MS, KEPT, LINGER_MS, LISTING, LOAD_FORMAT, LOAD_PATH, LOCK_NAMES, LOSS_MARGIN, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, RESTART, ROUTERS, SCHEDULE_PATH, SEED_DOCUMENT, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STANDBY_READY, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, basesOf, bootstrapOf, brokerEnv, buildOf, buildRecordOf, callOf, cancelsOf, cargoVersions, clockOffsetOf, closeOrExit, cloudOf, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, commitProblems, comparable, controllerEnv, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, drillOnce, drillSpan, dropFinding, dropOf, endpointAfter, envOf, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, frrConf, goHttpImage, goVersions, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, harnessRecord, harnessVersion, hashOf, highestHeld, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isFenceLine, isHopPlace, jarVersions, jitterOf, jitterOfLines, judge, kafkaArgs, kafkaClientImage, kafkaCutOf, kafkaOnce, kafkaPauseOnce, kafkaViewOf, lagsOf, laterMove, latestEndOf, leafLacks, leftAfter, loadOf, loadPlanOf, loadVerdict, locksOf, logNameOf, logTimeOf, manifestProblems, matches, measure, measureConntrack, measureDriver, measureGroup, measurePause, measurePorts, measureQueue, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicAt, monotonicOf, namesGroup, nodeClientImage, nodeCommand, nodeHttpImage, nodeIdOf, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, npmVersions, numbersOf, onMonotonic, openClusterLab, openFabric, openGroupLab, openKafkaLab, openLab, openReplicaSetLab, openStandby, pairArgs, pairOnce, pairingOf, pairingsOf, pauseReady, peersOf, pgClientImage, pingsOf, plainOf, planOf, programLines, programOf, quantileOf, readCpu, readHost, readOplog, reconnectFinding, reconnectOf, recordProblems, recoveryFindings, recoveryOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routeLine, routerImage, routesOf, runCell, seedScript, sentinelConfig, sessionsOf, shippedClientRecord, shippedSource, sliceQuery, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, standaloneCommand, standbyCommand, statusViewOf, stealShare, stopScript, streamsLoadOf, subjectClient, subjectProblems, subscriptionsOf, substrateHere, substrateKey, substrateOf, substrateOn, targetImageOf, targetSourceOf, templated, toolJsonOf, topicOf, topologyOf, toxicBody, unlockedManifests, unpinnedBases, unpoolable, verdictOf, versionsOf, viewOf, writesOf };
6976
8939
 
6977
8940
  //# sourceMappingURL=index.js.map