@mark1russell7/harness 0.14.3 → 0.14.4

package/build/index.js

@@ -1,4 +1,4 @@
-import { A as unpinnedBases, C as routerImage, D as targetImageOf, E as subjectProblems, F as locksOf, I as npmVersions, L as versionsOf, M as cargoVersions, N as goVersions, O as targetSourceOf, P as jarVersions, S as programOf, T as subjectClient, _ as mongoClientImage, a as SHIPPED_NAMES, b as pgClientImage, c as buildRecordOf, d as harnessImage, f as harnessVersion, g as inContext, h as imageProblems, i as SHIPPED_CLIENTS, j as LOCK_NAMES, k as unlockedManifests, l as goHttpImage, m as imageOf, n as PINNED, o as amqpClientImage, p as hikariClientImage, r as REPOSITORY, s as basesOf, t as CLIENT_NAMES, u as grpcClientImage, v as nodeClientImage, w as shippedSource, x as planOf, y as nodeHttpImage } from "./image-BMdV7yWD.js";
+import { A as unlockedManifests, C as programOf, D as subjectProblems, E as subjectClient, F as jarVersions, I as locksOf, L as npmVersions, M as LOCK_NAMES, N as cargoVersions, O as targetImageOf, P as goVersions, R as versionsOf, S as planOf, T as shippedSource, _ as kafkaClientImage, a as SHIPPED_NAMES, b as nodeHttpImage, c as buildRecordOf, d as harnessImage, f as harnessVersion, g as inContext, h as imageProblems, i as SHIPPED_CLIENTS, j as unpinnedBases, k as targetSourceOf, l as goHttpImage, m as imageOf, n as PINNED, o as amqpClientImage, p as hikariClientImage, r as REPOSITORY, s as basesOf, t as CLIENT_NAMES, u as grpcClientImage, v as mongoClientImage, w as routerImage, x as pgClientImage, y as nodeClientImage } from "./image-EZ8ioZ3L.js";
 import { quantity, valueIn } from "@mark1russell7/quantity";
 import { median, medianInterval as medianInterval$1, quantileSorted } from "@mark1russell7/rigor-statistics";
 import { createHash, randomBytes } from "node:crypto";
@@ -288,6 +288,10 @@ const groupVerdictOf = (prediction, counted, delta = NOISE) => {
 	const errors = /* @__PURE__ */ new Map();
 	for (const run of counted) for (const error of run.group?.errors ?? []) errors.set(error.name, [...errors.get(error.name) ?? [], valueIn(error.first, "s")]);
 	const lags = counted.flatMap((run) => run.group?.lags ?? []);
+	const copyCounts = counted.flatMap((run) => {
+		const copies = run.group?.writes.duplicates;
+		return copies === null || copies === void 0 ? [] : [copies.length];
+	});
 	return {
 		stages,
 		medians,
@@ -310,7 +314,13 @@ const groupVerdictOf = (prediction, counted, delta = NOISE) => {
 			runs: firsts.length,
 			first: s(median(firsts.sort((a, b) => a - b)))
 		})),
-		...lags.length === 0 ? {} : { lags: spreadOf(lags) }
+		...lags.length === 0 ? {} : { lags: spreadOf(lags) },
+		...prediction.group?.duplicates === void 0 ? {} : { duplicates: {
+			runs: copyCounts.length,
+			total: copyCounts.reduce((sum, n) => sum + n, 0),
+			median: medianCount(copyCounts),
+			max: copyCounts.length === 0 ? null : Math.max(...copyCounts)
+		} }
 	};
 };
 /** This function gives the verdict of the band of a cell: its containment and the equivalence of its median. */
@@ -335,6 +345,16 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
 		why: "no run counted"
 	};
 	if (prediction.detects === null) {
+		if (prediction.queue !== void 0 || prediction.ports !== void 0) {
+			const off = new Set(outside.map((each) => each.run)).size;
+			return {
+				...empty,
+				decision: off === 0 ? "holds" : "fails",
+				band: null,
+				outside,
+				why: `${counted.length - off} of ${counted.length} runs hold the law of the ${prediction.queue === void 0 ? "port range" : "queue"}; ${prediction.why}`
+			};
+		}
 		const crossed = counted.flatMap((run) => run.crossings === void 0 ? [] : [run.crossings]);
 		if (crossed.length > 0) {
 			const hop = crossed[0].hop;
@@ -363,16 +383,30 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
 			}
 			const outcomes = [...kinds].map(([kind, n]) => `${n} ${kind}`).join(", ");
 			const timed = times.length === 0 ? "no time" : `median ${s(median(times)).value} s from ${prediction.from}, range [${s(times[0]).value}, ${s(times.at(-1)).value}] s`;
-			const counts = prediction.group?.lost;
+			const { counts } = prediction;
 			if (counts !== void 0) {
+				const exact = counts.map((each) => `${each.what} ${each.count}`).join(", ");
+				return {
+					...empty,
+					decision: outside.length === 0 ? "holds" : "fails",
+					band: null,
+					outside,
+					median: times.length === 0 ? null : s(median(times)),
+					why: `${counted.length - outside.length} of ${counted.length} runs gave the exact counts (${exact}); ${outcomes}; ${timed}. ${prediction.why}`
+				};
+			}
+			const lost = prediction.group?.lost;
+			const copies = prediction.group?.duplicates;
+			if (lost !== void 0 || copies !== void 0) {
 				const off = new Set(outside.map((each) => each.run)).size;
+				const bands = [...lost === void 0 ? [] : [`lost writes [${lost.low}, ${lost.high}]`], ...copies === void 0 ? [] : [`extra copies [${copies.low}, ${copies.high}]`]];
 				return {
 					...empty,
 					decision: off === 0 ? "holds" : "fails",
 					band: null,
 					outside,
 					median: times.length === 0 ? null : s(median(times)),
-					why: `${counted.length - off} of ${counted.length} runs inside the band of lost writes [${counts.low}, ${counts.high}]; ${outcomes}; ${timed}. ${prediction.why}`
+					why: `${counted.length - off} of ${counted.length} runs inside the band of ${bands.join(" and the band of ")}; ${outcomes}; ${timed}. ${prediction.why}`
 				};
 			}
 			const missed = outside.filter((each) => counted.some((run) => name(run) === each.run && run.reconnect !== void 0));
@@ -529,7 +563,7 @@ const DROP_HANDLING = quantity(1.5, "ms");
 /** This function gives the window of the act of a drop, in seconds: `DROP_HANDLING`, or the pairing window (`delta`) when it is longer. */
 const dropWindow = (delta) => Math.max(valueIn(DROP_HANDLING, "s"), valueIn(delta, "s"));
 /** This function gives the interval between two instants on the one clock, in seconds to the microsecond. */
-const between$3 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
+const between$4 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
 /** This function gives the state of a state record after the change, and before it. TCP_ESTABLISHED is 1, TCP_SYN_SENT 2. */
 const newState$2 = (record) => record.new;
 const oldState$2 = (record) => record.old;
@@ -571,7 +605,7 @@ const eventOf = (socket, line, matches, delta = NOISE) => {
 */
 const leftAfter = (socket, line) => {
 	const left = socket.find((record) => record.ev === "state" && oldState$2(record) === 1 && (newState$2(record) === 4 || newState$2(record) === 7) && record.mono >= line.t);
-	return left === void 0 ? null : between$3(line.t, left.mono);
+	return left === void 0 ? null : between$4(line.t, left.mono);
 };
 /**
 * This function gives the end of the client process in a run, as the kernel stamps it. The process
@@ -663,19 +697,19 @@ const reconnectOf = (run, detected, cookie, from = detected) => {
 	const waits = starts.flatMap((start, at) => {
 		const next = starts[at + 1]?.t ?? Number.POSITIVE_INFINITY;
 		const dial = dials.find((each) => each.t >= start.t && each.t < next);
-		return dial === void 0 ? [] : [between$3(start.t, dial.t)];
+		return dial === void 0 ? [] : [between$4(start.t, dial.t)];
 	});
 	const fresh = run.kernel.filter((record) => record.netns === run.netns && record.cookie !== cookie && record.ev === "state" && record.mono > detected);
 	const syns = fresh.filter((record) => newState$2(record) === 2);
 	const established = fresh.find((record) => newState$2(record) === 1);
 	const connects = dials.flatMap((dial) => {
 		const syn = syns.find((record) => record.mono >= dial.t);
-		return syn === void 0 ? [] : [between$3(dial.t, syn.mono)];
+		return syn === void 0 ? [] : [between$4(dial.t, syn.mono)];
 	});
 	return {
 		waits,
-		syn: syns[0] === void 0 ? null : between$3(detected, syns[0].mono),
-		established: established === void 0 ? null : between$3(detected, established.mono),
+		syn: syns[0] === void 0 ? null : between$4(detected, syns[0].mono),
+		established: established === void 0 ? null : between$4(detected, established.mono),
 		connects
 	};
 };
@@ -702,7 +736,7 @@ const callOf = (events, request, detected) => {
 	};
 	return {
 		outcome: end.ev === "data" ? "a reply" : String(end["message"] ?? end["code"] ?? "an error"),
-		after: detected === void 0 ? null : between$3(detected, end.t)
+		after: detected === void 0 ? null : between$4(detected, end.t)
 	};
 };
 //#endregion
@@ -717,12 +751,22 @@ const ERRNO$1 = {
 	111: "ECONNREFUSED",
 	113: "EHOSTUNREACH"
 };
-const between$2 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
+const between$3 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
 const round = (seconds) => Math.round(seconds * 1e6) / 1e6;
 /** This function gives the value of a field of a line by its name. A dotted name is a field in a field. */
 const fieldOf = (line, path) => path.split(".").reduce((value, key) => value !== null && typeof value === "object" ? value[key] : void 0, line);
 /** This function tells if a line of the client matches a line match: its `ev`, and each field that the match names. */
 const matches = (line, match) => line.ev === match.ev && Object.entries(match.where ?? {}).every(([path, value]) => fieldOf(line, path) === value);
+/** This function gives the count of the lines that match. */
+const countOf = (lines, match) => lines.filter((line) => matches(line, match)).length;
+/** This function gives the finding of the line counts of a run, or null when each count is the exact one. */
+const countFinding = (counts, lines) => {
+	const off = counts.flatMap((each) => {
+		const seen = countOf(lines, each.match);
+		return seen === each.count ? [] : [`${each.what}: ${seen} lines, and the law gives ${each.count}`];
+	});
+	return off.length === 0 ? null : off.join("; ");
+};
 /** This function says a line match in words, for a finding. */
 const wordsOf = (match) => {
 	const where = Object.entries(match.where ?? {}).map(([path, value]) => `${path} ${String(value)}`);
@@ -1025,7 +1069,7 @@ const measureDriver = (cell, run) => {
 	const until = prediction.detects ? Infinity : after + valueIn(prediction.window, "s") * 1e9;
 	for (const match of driver.quiet ?? []) {
 		const line = events.find((event) => event.t > after && event.t <= until && matches(event, match));
-		if (line !== void 0) findings.push(`${wordsOf(match)} came ${between$2(after, line.t).value} s after the fault, where none should`);
+		if (line !== void 0) findings.push(`${wordsOf(match)} came ${between$3(after, line.t).value} s after the fault, where none should`);
 	}
 	const expected = prediction.detects ? void 0 : prediction.middlebox?.forgets;
 	if (!prediction.detects && (run.middlebox !== void 0 || expected !== void 0)) {
@@ -1035,14 +1079,14 @@ const measureDriver = (cell, run) => {
 		for (const socket of socketsOf(events)) {
 			const forgot = middlebox.find((line) => line.ev === "ct" && line["type"] === "DESTROY" && String(line["orig"] ?? "").startsWith(`${socket.local}>`) && line.t > socket.opened && (socket.ended === null || line.t < socket.ended) && line.t <= until);
 			if (expected === void 0 || socket.local !== own) {
-				if (forgot !== void 0) findings.push(`a router forgot the flow of the ${socket.role} socket ${between$2(after, forgot.t).value} s after the fault, where it should keep it`);
+				if (forgot !== void 0) findings.push(`a router forgot the flow of the ${socket.role} socket ${between$3(after, forgot.t).value} s after the fault, where it should keep it`);
 				continue;
 			}
 			const last = referenceOf(run, "last-received", driver.socket, false);
 			if (forgot === void 0) findings.push(`no router forgot the flow of the ${socket.role} socket in the window, where it should forget it`);
 			else if (last === void 0) findings.push(`the ${socket.role} socket got no segment before the fault`);
 			else {
-				const at = between$2(last, forgot.t).value;
+				const at = between$3(last, forgot.t).value;
 				const [low, high] = [round(valueIn(expected.low, "s") - noise), round(valueIn(expected.high, "s") + noise)];
 				if (at < low || at > high) findings.push(`a router forgot the flow of the ${socket.role} socket ${at} s after its last segment, outside [${low}, ${high}] s`);
 			}
@@ -1059,7 +1103,7 @@ const measureDriver = (cell, run) => {
 				elapsed: null
 			};
 		}
-		const elapsed = between$2(from, line.t);
+		const elapsed = between$3(from, line.t);
 		const name = nameOf$1(line, step.ends);
 		if (step.name !== "OK" && name !== step.name) findings.push(`${step.what}: it ended with ${name}, not ${step.name}`);
 		const [low, high] = [round(valueIn(step.low, "s") - noise), round(valueIn(step.high, "s") + noise)];
@@ -1093,7 +1137,7 @@ const measureDriver = (cell, run) => {
 			...nothing,
 			...stepped,
 			outcome: { kind: "answered" },
-			elapsed: between$2(from, line.t),
+			elapsed: between$3(from, line.t),
 			void: voided,
 			finding: findings[0] ?? null
 		};
@@ -1136,15 +1180,266 @@ const measureDriver = (cell, run) => {
 		...base,
 		...stepped,
 		outcome,
-		elapsed: between$2(reference, ended),
-		socket: seen === void 0 ? null : between$2(reference, seen.t),
-		noticed: report === void 0 || seen === void 0 ? null : between$2(report.at, seen.t),
-		steps: (prediction.from === "last-received" ? records.filter((record) => record.ev === "probe" && record.what === "keepalive" && record.mono > reference) : []).map((record) => between$2(reference, record.mono)),
+		elapsed: between$3(reference, ended),
+		socket: seen === void 0 ? null : between$3(reference, seen.t),
+		noticed: report === void 0 || seen === void 0 ? null : between$3(report.at, seen.t),
+		steps: (prediction.from === "last-received" ? records.filter((record) => record.ev === "probe" && record.what === "keepalive" && record.mono > reference) : []).map((record) => between$3(reference, record.mono)),
 		void: voided,
 		finding: findings[0] ?? null
 	};
 };
 //#endregion
+//#region src/queue.ts
+/** This bit is the SYN of the flags of an `rx` record of the prober. */
+const SYN = 2;
+/** This function gives the time between two stamps of CLOCK_MONOTONIC. */
+const between$2 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
+const numberOf$1 = (line, field) => {
+	const value = line[field];
+	return typeof value === "number" ? value : 0;
+};
+/** This function says if a time is inside a band. */
+const inside = (time, band) => {
+	const seconds = valueIn(time, "s");
+	return seconds >= valueIn(band.low, "s") && seconds <= valueIn(band.high, "s");
+};
+/** This function gives the words of a band. */
+const bandWords = (band) => `[${valueIn(band.low, "s")}, ${valueIn(band.high, "s")}] s`;
+const waitsOf = (from, stamps) => stamps.map((stamp, at) => between$2(at === 0 ? from : stamps[at - 1], stamp));
+/** This function finds the records of the sockets of the client that had this local address. */
+const socketsAt = (run, local) => {
+	const cookies = new Set(run.kernel.filter((record) => record.netns === run.netns && record.src === local).map((record) => record.cookie));
+	return run.kernel.filter((record) => record.netns === run.netns && cookies.has(record.cookie));
+};
+const readConnection = (name, run, expected) => {
+	const lineOf = (ev) => run.events.find((event) => event.ev === ev && event["name"] === name);
+	const dial = lineOf("dial");
+	const connected = lineOf("connected");
+	const failed = lineOf("failed");
+	const state = lineOf("state");
+	const socket = dial === void 0 ? [] : socketsAt(run, dial["local"]);
+	const sent = socket.filter((record) => record.ev === "retrans").map((record) => record.mono);
+	const rx = socket.filter((record) => record.ev === "rx" && ((record.seg?.flags ?? 0) & SYN) !== 0);
+	const from = dial?.t ?? 0;
+	const start = expected?.ends === null || expected?.ends === void 0 ? void 0 : lineOf(expected.ends.from);
+	const last = sent.at(-1);
+	return {
+		name,
+		connected: connected !== void 0,
+		syns: waitsOf(from, connected === void 0 ? sent : sent.filter((stamp) => stamp < connected.t)),
+		synAcks: waitsOf(from, rx.map((record) => record.mono)),
+		retransmissions: sent.length,
+		errno: failed === void 0 ? null : String(failed["errno"]),
+		ended: failed === void 0 || start === void 0 ? null : between$2(start.t, failed.t),
+		afterLast: failed === void 0 || last === void 0 ? null : between$2(last, failed.t),
+		unacked: state === void 0 ? null : numberOf$1(state, "unacked")
+	};
+};
+/** This function reads one run against the band of each wait in a list, and gives the words of each miss. */
+const waitFindings = (what, seen, bands) => {
+	if (seen.length !== bands.length) return [`${what}: ${seen.length} waits, not ${bands.length}`];
+	return seen.flatMap((wait, at) => inside(wait, bands[at]) ? [] : [`${what}: wait ${at} of ${valueIn(wait, "s")} s, outside ${bandWords(bands[at])}`]);
+};
+const connectionFindings = (expected, read, expiredAt, stamps) => {
+	const { name, ends } = expected;
+	const found = [];
+	if (read.connected !== expected.connects) found.push(`${name}: ${read.connected ? "connected" : "did not connect"}, against the prediction`);
+	if (expected.syns !== void 0) found.push(...waitFindings(`${name} SYN`, read.syns, expected.syns));
+	if (expected.synAcks !== void 0) found.push(...waitFindings(`${name} SYN-ACK`, read.synAcks, expected.synAcks));
+	if (expected.unacked !== void 0 && read.unacked !== expected.unacked) found.push(`${name}: ${read.unacked ?? "no"} segments without an ACK, not ${expected.unacked}`);
+	if (ends === null) {
+		if (read.errno !== null) found.push(`${name}: ended with ${read.errno}, and the prediction has no end`);
+		return found;
+	}
+	if (read.errno !== ends.errno) found.push(`${name}: ended with ${read.errno ?? "nothing"}, not ${ends.errno}`);
+	if (read.ended === null) return [...found, `${name}: no time for its end`];
+	if (!inside(read.ended, ends.within)) found.push(`${name}: its end came at ${valueIn(read.ended, "s")} s, outside ${bandWords(ends.within)}`);
+	if (ends.retransmitted === void 0) return found;
+	if (read.afterLast === null || !inside(read.afterLast, ends.retransmitted)) found.push(`${name}: its end came ${read.afterLast === null ? "with no retransmission" : `${valueIn(read.afterLast, "s")} s after its last retransmission`}, outside ${bandWords(ends.retransmitted)}`);
+	if (expiredAt === null) return found;
+	const before = stamps.at(-2);
+	const last = stamps.at(-1);
+	if (last === void 0 || last < expiredAt || before !== void 0 && before > expiredAt) found.push(`${name}: its last two retransmissions do not span the expiry`);
+	return found;
+};
+/**
+* This function reads one run of a queue cell. The client lines give the connections and the
+* counters at the two ends of the run. The target lines give the count of the queue. The records of
+* the prober give the waits of each socket.
+*/
+const measureQueue = (run, expected) => {
+	const { events, kernel } = run;
+	const lineOf = (ev) => events.find((event) => event.ev === ev);
+	const base = {
+		environment: run.environment,
+		index: run.index,
+		outcome: { kind: "answered" },
+		elapsed: null,
+		socket: null,
+		noticed: null,
+		steps: [],
+		probes: 0,
+		retransmissions: kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length,
+		acks: 0
+	};
+	const voided = (why) => ({
+		...base,
+		void: why,
+		finding: null
+	});
+	const opened = lineOf("opened");
+	const stats = lineOf("stats");
+	if (opened === void 0) return voided("the target did not open its listener");
+	if (stats === void 0 || lineOf("done") === void 0) return voided("the client did not reach its end");
+	if (run.lost === null) return voided("the prober wrote no clock line after the run, so nothing says what it dropped");
+	if (run.lost > 0) return voided(`the prober dropped ${run.lost} records`);
+	const lines = (run.server ?? []).filter((line) => line.ev === "queue" && line.t >= opened.t);
+	if (lines.length === 0) return voided("the target wrote no queue line in the run");
+	const named = [...new Set(events.filter((event) => event.ev === "dial").map((event) => String(event["name"])))];
+	const predicted = new Map(expected.connections.map((each) => [each.name, each]));
+	const connections = named.map((name) => readConnection(name, run, predicted.get(name)));
+	const admitted = expected.admitted === void 0 ? void 0 : connections.find((each) => each.name === expected.admitted);
+	if (expected.admitted !== void 0 && (admitted === void 0 || !admitted.connected || admitted.unacked !== 0)) return voided(`${expected.admitted} did not take the last place in the queue: the burst did not land in its sequence`);
+	const change = (field) => numberOf$1(stats, field) - numberOf$1(opened, field);
+	const burst = events.find((event) => event.ev === "dial" && event["name"] === "in")?.t ?? opened.t;
+	const counted = expected.expiry?.timeouts;
+	const expired = counted === void 0 ? void 0 : lines.find((line) => numberOf$1(line, "timeouts") - numberOf$1(opened, "timeouts") >= counted);
+	const accepted = events.find((event) => event.ev === "accepted");
+	const rise = expected.rise?.count ?? 0;
+	const low = accepted === void 0 ? -1 : lines.findIndex((line) => line.t >= accepted.t && numberOf$1(line, "count") < rise);
+	const risen = low < 0 ? void 0 : lines.slice(low).find((line) => numberOf$1(line, "count") >= rise);
+	const drained = events.filter((event) => event.ev === "accepted").reduce((sum, event) => sum + (Array.isArray(event["peers"]) ? event["peers"].length : 0), 0);
+	const queue = {
+		max: numberOf$1(opened, "max"),
+		held: Math.max(...lines.map((line) => numberOf$1(line, "count"))),
+		connected: connections.filter((each) => each.connected).length,
+		overflows: change("overflows"),
+		drops: change("drops"),
+		cookies: change("cookies"),
+		timeouts: change("timeouts"),
+		resets: change("resets"),
+		drained,
+		expiry: expired === void 0 ? null : between$2(burst, expired.t),
+		rise: risen === void 0 ? null : between$2(burst, risen.t),
+		connections
+	};
+	const found = [];
+	if (queue.max !== expected.max) found.push(`the limit is ${queue.max}, not ${expected.max}`);
+	if (queue.held !== expected.held) found.push(`the queue held ${queue.held}, not ${expected.held}`);
+	if (queue.connected !== expected.connected) found.push(`${queue.connected} connected, not ${expected.connected}`);
+	if (queue.overflows < expected.overflows.low || queue.overflows > expected.overflows.high) found.push(`${queue.overflows} overflows, outside [${expected.overflows.low}, ${expected.overflows.high}]`);
+	if (queue.resets !== expected.resets) found.push(`${queue.resets} resets, not ${expected.resets}`);
+	if (queue.cookies !== 0) found.push(`${queue.cookies} SYN cookies, and the law has none`);
+	if (expected.drained !== void 0 && queue.drained !== expected.drained) found.push(`${queue.drained} accepted, not ${expected.drained}`);
+	if (expected.expiry !== void 0 && (queue.expiry === null || !inside(queue.expiry, expected.expiry.within))) found.push(`the expiry came at ${queue.expiry === null ? "no time" : `${valueIn(queue.expiry, "s")} s`}, outside ${bandWords(expected.expiry.within)}`);
+	if (expected.rise !== void 0 && (queue.rise === null || !inside(queue.rise, expected.rise.within))) found.push(`the count got back to ${expected.rise.count} at ${queue.rise === null ? "no time" : `${valueIn(queue.rise, "s")} s`}, outside ${bandWords(expected.rise.within)}`);
+	for (const each of expected.connections) {
+		const read = connections.find((connection) => connection.name === each.name);
+		if (read === void 0) {
+			found.push(`${each.name}: the client did not dial it`);
+			continue;
+		}
+		const dial = events.find((event) => event.ev === "dial" && event["name"] === each.name);
+		const stamps = socketsAt(run, dial?.["local"]).filter((record) => record.ev === "retrans").map((record) => record.mono);
+		found.push(...connectionFindings(each, read, expired?.t ?? null, stamps));
+	}
+	return {
+		...base,
+		void: null,
+		finding: found.length === 0 ? null : found.join(". "),
+		queue
+	};
+};
+//#endregion
+//#region src/ports.ts
+/** This errno ends a search that found no port. */
+const NO_PORT = "EADDRNOTAVAIL";
+const portsOf = (line) => Array.isArray(line["ports"]) ? line["ports"].filter((port) => typeof port === "number") : [];
+/** This function says if the ports of the parity of the low port come first in one search. */
+const lowFirst = (ports, low) => {
+	const odd = ports.findIndex((port) => (port - low) % 2 !== 0);
+	return odd < 0 || ports.slice(odd).every((port) => (port - low) % 2 !== 0);
+};
+/**
+* This function reads one run of a port cell. The `round` lines give each search. The `closing` and
+* `freed` lines give when each held port comes back. The records of the prober give the
+* retransmissions.
+*/
+const measurePorts = (run, expected) => {
+	const { events, kernel } = run;
+	const retransmissions = kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length;
+	const base = {
+		environment: run.environment,
+		index: run.index,
+		outcome: { kind: "answered" },
+		elapsed: null,
+		socket: null,
+		noticed: null,
+		steps: [],
+		probes: 0,
+		retransmissions,
+		acks: 0
+	};
+	const voided = (why) => ({
+		...base,
+		void: why,
+		finding: null
+	});
+	if (events.find((event) => event.ev === "done") === void 0) return voided("the client did not reach its end");
+	if (run.lost === null) return voided("the prober wrote no clock line after the run, so nothing says what it dropped");
+	if (run.lost > 0) return voided(`the prober dropped ${run.lost} records`);
+	const rounds = events.filter((event) => event.ev === "round");
+	const closing = events.find((event) => event.ev === "closing");
+	const freedLines = events.filter((event) => event.ev === "freed");
+	const counts = {};
+	for (const round of rounds) {
+		const count = String(portsOf(round).length);
+		counts[count] = (counts[count] ?? 0) + 1;
+	}
+	const seen = [...new Set(rounds.flatMap((round) => portsOf(round).map((port) => port - expected.low)))].sort((a, b) => a - b);
+	const refusal = Math.max(0, ...rounds.map((round) => typeof round["refusal"] === "number" ? round["refusal"] : 0));
+	const first = freedLines[0];
+	const last = freedLines.at(-1);
+	const ports = {
+		rounds: rounds.length,
+		counts,
+		seen,
+		errnos: [...new Set(rounds.map((round) => String(round["errno"])))],
+		refusal: quantity(Math.round(refusal / 1e3) / 1e6, "s"),
+		firstFreed: closing === void 0 || first === void 0 ? null : between$2(closing.t, first.t),
+		lastFreed: closing === void 0 || last === void 0 ? null : between$2(closing.t, last.t),
+		freed: freedLines.length,
+		tries: last === void 0 || typeof last["tries"] !== "number" ? 0 : last["tries"]
+	};
+	const found = [];
+	const searches = expected.rounds * expected.destinations;
+	if (ports.rounds !== searches) found.push(`${ports.rounds} searches, not ${searches}`);
+	const stray = Object.keys(counts).filter((count) => !expected.counts.includes(Number(count)));
+	if (stray.length > 0) found.push(`a search gave ${stray.join(" or ")} connects, outside ${expected.counts.join(" or ")}`);
+	if (seen.join() !== expected.seen.join()) found.push(`the ports at offsets ${seen.join(" ")} were used, not ${expected.seen.join(" ")}`);
+	if (ports.errnos.join() !== NO_PORT) found.push(`the searches ended with ${ports.errnos.join(" and ")}, not ${NO_PORT}`);
+	if (valueIn(ports.refusal, "s") > valueIn(expected.refusedWithin, "s")) found.push(`a refused connect took ${valueIn(ports.refusal, "s")} s, above ${valueIn(expected.refusedWithin, "s")} s`);
+	if (expected.share !== void 0) {
+		const { count, low, high } = expected.share;
+		const share = (counts[String(count)] ?? 0) / Math.max(1, ports.rounds);
+		if (share < low || share > high) found.push(`the share of ${count} connects is ${share}, outside [${low}, ${high}]`);
+	}
+	if (expected.lowFirst === true && !rounds.every((round) => lowFirst(portsOf(round), expected.low))) found.push("a search got a port of the other parity before each port of the parity of the low port");
+	if (expected.retransmissions !== void 0 && retransmissions !== expected.retransmissions) found.push(`${retransmissions} retransmissions, not ${expected.retransmissions}`);
+	if (expected.freed !== void 0) {
+		const held = Math.max(...expected.counts);
+		if (ports.freed !== held) found.push(`${ports.freed} ports came back, not ${held}`);
+		for (const [name, time] of [["first", ports.firstFreed], ["last", ports.lastFreed]]) if (time === null || !inside(time, expected.freed)) found.push(`the ${name} port came back at ${time === null ? "no time" : `${valueIn(time, "s")} s`}, outside ${bandWords(expected.freed)}`);
+		if (ports.tries === 0) found.push("no connect was refused before the ports came back");
+	}
+	return {
+		...base,
+		void: null,
+		finding: found.length === 0 ? null : found.join(". "),
+		ports
+	};
+};
+//#endregion
 //#region src/recovery.ts
 /**
 * This module holds the pure parts of a recovery drill. These are the scripts of the shell, the reader
@@ -1516,12 +1811,32 @@ const writesOf = (run, after, primary) => {
 			held: kept === null ? null : kept.has(n)
 		};
 	});
+	const duplicates = holds === null ? null : [...holds].sort((a, b) => a - b).filter((n, at, all) => all[at - 1] === n);
+	const oldHolds = run.group?.oldHolds;
 	return {
 		sent: sent.length,
 		acked: acks.size,
 		old,
 		lost,
-		unknown
+		unknown,
+		duplicates,
+		...oldHolds === void 0 ? {} : { sets: setsOf(run, acks.keys(), unknown, new Set(oldHolds)) }
+	};
+};
+/**
+* This function splits the writes of a pair by its two final reads. A write with no reply is `kept`
+* on the new primary. It is `stranded` on the old primary alone, or `absent` from each server.
+*/
+const setsOf = (run, acked, unknown, old) => {
+	const kept = unknown.filter((each) => each.held === true).length;
+	const stranded = unknown.filter((each) => each.held !== true && old.has(each.n)).length;
+	return {
+		unknown: unknown.length,
+		kept,
+		stranded,
+		absent: unknown.length - kept - stranded,
+		missing: [...acked].filter((n) => !old.has(n)).length,
+		warnings: run.events.filter((event) => event.ev === "notice" && event["severity"] === "WARNING").length
 	};
 };
 /** This function gives the errors that the client met after an instant: each kind, its first time, and its count. */
@@ -1632,9 +1947,10 @@ const measureGroup = (cell, run) => {
 	const heal = run.heal ?? null;
 	const lands = expected?.lands ?? "new";
 	const cluster = cell.cluster !== void 0;
-	const replica = cell.replicaSet !== void 0 || cluster;
+	const kafka = cell.kafka !== void 0;
+	const replica = cell.replicaSet !== void 0 || cluster || kafka;
 	const stall = prediction.detects === false && prediction.recovers !== void 0;
-	const stages = cluster ? clusterStagesOf(run, t0, run.fault.t1) : replica ? replicaSetStagesOf(run, t0, run.fault.t1, lands) : stagesOf(timeline, t0);
+	const stages = kafka ? {} : cluster ? clusterStagesOf(run, t0, run.fault.t1) : replica ? replicaSetStagesOf(run, t0, run.fault.t1, lands) : stagesOf(timeline, t0);
 	const after = lands === "same" && heal !== null && (!replica || stall) ? heal.t0 : t0;
 	const sentAt = /* @__PURE__ */ new Map();
 	for (const event of events) if (event.ev === "sent") sentAt.set(Number(event["n"]), event.t);
@@ -1666,14 +1982,14 @@ const measureGroup = (cell, run) => {
 	if (voided === null && split !== void 0 && (elected === void 0 || elected.t > split.t)) voided = "the election split its votes and runs again after 2 × failover-timeout, which the band does not bound";
 	const odown = stages["+odown"];
 	if (voided === null && odown !== void 0 && stages["+try-failover"] === void 0) voided = "no Sentinel started a failover after +odown: each was in the hold-off of 2 × failover-timeout after an earlier attempt";
-	if (voided === null && cell.hold !== void 0 && prediction.detects !== null) {
+	if (voided === null && cell.hold !== void 0 && (prediction.detects !== null || kafka)) {
 		if (heal === null) voided = "the fault never lifted";
 		else {
 			const late = (heal.t0 - t0) / 1e9 - valueIn(cell.hold, "s");
 			if (late > valueIn(INJECT, "s")) voided = `the lift came ${Math.round(late * 1e4) / 10} ms after its hold: the injector was late, not the group`;
 		}
 	}
-	const firstAddress = cluster ? String(events.find((event) => event.ev === "connected")?.["at"] ?? "") : `${run.group?.members[primary]?.ip}:${(cell.group ?? cell.replicaSet)?.port}`;
+	const firstAddress = cluster ? String(events.find((event) => event.ev === "connected")?.["at"] ?? "") : `${run.group?.members[primary]?.ip}:${(cell.group ?? cell.replicaSet ?? cell.kafka)?.port}`;
 	const assumed = prediction.detects === null ? void 0 : prediction.rtt;
 	const first = kernel.find((record) => record.netns === run.netns && record.dst === firstAddress && record.tcp?.srtt !== void 0);
 	const rtt = first?.tcp?.srtt === void 0 ? null : valueIn(first.tcp.srtt, "us") / 1e3;
@@ -1693,11 +2009,13 @@ const measureGroup = (cell, run) => {
 	const findings = [];
 	const switched = replica ? stages["elected"] : stages["+switch-master"];
 	if (expected?.failover === false && switched !== void 0) findings.push(`the group failed over ${switched.value} s after the fault, where it should not`);
+	const leaderAfter = run.group?.after ?? null;
+	if (kafka && expected !== void 0 && leaderAfter !== null && expected.failover !== (leaderAfter !== primary)) findings.push(expected.failover ? `the leader stayed on ${primary}, where it should change` : `the leader moved to ${leaderAfter}, where it should stay on ${primary}`);
 	for (const stage of expected?.quiet ?? []) {
 		const at = stages[stage];
 		if (at !== void 0) findings.push(`${stage} came ${at.value} s after the fault, where it should not`);
 	}
-	if (accepted !== void 0 && lands === "same" && !cluster && to !== primary) findings.push(`the client wrote to ${to ?? String(accepted["at"])}, where it should stay on ${primary}`);
+	if (accepted !== void 0 && lands === "same" && !cluster && !kafka && to !== primary) findings.push(`the client wrote to ${to ?? String(accepted["at"])}, where it should stay on ${primary}`);
 	for (const error of expected?.errors ?? []) {
 		const met = group.errors.find((each) => each.name === error.name);
 		const noise = valueIn(noiseOf(cell), "s");
@@ -1710,9 +2028,22 @@ const measureGroup = (cell, run) => {
 		if (lost === null) findings.push("the group did not converge, thus the count of its lost writes is not known");
 		else if (lost.length < low || lost.length > high) findings.push(`the group lost ${lost.length} accepted writes, outside [${low}, ${high}]`);
 	}
+	if (expected?.duplicates !== void 0) {
+		const { low, high } = expected.duplicates;
+		const copies = group.writes.duplicates ?? null;
+		if (copies === null) findings.push("the group did not converge, thus the count of its extra copies is not known");
+		else if (run.group?.holds?.length === 0) findings.push("the group holds no write, thus the count of its extra copies says nothing");
+		else if (copies.length < low || copies.length > high) findings.push(`the group holds ${copies.length} extra copies, outside [${low}, ${high}]`);
+	}
+	const { sets } = group.writes;
+	for (const [name, band] of Object.entries(expected?.outcomes ?? {})) {
+		const count = sets?.[name];
+		if (count === void 0) findings.push(`the run has no count of ${name}`);
+		else if (count < band.low || count > band.high) findings.push(`the count of ${name} is ${count}, outside [${band.low}, ${band.high}]`);
+	}
 	if (group.recovery !== void 0) findings.push(...recoveryFindings(group.recovery, group.writes));
 	if (accepted === void 0) {
-		if (prediction.detects !== null) findings.push(lands === "new" ? "the client never wrote to a new primary in the window" : "the client never wrote again in the window");
+		if (prediction.detects !== null || kafka) findings.push(lands === "new" ? "the client never wrote to a new primary in the window" : "the client never wrote again in the window");
 		return {
 			...silent,
 			void: voided,
@@ -1981,7 +2312,7 @@ const requestOf = (events) => {
 };
 /** This function reads the socket of one run against its cell, as `measure` does without the second request of an HTTP client. */
 const measureSocket = (cell, run) => {
-	if (cell.group !== void 0 || cell.replicaSet !== void 0 || cell.cluster !== void 0) {
+	if (cell.group !== void 0 || cell.replicaSet !== void 0 || cell.cluster !== void 0 || cell.pair !== void 0 || cell.kafka !== void 0) {
 		const read = measureGroup(cell, run);
 		return {
 			environment: run.environment,
@@ -1999,6 +2330,8 @@ const measureSocket = (cell, run) => {
 			group: read.group
 		};
 	}
+	if (cell.prediction.detects === null && cell.prediction.queue !== void 0) return measureQueue(run, cell.prediction.queue);
+	if (cell.prediction.detects === null && cell.prediction.ports !== void 0) return measurePorts(run, cell.prediction.ports);
 	if (cell.prediction.detects !== null && cell.prediction.driver !== void 0) return measureDriver(cell, run);
 	const { events, kernel } = run;
 	const ready = events.find((event) => event.ev === "ready");
@@ -2155,12 +2488,13 @@ const measureSocket = (cell, run) => {
 		}, run)) };
 		const reconnect = prediction.reconnects === void 0 || ended === void 0 ? void 0 : reconnectOf(run, ended, cookie, drop?.line.t ?? ended);
 		const late = reconnect === void 0 || prediction.reconnects === void 0 ? null : reconnectFinding(prediction.reconnects, reconnect, noiseOf(cell));
+		const counted = prediction.counts === void 0 ? null : countFinding(prediction.counts, run.server ?? []);
 		return {
 			...base,
 			...nothing,
 			...chain,
 			...reconnect === void 0 ? {} : { reconnect },
-			finding: late,
+			finding: late ?? counted,
 			outcome: ended !== void 0 ? outcome : { kind: "answered" },
 			...from === void 0 || until === void 0 ? {} : { elapsed: between(from, until) },
 			noticed: prediction.from === void 0 ? null : noticed,
@@ -2918,7 +3252,7 @@ const TAIL_LINES = 60;
 const OUTPUT_LINES = 2e3;
 /** This time is how long a write of a hop waits for the line that confirms it, in milliseconds. */
 const CONFIRM_MS = 1e4;
-const sleep$4 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const sleep$6 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** This function gives the arguments of the injector for the options of a fault. */
 const optionArgs = (options) => [
 	...options.direction === void 0 ? [] : ["--direction", options.direction],
@@ -3460,6 +3794,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 		};
 		const proxies = inChain.filter((hop) => hop.hop.kind === "proxy");
 		return {
+			serverIp,
 			targetIp,
 			targetPort,
 			clientIp,
@@ -3524,7 +3859,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 				return stamped(where, [
 					"/opt/harness/inject",
 					kind,
-					peer,
+					options.peers?.join(",") ?? peer,
 					...optionArgs(options),
 					...via
 				]);
@@ -3534,7 +3869,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 				await output(where, [
 					"/opt/harness/inject",
 					LIFT[kind],
-					peer,
+					options.peers?.join(",") ?? peer,
 					...optionArgs(options),
 					...via
 				]);
@@ -3586,6 +3921,18 @@ const openLab = async (target, kind = "tcp", topology) => {
 				if (target.close === void 0) throw new Error(`${target.image} names no way to close its clients`);
 				await output(server, [...target.close]);
 			},
+			async exec(argv) {
+				return output(server, [...argv]);
+			},
+			async sibling(siblingImage, argv, ready) {
+				const started = await new GenericContainer(siblingImage).withNetwork(last).withWaitStrategy(Wait.forLogMessage(ready)).withCommand([...argv]).start();
+				opened.push(started);
+				return {
+					ip: await ipOn(started, last),
+					netns: await netnsOf(started),
+					exec: (command) => output(started, [...command])
+				};
+			},
 			async agent(agentImage, argv, options = {}) {
 				const Shared = shared(GenericContainer);
 				const seen = lines();
@@ -3726,7 +4073,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 						};
 						while (!confirmed()) {
 							if (Date.now() - started > CONFIRM_MS) throw new Error(`the hop ${event.hop} did not confirm the write of ${event.write.path}: no line "${confirm}" in ${CONFIRM_MS / 1e3} s`);
-							await sleep$4(50);
+							await sleep$6(50);
 						}
 					}
 					return stamp;
@@ -3864,6 +4211,213 @@ const openLab = async (target, kind = "tcp", topology) => {
 	}
 };
 //#endregion
+//#region src/standby.ts
+/**
+* This module is the second server of a pair (`Cell.pair`). The lab of the cell has the primary, the
+* topology in front of it, the injector and the prober. This module adds one standby on the network
+* of the primary, and it does one run:
+*
+* 1. The client writes a stream of sequence numbers through the topology.
+* 2. The fault cuts the primary from the standby, or from each peer. The cut stays to the end.
+* 3. At the end of the hold, the injector stops each process of the primary with SIGKILL.
+* 4. The harness promotes the standby, and reads what it holds.
+* 5. The harness starts the old primary, and reads what its crash recovery gives.
+*
+* The standby has no way to the WAL of the old primary, because the primary stops under the cut. A
+* promotion cannot be undone, thus each run has its own environment (`Cell.fresh`).
+*/
+/** This table holds one row for each write of the client. */
+const TABLE = "writes";
+/** This time bounds each wait of the setup of the pair, in ms. */
+const SETUP_MS = 6e4;
+/** This count is the acknowledged writes before the fault: the stream is steady. */
+const STEADY$4 = 20;
+/** This time is the wait after the stop of the primary, in ms: each writer sends its last write in it. */
+const LAST_WRITE_MS = 2e3;
+/** This time bounds the wait for the last line of the client, in ms. */
+const STOP_MS = 5e3;
+/** This text is the line of the log of a standby that serves reads (src/backend/postmaster/postmaster.c:3740). */
+const STANDBY_READY = "database system is ready to accept read-only connections";
+/** This text is the write of the client, with its sequence number. */
+const INSERT = `INSERT INTO ${TABLE} VALUES ($n)`;
+const sleep$5 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/** This function gives the argv of one statement in psql, on the server in the same container. */
+const psql = (statement) => [
+	"psql",
+	"-h",
+	"127.0.0.1",
+	"-U",
+	"postgres",
+	"-At",
+	"-c",
+	statement
+];
+/** This function gives the command of a standby: a base backup of the primary, then the server in recovery. */
+const standbyCommand = (primary) => [
+	"sh",
+	"-c",
+	`chown postgres:postgres "$PGDATA"; chmod 0700 "$PGDATA"; until gosu postgres pg_basebackup -h ${primary} -U postgres -D "$PGDATA" -X stream -c fast -R; do gosu postgres sh -c 'rm -rf "$PGDATA"/*'; sleep 0.5; done; exec gosu postgres postgres`
+];
+/** This command starts the old primary again, after the stop of each of its processes. Its crash recovery runs before the wait ends. */
+const RESTART = [
+	"sh",
+	"-c",
+	"exec gosu postgres pg_ctl -D \"$PGDATA\" -l /tmp/restart.log -w -t 60 start"
+];
+/** This function reads the output of a query for sequence numbers into a sorted list. */
+const numbersOf = (text) => text.split("\n").map((line) => line.trim()).filter((line) => line !== "").map(Number).sort((a, b) => a - b);
+/** This function gives the arguments of the client of a run. */
+const pairArgs = (cell, lab) => {
+	const pair = cell.pair;
+	const hold = cell.hold === void 0 ? 0 : valueIn(cell.hold, "s");
+	return [
+		lab.targetIp,
+		String(lab.targetPort),
+		"--mode",
+		"stream",
+		"--interval-ms",
+		String(valueIn(pair.interval, "ms")),
+		"--writers",
+		String(pair.writers),
+		"--query",
+		INSERT,
+		"--max-s",
+		String(Math.ceil(hold) + 60),
+		...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
+	];
+};
+/** This function gives the peers of the cut of a pair: the standby, or the standby and the peer of the primary in the topology. */
+const peersOf$1 = (pair, standby, lab) => pair.cuts === "standby" ? [standby] : [standby, lab.topology?.targetPeer ?? lab.clientIp];
+/** This function gives the instants of the cancels of a hold, in ns from the fault: equal parts of the hold. */
+const cancelsOf = (pair, holdNs) => {
+	const count = pair.cancels ?? 0;
+	return Array.from({ length: count }, (_, at) => (at + 1) * holdNs / (count + 1));
+};
+/** This function asks until the primary reports the standby in a state of replication. */
+const untilStreaming = async (lab, state) => {
+	const started = Date.now();
+	for (;;) {
+		const seen = await lab.exec(psql("SELECT sync_state FROM pg_stat_replication WHERE state = 'streaming'"));
+		if (seen === state) return;
+		if (Date.now() - started > SETUP_MS) throw new Error(`the standby did not stream as ${state} in ${SETUP_MS} ms: it reports "${seen}"`);
+		await sleep$5(100);
+	}
+};
+/**
+* This function opens the pair of an environment. The table comes first and the standby next: the
+* base backup has it. A synchronous standby is named last, thus no setup step waits for it.
+*/
+const openStandby = async (lab, cell) => {
+	const pair = cell.pair;
+	await lab.exec([
+		"sh",
+		"-c",
+		"echo \"host replication all all trust\" >> \"$PGDATA/pg_hba.conf\""
+	]);
+	await lab.exec(psql("SELECT pg_reload_conf()"));
+	await lab.exec(psql(`CREATE TABLE ${TABLE} (n integer PRIMARY KEY)`));
+	const standby = await lab.sibling(await targetImageOf(cell.target), standbyCommand(lab.serverIp), STANDBY_READY);
+	await untilStreaming(lab, "async");
+	if (pair.synchronous) {
+		await lab.exec(psql("ALTER SYSTEM SET synchronous_standby_names = '*'"));
+		await lab.exec(psql("SELECT pg_reload_conf()"));
+		await untilStreaming(lab, "sync");
+	}
+	return standby;
+};
+/** This function waits for a count of acknowledged writes. */
+const untilSteady = async (probe) => {
+	const started = Date.now();
+	while (probe.events.filter((event) => event.ev === "ack").length < STEADY$4) {
+		if (Date.now() - started > SETUP_MS) throw new Error(`the client did not write ${STEADY$4} times in ${SETUP_MS} ms`);
+		await sleep$5(10);
+	}
+};
+/** This function gives the count of lost records in the latest clock line of the prober. */
+const droppedSoFar$5 = (records) => {
+	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
+	return 0;
+};
+/**
+* This function does one run of a pair. A dry run writes and stops, with no fault: the warm-up of the
+* environment. It removes its rows, thus the numbers of the run start on an empty table.
+*/
+const pairOnce = async (lab, standby, cell, environment, index, dry = false) => {
+	const pair = cell.pair;
+	await lab.forget();
+	const from = lab.kernel.length;
+	const lostBefore = droppedSoFar$5(lab.kernel);
+	const probe = await lab.probe(pairArgs(cell, lab));
+	const holdNs = cell.hold === void 0 ? 0 : valueIn(cell.hold, "s") * 1e9;
+	let fault = null;
+	let heal = null;
+	try {
+		await probe.until(["ready"], 15e3);
+		await untilSteady(probe);
+		if (!dry) {
+			fault = await lab.fault("blackhole", "target", { peers: peersOf$1(pair, standby.ip, lab) });
+			const started = Date.now();
+			for (const at of cancelsOf(pair, holdNs)) {
+				await sleep$5(Math.max(0, at / 1e6 - (Date.now() - started)));
+				await lab.exec(psql("SELECT count(pg_cancel_backend(pid)) FROM pg_stat_activity WHERE wait_event = 'SyncRep'"));
+			}
+			heal = await lab.signal("KILL", fault.t0 + holdNs);
+			await sleep$5(LAST_WRITE_MS);
+			if (cell.target.kill !== void 0) await lab.exec(cell.target.kill);
+		}
+		await probe.go().catch(() => void 0);
+		await probe.until(["done"], STOP_MS);
+	} finally {
+		await probe.stop();
+	}
+	if (dry) await lab.exec(psql(`TRUNCATE ${TABLE}`));
+	const settled = await lab.settle();
+	const lost = settled === null ? null : settled - lostBefore;
+	const read = `SELECT n FROM ${TABLE} ORDER BY n`;
+	let holds = null;
+	let oldHolds = [];
+	if (!dry) {
+		await standby.exec(psql("SELECT pg_promote(true, 60)"));
+		holds = numbersOf(await standby.exec(psql(read)));
+		await lab.exec([...RESTART]);
+		oldHolds = numbersOf(await lab.exec(psql(read)));
+	}
+	return {
+		environment,
+		index,
+		events: [...probe.events],
+		fault,
+		...heal === null ? {} : { heal },
+		kernel: lab.kernel.slice(from),
+		netns: lab.netns.client,
+		targetNetns: lab.netns.target,
+		lost,
+		...lab.topology === null ? {} : {
+			topology: lab.topology,
+			middlebox: []
+		},
+		group: {
+			members: {
+				primary: {
+					ip: lab.serverIp,
+					netns: lab.netns.target
+				},
+				standby: {
+					ip: standby.ip,
+					netns: standby.netns
+				}
+			},
+			sentinels: {},
+			primary: "primary",
+			at: "primary",
+			timeline: [],
+			after: dry ? "primary" : "standby",
+			holds,
+			oldHolds
+		}
+	};
+};
+//#endregion
 //#region src/pins.ts
 /**
 * This module tells if a manifest pins each program that a run operated. A record that does not pin
@@ -3911,6 +4465,21 @@ const manifestProblems = (manifest) => [
 ];
 /** This function gives the problem of the commit of a record: no commit, or a text that is not a commit of git. */
 const commitProblems = (commit) => commit === void 0 ? ["the record has no commit of the code that operated"] : COMMIT.test(commit) ? [] : [`the record has the commit ${commit}, which is not 40 hex digits`];
+/** This problem is the problem of a checkout with changes: its commit does not hold the code that operated. */
+const DIRTY = "the checkout has changes to tracked files that no commit holds";
+/** This problem is the problem of a record of a local debug run. */
+const DEBUG = "the record is of a local debug run, and no page reads it";
+/**
+* This function gives each problem that stops a record from pinning the programs of its runs. A
+* record with a problem is not a record (SYS-101). The writer of the cells refuses it, and the read
+* of a schedule refuses it. A commit with changes to tracked files is not a pin.
+*/
+const recordProblems = (record) => [.../* @__PURE__ */ new Set([
+	...commitProblems(record.code?.commit),
+	...record.code?.dirty === true ? [DIRTY] : [],
+	...record.manifests.flatMap((manifest) => manifestProblems(manifest)),
+	...record.debug === true ? [DEBUG] : []
+])];
 //#endregion
 //#region src/group.ts
 /** This function gives the configuration file of a member: its port, its settings, and the primary that a replica follows. */
@@ -3929,19 +4498,19 @@ const sentinelConfig = (group, primary) => [
 ].join("\n");
 /**
 * This function gives the peers that a fault on a part cuts it from, by their addresses. `all` is
-* each other part and the client, `group` is the other parts, and `clients` is the client. `one`
-* and `one-way` are the first other member. The observer is not a peer, thus the timeline of the
+* each other part and the client, `group` is the other parts, and `clients` and `to-clients` are
+* the client. `one` and `one-way` are the first other member. The observer is not a peer, thus the timeline of the
 * group stays whole.
 */
 const peersOf = (parts, client, part, cuts) => {
 	const others = [...Object.entries(parts.members), ...Object.entries(parts.sentinels)].filter(([name]) => name !== part).map(([, at]) => at.ip);
-	if (cuts === "clients") return [client];
+	if (cuts === "clients" || cuts === "to-clients") return [client];
 	if (cuts === "group") return others;
 	if (cuts === "one" || cuts === "one-way") return Object.entries(parts.members).filter(([name]) => name !== part).slice(0, 1).map(([, at]) => at.ip);
 	return [...others, client];
 };
-/** This function gives the direction of the rules of a cut: `one-way` drops only the segments from the part. */
-const directionOf = (cuts) => cuts === "one-way" ? ["--direction", "out"] : [];
+/** This function gives the direction of the rules of a cut: `one-way` and `to-clients` drop only the segments from the part. */
+const directionOf = (cuts) => cuts === "one-way" || cuts === "to-clients" ? ["--direction", "out"] : [];
 /** This function reads the flat reply of a command line client into its fields: name, value, name, value. */
 const fieldsOf = (text) => {
 	const out = [];
@@ -4034,7 +4603,7 @@ const viewOf = (group, ips, sentinels, members) => {
 		why: "healthy"
 	};
 };
-const sleep$3 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const sleep$4 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** This time is the longest that a group takes to converge before a run, or after it. */
 const CONVERGE_MS = 24e4;
 /**
@@ -4229,7 +4798,7 @@ const openGroupLab = async (group, options = {}) => {
 					}));
 					calm = last.healthy ? calm + 1 : 0;
 					if (calm >= 2 && last.primary !== null) return last.primary;
-					await sleep$3(1e3);
+					await sleep$4(1e3);
 				}
 				throw new Error(`the group did not converge in ${within} ms: ${last.why}`);
 			},
@@ -4250,11 +4819,16 @@ const openGroupLab = async (group, options = {}) => {
 				]);
 			},
 			async holds(primary) {
-				return (await cli(primary, [
+				return holdsOf(await cli(primary, [
 					"--scan",
 					"--pattern",
 					"w:*"
-				])).split("\n").map((key) => Number(key.trim().slice(2))).filter((n) => Number.isInteger(n) && n > 0).sort((a, b) => a - b);
+				]), await cli(primary, [
+					"LRANGE",
+					"w",
+					"0",
+					"-1"
+				]));
 			},
 			partOf: roles,
 			async client(args) {
@@ -4309,7 +4883,7 @@ const openGroupLab = async (group, options = {}) => {
 				const started = Date.now();
 				while (await cli(member, ["PING"]).catch(() => "") !== "PONG") {
 					if (Date.now() - started > 3e4) throw new Error(`${member} did not start again`);
-					await sleep$3(200);
+					await sleep$4(200);
 				}
 			},
 			failover(coordinated) {
@@ -4425,7 +4999,7 @@ const openGroupLab = async (group, options = {}) => {
 	}
 };
 /** This count is the number of accepted writes that a run waits for before its fault: the stream is steady then. */
-const STEADY$2 = 20;
+const STEADY$3 = 20;
 /** This time is how long a run continues after its outcome, so that the record shows what came next. */
 const TAIL_MS$3 = 2e3;
 /** This function gives the address of a member, with the port of the group. */
@@ -4435,22 +5009,32 @@ const deadlineOf = (cell) => {
 	const { prediction } = cell;
 	return (prediction.detects === true ? valueIn(prediction.high, "s") : prediction.detects === false && prediction.recovers !== void 0 ? valueIn(prediction.recovers.high, "s") : valueIn(prediction.window, "s")) * 1e3 + 1e4;
 };
+/**
+* This function gives the sequence numbers that a group holds, in order, from the output of its
+* command line. They are the keys `w:<n>` of a stream of SET, and the entries of the list `w` of a
+* stream of RPUSH. A list keeps each copy, thus a number can repeat.
+*/
+const holdsOf = (keys, list) => [...keys.split("\n").map((key) => Number(key.trim().slice(2))), ...list.split("\n").map((entry) => Number(entry.trim()))].filter((n) => Number.isInteger(n) && n > 0).sort((a, b) => a - b);
 /** This function gives the arguments of the client of a group cell. */
 const clientArgs = (cell, lab) => {
+	const sentinels = cell.group.sentinels.map((name) => `${lab.sentinels[name].ip}:${cell.group.sentinelPort}`);
+	const driver = cell.probe.driver?.name === "ioredis" ? cell.probe.driver : void 0;
 	return [
 		"--sentinels",
-		cell.group.sentinels.map((name) => `${lab.sentinels[name].ip}:${cell.group.sentinelPort}`).join(","),
+		sentinels.join(","),
 		"--name",
 		cell.group.name,
 		"--interval-ms",
 		String(cell.probe.interval === void 0 ? 50 : valueIn(cell.probe.interval, "ms")),
 		"--max-s",
 		String(Math.ceil(deadlineOf(cell) / 1e3 + 120)),
-		...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
+		...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)],
+		...driver?.op === void 0 ? [] : ["--op", driver.op],
+		...driver?.window === void 0 ? [] : ["--window", String(driver.window)]
 	];
 };
 /** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
-const waitFor$2 = (client, match, within) => new Promise((resolve) => {
+const waitFor$3 = (client, match, within) => new Promise((resolve) => {
 	const started = Date.now();
 	const look = () => {
 		const found = client.events.find(match);
@@ -4466,7 +5050,7 @@ const subscriptionsOf = (timeline) => {
 	return [...latest.values()];
 };
 /** This function gives the count of lost records in the latest clock line of the prober. */
-const droppedSoFar$3 = (records) => {
+const droppedSoFar$4 = (records) => {
 	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
 	return 0;
 };
@@ -4488,7 +5072,7 @@ const groupOnce = async (lab, cell, environment, index) => {
 	await lab.forget();
 	const from = lab.kernel.length;
 	const timelineFrom = lab.timeline.length;
-	const lostBefore = droppedSoFar$3(lab.kernel);
+	const lostBefore = droppedSoFar$4(lab.kernel);
 	const client = await lab.client(clientArgs(cell, lab));
 	const lands = cell.prediction.group?.lands ?? "new";
 	const place = cell.faultAt ?? { role: "primary" };
@@ -4498,7 +5082,7 @@ const groupOnce = async (lab, cell, environment, index) => {
 	const undo = [];
 	try {
 		await client.until(["ready"], 6e4);
-		if (await waitFor$2(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$2, 3e4) === void 0) throw new Error(`the client did not write ${STEADY$2} times in 30 s`);
+		if (await waitFor$3(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$3, 3e4) === void 0) throw new Error(`the client did not write ${STEADY$3} times in 30 s`);
 		let healing = null;
 		switch (cell.fault) {
 			case "blackhole":
@@ -4512,7 +5096,7 @@ const groupOnce = async (lab, cell, environment, index) => {
 				undo.push(() => lab.lift(kind, part, cuts));
 				if (cell.stall !== void 0) {
 					const t1 = fault.t1;
-					const sent = await waitFor$2(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
+					const sent = await waitFor$3(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
 					if (sent !== void 0) healing = lab.lift(kind, part, cuts, sent.t + valueIn(cell.stall, "s") * 1e9);
 				} else if (cell.hold !== void 0) healing = lab.lift(kind, part, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
 				break;
@@ -4546,11 +5130,11 @@ const groupOnce = async (lab, cell, environment, index) => {
 		const t0 = fault.t0;
 		const old = addressOf(lab, primary);
 		const deadline = deadlineOf(cell);
-		if ((lands === "new" ? await waitFor$2(client, (event) => event.ev === "ack" && event.t > t0 && typeof event["at"] === "string" && event["at"] !== old, deadline) : await (async () => {
+		if ((lands === "new" ? await waitFor$3(client, (event) => event.ev === "ack" && event.t > t0 && typeof event["at"] === "string" && event["at"] !== old, deadline) : await (async () => {
 			await healed;
 			const after = heal?.t0 ?? t0;
-			return waitFor$2(client, (event) => event.ev === "ack" && event.t > after, deadline);
-		})()) !== void 0) await sleep$3(TAIL_MS$3);
+			return waitFor$3(client, (event) => event.ev === "ack" && event.t > after, deadline);
+		})()) !== void 0) await sleep$4(TAIL_MS$3);
 		await healed;
 	} finally {
 		for (const step of undo.reverse()) await step().catch(() => void 0);
@@ -4643,6 +5227,10 @@ const cutOf = (set, members, client, part, cuts) => {
 		peers: others.slice(0, 1),
 		direction: "out"
 	};
+	if (cuts === "to-clients") return {
+		peers: [client],
+		direction: "out"
+	};
 	return { peers: [...others, client] };
 };
 /** This function gives the command of the server of each member. The server is the child of a shell, thus a fin can stop it and the container stays up. */
@@ -4714,7 +5302,7 @@ const statusViewOf = (set, ips, status) => {
 	};
 };
 /** This function waits for a time, in ms. */
-const sleep$2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const sleep$3 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** These are the arguments of replSetStepDown: the time that the old primary cannot stand again, and the wait for a secondary to catch up. */
 const STEP_DOWN = {
 	seconds: 60,
@@ -4862,7 +5450,7 @@ const openReplicaSetLab = async (set, backups = false) => {
 				calm = last.healthy ? calm + 1 : 0;
 				if (last.primary !== null) known = last.primary;
 				if (calm >= 2 && last.primary !== null) return last.primary;
-				await sleep$2(1e3);
+				await sleep$3(1e3);
 			}
 			throw new Error(`the set did not converge in ${within} ms: ${last.why}`);
 		};
@@ -4968,7 +5556,7 @@ const openReplicaSetLab = async (set, backups = false) => {
 				const started = Date.now();
 				while (!(await shell(member, "print(db.hello().ok)").catch(() => "")).includes("1")) {
 					if (Date.now() - started > 6e4) throw new Error(`${member} did not start again`);
-					await sleep$2(500);
+					await sleep$3(500);
 				}
 			},
 			exec(member, argv) {
@@ -5071,6 +5659,8 @@ const openReplicaSetLab = async (set, backups = false) => {
 };
 /** This time is how long a run continues after its outcome, so that the record shows what came next. */
 const TAIL_MS$2 = 2e3;
+/** This function gives how long a run of a cell continues after its lift, in ms: the tail, and the watch of the cell. */
+const afterLiftOf = (cell) => TAIL_MS$2 + (cell.watch === void 0 ? 0 : valueIn(cell.watch, "s") * 1e3);
 /** This function gives the arguments of the client of a replica set cell: each member, and the stream of writes. */
 const replicaSetArgs = (cell, members) => {
 	const set = cell.replicaSet;
@@ -5094,7 +5684,7 @@ const replicaSetArgs = (cell, members) => {
 	];
 };
 /** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
-const waitFor$1 = (client, match, within) => new Promise((resolve) => {
+const waitFor$2 = (client, match, within) => new Promise((resolve) => {
 	const started = Date.now();
 	const look = () => {
 		const found = client.events.find(match);
@@ -5104,7 +5694,7 @@ const waitFor$1 = (client, match, within) => new Promise((resolve) => {
 	look();
 });
 /** This function gives the count of lost records in the latest clock line of the prober. */
-const droppedSoFar$2 = (records) => {
+const droppedSoFar$3 = (records) => {
 	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
 	return 0;
 };
@@ -5129,7 +5719,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 	await lab.forget();
 	const from = lab.kernel.length;
 	const logFrom = lab.log.length;
-	const lostBefore = droppedSoFar$2(lab.kernel);
+	const lostBefore = droppedSoFar$3(lab.kernel);
 	const client = await lab.client(replicaSetArgs(cell, lab.members));
 	const lands = cell.prediction.group?.lands ?? "new";
 	const place = cell.faultAt ?? { role: "primary" };
@@ -5139,7 +5729,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 	const undo = [];
 	try {
 		await client.until(["ready"], 6e4);
-		if (await waitFor$1(client, (event) => event.ev === "ack" && Number(event["n"]) >= 20, 3e4) === void 0) throw new Error(`the client did not write 20 times in 30 s`);
+		if (await waitFor$2(client, (event) => event.ev === "ack" && Number(event["n"]) >= 20, 3e4) === void 0) throw new Error(`the client did not write 20 times in 30 s`);
 		let healing = null;
 		switch (cell.fault) {
 			case "blackhole":
@@ -5153,7 +5743,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 				undo.push(() => lab.lift(kind, part, cuts));
 				if (cell.stall !== void 0) {
 					const t1 = fault.t1;
-					const sent = await waitFor$1(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
+					const sent = await waitFor$2(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
 					healing = lab.lift(kind, part, cuts, (sent?.t ?? t1) + valueIn(cell.stall, "s") * 1e9);
 				} else if (cell.hold !== void 0) healing = lab.lift(kind, part, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
 				break;
@@ -5188,10 +5778,10 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 		const t1 = fault.t1;
 		const old = `${lab.members[primary].ip}:${set.port}`;
 		const sentAt = (n) => client.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
-		if (await waitFor$1(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1 && (lands === "same" || typeof event["at"] === "string" && event["at"] !== old), deadlineOf(cell)) !== void 0) await sleep$2(TAIL_MS$2);
+		if (await waitFor$2(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1 && (lands === "same" || typeof event["at"] === "string" && event["at"] !== old), deadlineOf(cell)) !== void 0) await sleep$3(TAIL_MS$2);
 		if (healing !== null) {
 			await healed.catch(() => void 0);
-			await sleep$2(TAIL_MS$2);
+			await sleep$3(afterLiftOf(cell));
 		}
 	} finally {
 		for (const step of undo.reverse()) await step().catch(() => void 0);
@@ -5249,7 +5839,7 @@ const until = async (what, check, within) => {
 	const started = Date.now();
 	while (!await check().catch(() => false)) {
 		if (Date.now() - started > within) throw new Error(`${what} did not come in ${within} ms`);
-		await sleep$2(500);
+		await sleep$3(500);
 	}
 };
 /** This function gives the time that the client sent a write. Zero is no write. */
@@ -5307,7 +5897,7 @@ const backupsOf = async (lab, drill, primary, writer) => {
 	}];
 	const every = valueIn(drill.sliceEvery, "s") * 1e3;
 	for (let slice = 1; slice <= drill.slices; slice += 1) {
-		await sleep$2(Math.max(0, ended + slice * every - Date.now()));
+		await sleep$3(Math.max(0, ended + slice * every - Date.now()));
 		const out = `${VAULT}/slice-${slice}`;
 		const begun = Date.now();
 		await lab.vault([
@@ -5351,7 +5941,7 @@ const lose = async (lab, drill, primary, last) => {
 	const parts = drill.recovery === "restore" ? [primary, ...lab.set.members.filter((name) => name !== primary)] : [primary];
 	const age = valueIn(drill.age, "s");
 	const at = last === null || drill.quiet === true ? void 0 : last.t + age * 1e9;
-	if (at === void 0) await sleep$2(age * 1e3);
+	if (at === void 0) await sleep$3(age * 1e3);
 	const stamps = await Promise.all(parts.map((part) => lab.signal(part, "KILL", at)));
 	for (const part of parts) {
 		await lab.exec(part, [
@@ -5442,7 +6032,7 @@ const drillOnce = async (lab, cell, environment, index) => {
 	const tool = (await lab.vault(["mongodump", "--version"])).split("\n")[0].trim();
 	const from = lab.kernel.length;
 	const logFrom = lab.log.length;
-	const lostBefore = droppedSoFar$2(lab.kernel);
+	const lostBefore = droppedSoFar$3(lab.kernel);
 	const writer = await lab.client(replicaSetArgs(cell, lab.members));
 	const restores = drill.recovery === "restore";
 	let successor = null;
@@ -5452,7 +6042,7 @@ const drillOnce = async (lab, cell, environment, index) => {
 	const backups = [];
 	try {
 		await writer.until(["ready"], 6e4);
-		if (await waitFor$1(writer, (event) => event.ev === "ack" && Number(event["n"]) >= 20, 3e4) === void 0) throw new Error(`the client did not write 20 times in 30 s`);
+		if (await waitFor$2(writer, (event) => event.ev === "ack" && Number(event["n"]) >= 20, 3e4) === void 0) throw new Error(`the client did not write 20 times in 30 s`);
 		if (drill.quiet === true) await writer.stop();
 		if (restores) backups.push(...await backupsOf(lab, drill, primary, writer));
 		const stamp = await lose(lab, drill, primary, backups.at(-1)?.last ?? null);
@@ -5466,11 +6056,11 @@ const drillOnce = async (lab, cell, environment, index) => {
 				"--from",
 				String(second)
 			]);
-			if (await waitFor$1(successor, (event) => event.ev === "ack", deadlineOf(cell)) !== void 0) await sleep$2(TAIL_MS$2);
+			if (await waitFor$2(successor, (event) => event.ev === "ack", deadlineOf(cell)) !== void 0) await sleep$3(TAIL_MS$2);
 		} else {
 			const old = `${lab.members[primary].ip}:${set.port}`;
 			const sentAt = (n) => writer.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
-			if (await waitFor$1(writer, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > stamp.t1 && typeof event["at"] === "string" && event["at"] !== old, deadlineOf(cell)) !== void 0) await sleep$2(TAIL_MS$2);
+			if (await waitFor$2(writer, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > stamp.t1 && typeof event["at"] === "string" && event["at"] !== old, deadlineOf(cell)) !== void 0) await sleep$3(TAIL_MS$2);
 			await lab.restart(primary);
 		}
 	} finally {
@@ -5632,6 +6222,7 @@ const clusterViewOf = (cluster, rows) => {
 * This function gives the peers that a fault on a node cuts it from. `all` is each other node and the
 * client, `group` is the other nodes, and `clients` is the client. `one` is the first other node in
 * the sequence of the cluster, and `one-way` is the same node for the segments from the part only.
+* `to-clients` is the client, for the segments from the part only.
 */
 const clusterCutOf = (cluster, members, client, part, cuts) => {
 	const others = cluster.members.filter((name) => name !== part).map((name) => members[name].ip);
@@ -5642,9 +6233,13 @@ const clusterCutOf = (cluster, members, client, part, cuts) => {
 		peers: others.slice(0, 1),
 		direction: "out"
 	};
+	if (cuts === "to-clients") return {
+		peers: [client],
+		direction: "out"
+	};
 	return { peers: [...others, client] };
 };
-const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const sleep$2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /**
 * This function opens one environment of a cluster cell:
 * - one new network, with each node, the client and the admin on it.
@@ -5806,7 +6401,7 @@ const openClusterLab = async (cluster) => {
 				calm = last.healthy ? calm + 1 : 0;
 				if (last.primary !== null) known = last.primary;
 				if (calm >= 2 && last.primary !== null) return last.primary;
-				await sleep$1(1e3);
+				await sleep$2(1e3);
 			}
 			throw new Error(`the queue did not converge in ${within} ms: ${last.why}`);
 		};
@@ -5822,7 +6417,7 @@ const openClusterLab = async (cluster) => {
 			const running = (text.trim() === "" ? {} : toolJsonOf(text)).running_nodes ?? [];
 			if (cluster.members.every((name) => running.includes(nodeNameOf(name)))) break;
 			if (Date.now() - started > 12e4) throw new Error(`the cluster did not form in 120 s: ${running.join(", ")} run`);
-			await sleep$1(1e3);
+			await sleep$2(1e3);
 		}
 		const declared = lastLine(await output(commander, [
 			"node",
@@ -5942,7 +6537,7 @@ const openClusterLab = async (cluster) => {
 					"check_running"
 				]).then(() => true).catch(() => false)) {
 					if (Date.now() - begun > 12e4) throw new Error(`${member} did not start again`);
-					await sleep$1(1e3);
+					await sleep$2(1e3);
 				}
 			},
 			stepDown(leader, to) {
@@ -6044,7 +6639,7 @@ const openClusterLab = async (cluster) => {
 	}
 };
 /** This count is the number of confirmed publishes that a run waits for before its fault: the stream is steady then. */
-const STEADY = 20;
+const STEADY$1 = 20;
 /** This time is how long a run continues after its outcome, so that the record shows what came next. */
 const TAIL_MS$1 = 2e3;
 /**
@@ -6073,7 +6668,7 @@ const clusterArgs = (cell, members, leader) => [
 	...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
 ];
 /** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
-const waitFor = (client, match, within) => new Promise((resolve) => {
+const waitFor$1 = (client, match, within) => new Promise((resolve) => {
 	const started = Date.now();
 	const look = () => {
 		const found = client.events.find(match);
@@ -6083,7 +6678,7 @@ const waitFor = (client, match, within) => new Promise((resolve) => {
 	look();
 });
 /** This function gives the count of lost records in the latest clock line of the prober. */
-const droppedSoFar$1 = (records) => {
+const droppedSoFar$2 = (records) => {
 	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
 	return 0;
 };
@@ -6113,7 +6708,7 @@ const stampOf = async (lab, client, logFrom, marks, wall) => {
 			},
 			line
 		};
-		await sleep$1(100);
+		await sleep$2(100);
 	}
 	return {
 		stamp: monotonicOf(wall, client.events),
@@ -6150,7 +6745,7 @@ const clusterOnce = async (lab, cell, environment, index) => {
 	await lab.forget();
 	const from = lab.kernel.length;
 	const logFrom = lab.log.length;
-	const lostBefore = droppedSoFar$1(lab.kernel);
+	const lostBefore = droppedSoFar$2(lab.kernel);
 	const client = await lab.client(clusterArgs(cell, lab.members, leader));
 	const place = cell.faultAt ?? { role: "primary" };
 	let fault = null;
@@ -6161,7 +6756,7 @@ const clusterOnce = async (lab, cell, environment, index) => {
 	const undo = [];
 	try {
 		await client.until(["ready"], 6e4);
-		if (await waitFor(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY, 3e4) === void 0) throw new Error(`the client did not publish ${STEADY} times in 30 s`);
+		if (await waitFor$1(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$1, 3e4) === void 0) throw new Error(`the client did not publish ${STEADY$1} times in 30 s`);
 		let healing = null;
 		switch (cell.fault) {
 			case "blackhole":
@@ -6215,10 +6810,10 @@ const clusterOnce = async (lab, cell, environment, index) => {
 		healed.catch(() => void 0);
 		const t1 = fault.t1;
 		const sentAt = (n) => client.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
-		if (await waitFor(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1, deadlineOf(cell)) !== void 0) await sleep$1(TAIL_MS$1);
+		if (await waitFor$1(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1, deadlineOf(cell)) !== void 0) await sleep$2(TAIL_MS$1);
 		if (healing !== null) {
 			await healed.catch(() => void 0);
-			await sleep$1(TAIL_MS$1);
+			await sleep$2(TAIL_MS$1);
 		}
 	} finally {
 		for (const step of undo.reverse()) await step().catch(() => void 0);
@@ -6250,42 +6845,540 @@ const clusterOnce = async (lab, cell, environment, index) => {
 	};
 };
 //#endregion
-//#region src/run.ts
+//#region src/kafka.ts
 /**
-* This function measures and judges the raw records of a cell, with the code of the harness at
-* this time. `runCell` uses it during a cell. A holder of the raw records of a cell can use it
-* again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.
+* This module is the lab of a group of Kafka brokers in KRaft mode with one controller. It opens one
+* environment for a Kafka cell, for the runs of the cell. Each node is a container from the stock
+* image, in its own network namespace, and an injector sidecar shares its network and process
+* namespaces. The image starts by its own command, and each setting comes from the environment.
+*
+* Each run makes one new topic, thus each run starts with an empty log and its client counts from 1.
+* The client is the producer of kafka-clients. The commander holds the admin, on the network but
+* not in the namespace that the prober watches. The record of a run is the lines of the client and
+* the whole partition, read to its high watermark after the group converged.
 */
-const judge = (cell, runs) => {
-	const measured = runs.filter((run) => run.warmup !== true).map((run) => measure(cell, run));
+/** This path is the class path of the Kafka image: the JVM expands the star itself. */
+const CLASS_PATH = "/opt/harness/kafka/classes:/opt/harness/kafka/lib/*";
+/** This option is the heap of the client and of the admin. It has no source: the two hold a producer or a read of one partition. */
+const CLIENT_HEAP = "-Xmx256m";
+/** This function gives the node id of a name. A broker has its place in the list from 1, and the controller has the id after them. */
+const nodeIdOf = (kafka, name) => {
+	const at = kafka.brokers.indexOf(name);
+	if (at !== -1) return at + 1;
+	if (name === kafka.controller) return kafka.brokers.length + 1;
+	throw new Error(`the Kafka group has no node ${name}`);
+};
+/**
+* This function gives the variable of a server property for the image. The wrapper of the image
+* reverses this map (KafkaDockerWrapper.scala:188-203). After the prefix `KAFKA_`, `___` is a
+* hyphen, `__` is an underscore and `_` is a dot.
+*/
+const envOf = (property) => `KAFKA_${property.replace(/_/g, "__").replace(/-/g, "___").replace(/\./g, "_").toUpperCase()}`;
+/** This function gives the variables of each node: the listener names, the voters, the log and the settings of the group. */
+const commonEnv = (kafka) => ({
+	CLUSTER_ID: kafka.clusterId,
+	KAFKA_HEAP_OPTS: kafka.heap,
+	KAFKA_CONTROLLER_QUORUM_VOTERS: `${nodeIdOf(kafka, kafka.controller)}@${kafka.controller}:${kafka.controllerPort}`,
+	KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER",
+	KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT",
+	KAFKA_LOG_DIRS: "/var/lib/kafka/data",
+	...Object.fromEntries(Object.entries(kafka.settings).map(([property, value]) => [envOf(property), value]))
+});
+/** This function gives the environment of a broker. */
+const brokerEnv = (kafka, name) => ({
+	...commonEnv(kafka),
+	KAFKA_PROCESS_ROLES: "broker",
+	KAFKA_NODE_ID: String(nodeIdOf(kafka, name)),
+	KAFKA_LISTENERS: `PLAINTEXT://0.0.0.0:${kafka.port}`,
+	KAFKA_ADVERTISED_LISTENERS: `PLAINTEXT://${name}:${kafka.port}`,
+	KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
+});
+/** This function gives the environment of the controller. The start script of the image stops on an advertised listener here (configure:43-48). */
+const controllerEnv = (kafka) => ({
+	...commonEnv(kafka),
+	KAFKA_PROCESS_ROLES: "controller",
+	KAFKA_NODE_ID: String(nodeIdOf(kafka, kafka.controller)),
+	KAFKA_LISTENERS: `CONTROLLER://0.0.0.0:${kafka.controllerPort}`
+});
+/**
+* This function judges a view of a partition: its leader, and if the group is healthy. A healthy
+* group has each broker not fenced, one leader, and each broker in the in-sync replicas.
+*/
+const kafkaViewOf = (kafka, view) => {
+	const primary = view.leader === void 0 || view.leader === null ? null : kafka.brokers[view.leader - 1] ?? null;
+	const all = kafka.brokers.length;
+	if (view.brokers.length !== all) return {
+		primary,
+		healthy: false,
+		why: `${view.brokers.length} brokers are not fenced`
+	};
+	if (primary === null) return {
+		primary,
+		healthy: false,
+		why: "the partition has no leader"
+	};
+	if (view.replicas?.length !== all) return {
+		primary,
+		healthy: false,
+		why: `the partition has ${view.replicas?.length ?? 0} replicas`
+	};
+	if (view.isr?.length !== all) return {
+		primary,
+		healthy: false,
+		why: `the partition has ${view.isr?.length ?? 0} in-sync replicas`
+	};
 	return {
-		measured,
-		verdict: verdictOf(cell.prediction, measured, noiseOf(cell))
+		primary,
+		healthy: true,
+		why: "healthy"
 	};
 };
-/** This function gives the count of lost records in the latest clock line of the prober, since it began. */
-const droppedSoFar = (records) => {
-	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
-	return 0;
-};
-/** This function gives the argv of the client for a cell, with a deadline far after all that it waits for. */
-const probeArgs = (cell, lab) => {
-	const { prediction, probe } = cell;
-	const longest = (prediction.detects === true && prediction.driver !== void 0 ? longestOf(cell) : prediction.detects ? latestEndOf(prediction) : valueIn(prediction.window, "s")) + (cell.idleBefore === void 0 ? 0 : valueIn(cell.idleBefore, "s"));
-	const args = [
-		lab.targetIp,
-		String(lab.targetPort),
-		"--mode",
-		probe.mode
-	];
-	if (probe.ws !== void 0) {
-		const { ws } = probe;
-		const waits = (prediction.detects !== false ? prediction.reconnects?.waits ?? [] : []).reduce((sum, wait) => sum + valueIn(wait.high, "s"), 0);
-		args.push("--websocket", ws.socket);
-		args.push("--heartbeat", ws.heartbeat === null ? "off" : "on");
-		if (ws.heartbeat !== null) args.push("--heartbeat-interval-ms", String(valueIn(ws.heartbeat.interval, "ms")), "--heartbeat-timeout-ms", String(valueIn(ws.heartbeat.timeout, "ms")), "--heartbeat-misses", String(ws.heartbeat.misses));
-		args.push("--reconnect", ws.reconnect === null ? "off" : "on");
-		if (ws.reconnect !== null) args.push("--reconnect-initial-ms", String(valueIn(ws.reconnect.initialDelay, "ms")), "--reconnect-multiplier", String(ws.reconnect.multiplier), "--reconnect-max-ms", String(valueIn(ws.reconnect.maxDelay, "ms")), "--reconnect-attempts", String(ws.reconnect.attempts), "--reconnect-jitter", String(ws.reconnect.jitter));
+/**
+* This function gives the peers that a fault on a broker cuts it from. Only `group` has a law: the
+* other brokers and the controller, and the client keeps its route to the broker.
+*/
+const kafkaCutOf = (kafka, members, controller, part, cuts) => {
+	if (cuts !== "group") throw new Error(`a Kafka cell has no cut ${cuts}`);
+	return { peers: [...kafka.brokers.filter((name) => name !== part).map((name) => members[name].ip), controller.ip] };
+};
+/** This function gives the addresses that the client and the admin bootstrap from. */
+const bootstrapOf = (kafka) => kafka.brokers.map((name) => `${name}:${kafka.port}`).join(",");
+/** This function gives the name of the topic of a run. A warm-up has the index -1. */
+const topicOf = (environment, index) => `cells-e${environment}-r${index + 1}`;
+const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/**
+* This function opens one environment of a Kafka cell:
+* - one new network, with each node, the client and the commander on it.
+* - the controller first, then each broker: the stock image with its own command. The brokers
+*   register with the controller, thus they need it first.
+* - an injector sidecar for each node, in its network and process namespaces.
+* - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
+* - the prober, which watches the namespace of the client.
+*/
+const openKafkaLab = async (kafka) => {
+	const image = await harnessImage();
+	const clientImage = await kafkaClientImage();
+	const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();
+	const runtime = await getContainerRuntimeClient();
+	const opened = [];
+	let network;
+	const close = async () => {
+		for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
+		await network?.stop().catch(() => void 0);
+	};
+	const ipOf = async (container) => {
+		const ip = (await runtime.container.inspect(runtime.container.getById(container.getId()))).NetworkSettings.Networks[network.getName()]?.IPAddress;
+		if (ip === void 0 || ip === "") throw new Error(`${container.getName()} has no address on ${network.getName()}`);
+		return ip;
+	};
+	try {
+		network = await new Network().start();
+		const started = network;
+		const boxes = /* @__PURE__ */ new Map();
+		const injectors = /* @__PURE__ */ new Map();
+		const parts = {};
+		const raw = {};
+		const openNode = async (name, environment) => {
+			let partial = "";
+			const box = await new GenericContainer(kafka.image).withNetwork(started).withNetworkAliases(name).withHostname(name).withEnvironment(environment).withWaitStrategy(Wait.forLogMessage(kafka.ready)).withStartupTimeout(18e4).withLogConsumer((stream) => stream.on("data", (chunk) => {
+				partial += chunk.toString();
+				const split = partial.split("\n");
+				partial = split.pop() ?? "";
+				for (const text of split) {
+					const tail = raw[name] ??= [];
+					tail.push(text);
+					if (tail.length > 400) tail.shift();
+				}
+			})).start();
+			opened.push(box);
+			boxes.set(name, box);
+			const injector = await withPid(new GenericContainer(image).withNetworkMode(`container:${box.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW"), `container:${box.getId()}`).start();
+			opened.push(injector);
+			injectors.set(name, injector);
+			parts[name] = {
+				ip: await ipOf(box),
+				netns: await netnsOf(injector)
+			};
+		};
+		await openNode(kafka.controller, controllerEnv(kafka));
+		for (const name of kafka.brokers) await openNode(name, brokerEnv(kafka, name));
+		const members = Object.fromEntries(kafka.brokers.map((name) => [name, parts[name]]));
+		const controller = parts[kafka.controller];
+		const holder = await new GenericContainer(image).withNetwork(network).start();
+		opened.push(holder);
+		const admin = await new GenericContainer(image).withNetworkMode(`container:${holder.getId()}`).withPrivilegedMode().start();
+		opened.push(admin);
+		const clientIp = await ipOf(holder);
+		const commander = await new GenericContainer(clientImage).withNetwork(network).withCommand([
+			"sh",
+			"-c",
+			"while :; do sleep 3600; done"
+		]).start();
+		opened.push(commander);
+		const clientNetns = await netnsOf(admin);
+		const records = lines();
+		const prober = await new GenericContainer(image).withPrivilegedMode().withNetworkMode("none").withCommand(["/opt/harness/prober", String(clientNetns)]).withLogConsumer(records.consume).start();
+		opened.push(prober);
+		const begun = await records.until(["begin", "fail"], 3e4, "prober");
+		if (begun.ev === "fail") throw new Error(`the prober failed: ${String(begun["why"])}`);
+		const box = (name) => {
+			const found = boxes.get(name);
+			if (found === void 0) throw new Error(`the group has no node ${name}`);
+			return found;
+		};
+		const injectorOf = (name) => {
+			const found = injectors.get(name);
+			if (found === void 0) throw new Error(`the group has no node ${name}`);
+			return found;
+		};
+		const lastLine = (text) => text.split("\n").filter((line) => line.trim() !== "").at(-1) ?? "";
+		const stamped = async (where, argv) => {
+			const stamp = JSON.parse(lastLine(await output(where, argv)));
+			return {
+				t0: stamp.t0,
+				t1: stamp.t1
+			};
+		};
+		const after = (at) => at === void 0 ? [] : ["--at", String(Math.round(at))];
+		const bootstrap = bootstrapOf(kafka);
+		const tool = async (...argv) => lastLine(await output(commander, [
+			"java",
+			CLIENT_HEAP,
+			"-cp",
+			CLASS_PATH,
+			"AdminKafka",
+			bootstrap,
+			...argv
+		]));
+		const view = async (topic) => JSON.parse(await tool("view", ...topic === void 0 ? [] : [topic]));
+		return {
+			kafka,
+			clientIp,
+			members,
+			controller,
+			netns: { client: clientNetns },
+			kernel: records.seen,
+			raw,
+			async sysctl(values) {
+				const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
+				if (settings.length > 0) await output(admin, [
+					"sysctl",
+					"-q",
+					"-w",
+					...settings
+				]);
+				const read = {};
+				for (const key of Object.keys(values)) read[key] = await output(admin, [
+					"sysctl",
+					"-n",
+					key
+				]);
+				return read;
+			},
+			async brokersUp(within = CONVERGE_MS) {
+				const begin = Date.now();
+				let last = "no view yet";
+				while (Date.now() - begin < within) {
+					const read = await view().catch((error) => error.message.slice(0, 200));
+					if (typeof read !== "string" && read.brokers.length === kafka.brokers.length) return;
+					last = typeof read === "string" ? read : `${read.brokers.length} brokers are not fenced`;
+					await sleep$1(1e3);
+				}
+				throw new Error(`the brokers did not come up in ${within} ms: ${last}`);
+			},
+			async create(topic) {
+				const ids = kafka.brokers.map((name) => nodeIdOf(kafka, name)).join(",");
+				const reply = await tool("create", topic, String(kafka.minInsync), ids);
+				if (!reply.includes("\"ok\":true")) throw new Error(`the create of the topic failed: ${reply}`);
+			},
+			async converge(topic, within = CONVERGE_MS) {
+				const begin = Date.now();
+				let calm = 0;
+				let last = {
+					primary: null,
+					healthy: false,
+					why: "no view yet"
+				};
+				while (Date.now() - begin < within) {
+					last = await view(topic).then((read) => kafkaViewOf(kafka, read), (error) => ({
+						primary: null,
+						healthy: false,
+						why: error.message.slice(0, 200)
+					}));
+					calm = last.healthy ? calm + 1 : 0;
+					if (calm >= 2 && last.primary !== null) return last.primary;
+					await sleep$1(1e3);
+				}
+				throw new Error(`the partition did not converge in ${within} ms: ${last.why}`);
+			},
+			async holds(topic) {
+				const reply = await tool("holds", topic);
+				const { held } = JSON.parse(reply);
+				if (held === void 0) throw new Error(`the read of the partition failed: ${reply}`);
+				return held.map(Number).filter((n) => Number.isInteger(n) && n > 0);
+			},
+			async client(args) {
+				const events = lines();
+				const container = await new GenericContainer(clientImage).withNetworkMode(`container:${holder.getId()}`).withCommand([
+					"java",
+					CLIENT_HEAP,
+					"-cp",
+					CLASS_PATH,
+					"ClientKafka",
+					bootstrap,
+					...args
+				]).withLogConsumer(events.consume).start();
+				return {
+					events: events.seen,
+					until: (kinds, within) => events.until(kinds, within, "client"),
+					stop: () => container.stop({ timeout: 2 }).then(() => void 0).catch(() => void 0)
+				};
+			},
+			fault(kind, part, cuts, at) {
+				const cut = kafkaCutOf(kafka, members, controller, part, cuts);
+				return stamped(injectorOf(part), [
+					"/opt/harness/inject",
+					kind,
+					cut.peers.join(","),
+					...after(at)
+				]);
+			},
+			lift(kind, part, cuts, at) {
+				const cut = kafkaCutOf(kafka, members, controller, part, cuts);
+				return stamped(injectorOf(part), [
+					"/opt/harness/inject",
+					kind === "rst" ? "clear-rst" : "clear",
+					cut.peers.join(","),
+					...after(at)
+				]);
+			},
+			async forget() {
+				await output(admin, [
+					"ip",
+					"tcp_metrics",
+					"flush",
+					"all"
+				]);
+			},
+			async settle() {
+				const from = records.seen.length;
+				const clock = await new Promise((resolve) => {
+					const begin = Date.now();
+					const look = () => {
+						const line = records.seen.slice(from).find((record) => record.ev === "clock");
+						if (line !== void 0) resolve(line);
+						else if (Date.now() - begin > 5e3) resolve(null);
+						else setTimeout(look, 50);
+					};
+					look();
+				});
+				return clock === null ? null : Number(clock["lost"] ?? 0);
+			},
+			async manifest(sysctls) {
+				const config = await output(admin, ["zcat", "/proc/config.gz"]).catch(() => "");
+				const inspect = async (container) => runtime.container.inspect(runtime.container.getById(container.getId()));
+				const settings = {
+					settings: Object.entries(kafka.settings).map(([property, value]) => `${property}=${value}`).join("; "),
+					heap: kafka.heap,
+					"min.insync.replicas": String(kafka.minInsync)
+				};
+				const list = async (names) => {
+					const out = [];
+					for (const name of names) {
+						const version = lastLine(await output(box(name), ["/opt/kafka/bin/kafka-topics.sh", "--version"]).catch(() => ""));
+						out.push({
+							name,
+							ip: parts[name].ip,
+							id: (await inspect(box(name))).Image,
+							settings: {
+								version,
+								...settings
+							}
+						});
+					}
+					return out;
+				};
+				const brokers = await list(kafka.brokers);
+				return {
+					kernel: await output(holder, ["uname", "-r"]),
+					architecture: await output(holder, ["uname", "-m"]),
+					hz: /^CONFIG_HZ=(\d+)$/m.exec(config)?.[1] ?? null,
+					boot_id: await output(holder, ["cat", "/proc/sys/kernel/random/boot_id"]),
+					docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
+					target: {
+						image: kafka.image,
+						id: brokers[0].id,
+						settings: brokers[0].settings,
+						sysctls: {}
+					},
+					harness: {
+						image,
+						id: (await inspect(holder)).Image,
+						packages: (await output(holder, [
+							"apk",
+							"info",
+							"-v"
+						])).split("\n").sort()
+					},
+					client: {
+						kind: "image",
+						image: clientImage
+					},
+					netns: {
+						client: clientNetns,
+						target: members[kafka.brokers[0]].netns
+					},
+					sysctls,
+					group: {
+						members: brokers,
+						sentinels: await list([kafka.controller]),
+						observer: { image: clientImage }
+					}
+				};
+			},
+			close
+		};
+	} catch (error) {
+		await close();
+		throw error;
+	}
+};
+/** This count is the number of acknowledged writes that a run waits for before its fault: the stream is steady then. */
+const STEADY = 20;
+/**
+* This time is how long a run continues after its lift. The record then shows the step-down of the
+* leader and the writes after it. It has no source. It is above the largest step-down of the
+* law, which is approximately 2 s after the lift.
+*/
+const KAFKA_TAIL_MS = 1e4;
+/** This function gives the arguments of the client of a Kafka cell: the topic, the writers, the stream of writes and the settings. */
+const kafkaArgs = (cell, topic) => [
+	"--topic",
+	topic,
+	"--writers",
+	String(cell.kafka.writers),
+	"--interval-ms",
+	String(cell.probe.interval === void 0 ? 50 : valueIn(cell.probe.interval, "ms")),
+	"--max-s",
+	String(Math.ceil(deadlineOf(cell) / 1e3 + (cell.hold === void 0 ? 0 : valueIn(cell.hold, "s")) + 120)),
+	...Object.entries(cell.driverOptions ?? {}).flatMap(([name, value]) => ["--set", `${name}=${String(value)}`])
+];
+/** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
+const waitFor = (client, match, within) => new Promise((resolve) => {
+	const started = Date.now();
+	const look = () => {
+		const found = client.events.find(match);
+		if (found !== void 0 || Date.now() - started > within) resolve(found);
+		else setTimeout(look, 20);
+	};
+	look();
+});
+/** This function gives the count of lost records in the latest clock line of the prober. */
+const droppedSoFar$1 = (records) => {
+	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
+	return 0;
+};
+/**
+* This function does one run of a Kafka cell:
+* - Each broker is up, the topic of the run exists, and its partition is whole. The client starts,
+*   and its stream of writes gets steady.
+* - The fault cuts the leader from the other brokers and the controller, and it lifts at its instant
+*   in the VM. The client keeps its route to the leader, thus only the group knows of the cut.
+* - The run waits for the lift and for the tail after it.
+* - Each fault goes, the client stops, and the partition converges again. The admin then reads
+*   the whole partition, thus the record tells which writes the group kept.
+*/
+const kafkaOnce = async (lab, cell, environment, index) => {
+	if (cell.fault !== "partition") throw new Error(`a Kafka cell has no fault ${cell.fault}`);
+	if (cell.hold === void 0) throw new Error("a Kafka cell needs a hold: its law reads it");
+	const place = cell.faultAt ?? { role: "primary" };
+	if (place.role !== "primary") throw new Error(`a Kafka cell has no fault on the ${place.role}`);
+	const cuts = place.cuts ?? "group";
+	const topic = topicOf(environment, index);
+	await lab.brokersUp();
+	await lab.create(topic);
+	const leader = await lab.converge(topic);
+	await lab.forget();
+	const from = lab.kernel.length;
+	const lostBefore = droppedSoFar$1(lab.kernel);
+	const client = await lab.client(kafkaArgs(cell, topic));
+	let fault = null;
+	let heal = null;
+	const undo = [];
+	try {
+		await client.until(["ready"], 6e4);
+		if (await waitFor(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY, 3e4) === void 0) throw new Error(`the client did not write ${STEADY} times in 30 s`);
+		fault = await lab.fault("blackhole", leader, cuts);
+		undo.push(() => lab.lift("blackhole", leader, cuts));
+		heal = await lab.lift("blackhole", leader, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
+		undo.length = 0;
+		await sleep$1(KAFKA_TAIL_MS);
+	} finally {
+		for (const step of undo.reverse()) await step().catch(() => void 0);
+		await client.stop();
+	}
+	const settled = await lab.settle();
+	const lost = settled === null ? null : settled - lostBefore;
+	const after = await lab.converge(topic).catch(() => null);
+	const holds = after === null ? null : await lab.holds(topic);
+	return {
+		environment,
+		index,
+		events: [...client.events],
+		fault,
+		...heal === null ? {} : { heal },
+		kernel: lab.kernel.slice(from),
+		netns: lab.netns.client,
+		targetNetns: lab.members[leader].netns,
+		lost,
+		group: {
+			members: lab.members,
+			sentinels: { [lab.kafka.controller]: lab.controller },
+			primary: leader,
+			at: leader,
+			timeline: [],
+			after,
+			holds
+		}
+	};
+};
+//#endregion
+//#region src/run.ts
+/**
+* This function measures and judges the raw records of a cell, with the code of the harness at
+* this time. `runCell` uses it during a cell. A holder of the raw records of a cell can use it
+* again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.
+*/
+const judge = (cell, runs) => {
+	const measured = runs.filter((run) => run.warmup !== true).map((run) => measure(cell, run));
+	return {
+		measured,
+		verdict: verdictOf(cell.prediction, measured, noiseOf(cell))
+	};
+};
+/** This function gives the count of lost records in the latest clock line of the prober, since it began. */
+const droppedSoFar = (records) => {
+	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
+	return 0;
+};
+/** This function gives the argv of the client for a cell, with a deadline far after all that it waits for. */
+const probeArgs = (cell, lab) => {
+	const { prediction, probe } = cell;
+	const longest = (prediction.detects === true && prediction.driver !== void 0 ? longestOf(cell) : prediction.detects ? latestEndOf(prediction) : valueIn(prediction.window, "s")) + (cell.idleBefore === void 0 ? 0 : valueIn(cell.idleBefore, "s"));
+	const args = [
+		lab.targetIp,
+		String(lab.targetPort),
+		"--mode",
+		probe.mode
+	];
+	if (probe.ws !== void 0) {
+		const { ws } = probe;
+		const waits = (prediction.detects !== false ? prediction.reconnects?.waits ?? [] : []).reduce((sum, wait) => sum + valueIn(wait.high, "s"), 0);
+		args.push("--websocket", ws.socket);
+		args.push("--heartbeat", ws.heartbeat === null ? "off" : "on");
+		if (ws.heartbeat !== null) args.push("--heartbeat-interval-ms", String(valueIn(ws.heartbeat.interval, "ms")), "--heartbeat-timeout-ms", String(valueIn(ws.heartbeat.timeout, "ms")), "--heartbeat-misses", String(ws.heartbeat.misses));
+		args.push("--reconnect", ws.reconnect === null ? "off" : "on");
+		if (ws.reconnect !== null) args.push("--reconnect-initial-ms", String(valueIn(ws.reconnect.initialDelay, "ms")), "--reconnect-multiplier", String(ws.reconnect.multiplier), "--reconnect-max-ms", String(valueIn(ws.reconnect.maxDelay, "ms")), "--reconnect-attempts", String(ws.reconnect.attempts), "--reconnect-jitter", String(ws.reconnect.jitter));
 		args.push("--request-timeout-ms", String(valueIn(ws.requestTimeout, "ms")));
 		args.push("--connection-timeout-ms", String(valueIn(ws.connectionTimeout, "ms")));
 		if (ws.hold !== void 0) args.push("--hold-ms", String(valueIn(ws.hold, "ms")));
@@ -6319,6 +7412,7 @@ const probeArgs = (cell, lab) => {
 	if (probe.driver?.name === "pg" || probe.driver?.name === "hikari" || probe.driver?.name === "amqp") {
 		if (probe.driver.name !== "amqp" && probe.driver.query !== void 0) args.push("--query", probe.driver.query);
 		if (probe.driver.name !== "amqp" && probe.driver.inflight !== void 0) args.push("--inflight-query", probe.driver.inflight);
+		if (probe.driver.name === "hikari" && probe.driver.burst !== void 0) args.push("--burst", String(probe.driver.burst.count), "--burst-gap-ms", String(Math.round(valueIn(probe.driver.burst.gap, "ms"))));
 		if (cell.before === true) args.push("--before");
 		if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
 		args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
@@ -6480,6 +7574,19 @@ const dials = (probe, count, within) => new Promise((resolve) => {
 	};
 	look();
 });
+/** This time is the most that a run waits for the lines of the target after the end of its client. */
+const LINES_MS = 5e3;
+/**
+* This function waits for the lines of a run's counts. The lines of the target come on their own
+* stream, thus they can come after the last line of the client. A line above a count comes in the
+* pause after the wait, thus it is in the run that wrote it.
+*/
+const linesCounted = async (lab, from, counts) => {
+	const started = Date.now();
+	const short = () => counts.some((each) => countOf(lab.server.slice(from), each.match) < each.count);
+	while (short() && Date.now() - started < LINES_MS) await sleep(20);
+	await sleep(250);
+};
 /** This function gives the netem specification of a cell: its latency, then the netem of its fault when it has one. */
 const netemSpec = (cell, fault) => {
 	const parts = [...cell.latency === void 0 ? [] : [`delay ${valueIn(cell.latency, "ms")}ms`], ...fault === void 0 ? [] : [fault]];
@@ -6680,6 +7787,7 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					const gap = cell.probe.http?.gap === void 0 ? 0 : valueIn(cell.probe.http.gap, "s");
 					const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
 					await probe.until(HTTP_DONE, (gap + window) * 2e3 + 1e4).catch(() => void 0);
+					if (prediction.detects === null && prediction.counts !== void 0) await linesCounted(lab, serverFrom, prediction.counts);
 				} else if (cell.fault === "stall") {
 					if (cell.stall === void 0) throw new Error(`${cell.name} says no stall's length`);
 					const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
@@ -6794,6 +7902,8 @@ const inEnvironment = async (cell, environment, count, say) => {
 	if (cell.group !== void 0) return inGroupEnvironment(cell, environment, count, say);
 	if (cell.replicaSet !== void 0) return inReplicaSetEnvironment(cell, environment, count, say);
 	if (cell.cluster !== void 0) return inClusterEnvironment(cell, environment, count, say);
+	if (cell.kafka !== void 0) return inKafkaEnvironment(cell, environment, count, say);
+	if (cell.pair !== void 0) return inPairEnvironment(cell, environment, count, say);
 	const lab = await openLab(cell.target, cell.client ?? cell.probe.driver?.name ?? "tcp", cell.topology);
 	try {
 		const settings = await lab.configure(cell.targetSettings ?? {});
@@ -6891,6 +8001,36 @@ const inReplicaSetEnvironment = async (cell, environment, count, say) => {
 	}
 };
 /**
+* This function does the part of the runs of a pair cell in one environment. The warm-up is a dry
+* run. The run promotes the standby, thus a pair cell has an environment for each run.
+*/
+const inPairEnvironment = async (cell, environment, count, say) => {
+	const lab = await openLab(cell.target, cell.client ?? cell.probe.driver?.name ?? "tcp", cell.topology);
+	try {
+		await lab.warm();
+		const manifest = await lab.manifest(await lab.sysctl(cell.sysctls));
+		const standby = await openStandby(lab, cell);
+		const warmup = {
+			...await pairOnce(lab, standby, cell, environment, -1, true),
+			warmup: true
+		};
+		say(line(cell, warmup, measure(cell, warmup)));
+		const runs = [];
+		for (let index = 0; index < count; index += 1) {
+			const run = await pairOnce(lab, standby, cell, environment, index);
+			runs.push(run);
+			say(line(cell, run, measure(cell, run)));
+		}
+		return {
+			manifest,
+			runs,
+			warmup
+		};
+	} finally {
+		await lab.close();
+	}
+};
+/**
 * This function does the part of the runs of a cluster cell in one environment. The warm-up is a
 * whole run with its fault, as for a replica set. The queue converges before each run.
 */
@@ -6919,6 +8059,34 @@ const inClusterEnvironment = async (cell, environment, count, say) => {
 	}
 };
 /**
+* This function does the part of the runs of a Kafka cell in one environment. The warm-up is a whole
+* run with its fault, as for a cluster. Each run makes its own topic.
+*/
+const inKafkaEnvironment = async (cell, environment, count, say) => {
+	const lab = await openKafkaLab(cell.kafka);
+	try {
+		const manifest = await lab.manifest(await lab.sysctl(cell.sysctls));
+		const warmup = {
+			...await kafkaOnce(lab, cell, environment, -1),
+			warmup: true
+		};
+		say(line(cell, warmup, measure(cell, warmup)));
+		const runs = [];
+		for (let index = 0; index < count; index += 1) {
+			const run = await kafkaOnce(lab, cell, environment, index);
+			runs.push(run);
+			say(line(cell, run, measure(cell, run)));
+		}
+		return {
+			manifest,
+			runs,
+			warmup
+		};
+	} finally {
+		await lab.close();
+	}
+};
+/**
 * This function does the runs of a cell. They go across new environments, and each environment records
 * its manifest. Then the verdict judges the runs. A median across the edge of the band extends
 * the cell from 11 runs to 22, and then to 44, each extension in new environments. With a
@@ -6972,6 +8140,6 @@ const runCell = async (cell, say = () => {}) => {
 	}
 };
 //#endregion
-export { AWS_FROM, CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, DAEMONS, DROP_HANDLING, FABRIC_CLIENT, FABRIC_DEVICE, FABRIC_TARGET, FRR_IMAGE, GIVES_AWAY, HASH_SCRIPT, INJECT, KEPT, LINGER_MS, LISTING, LOCK_NAMES, LOSS_MARGIN, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, ROUTERS, SEED_DOCUMENT, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, basesOf, buildOf, buildRecordOf, callOf, cargoVersions, clockOffsetOf, closeOrExit, cloudOf, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, commitProblems, comparable, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, drillOnce, drillSpan, dropFinding, dropOf, endpointAfter, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, frrConf, goHttpImage, goVersions, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, harnessRecord, harnessVersion, hashOf, highestHeld, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isHopPlace, jarVersions, jitterOf, jitterOfLines, judge, lagsOf, laterMove, latestEndOf, leafLacks, leftAfter, locksOf, logNameOf, manifestProblems, matches, measure, measureDriver, measureGroup, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicAt, monotonicOf, nodeClientImage, nodeCommand, nodeHttpImage, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, npmVersions, onMonotonic, openClusterLab, openFabric, openGroupLab, openLab, openReplicaSetLab, pairingOf, pairingsOf, peersOf, pgClientImage, pingsOf, plainOf, planOf, programLines, programOf, quantileOf, readCpu, readHost, readOplog, reconnectFinding, reconnectOf, recoveryFindings, recoveryOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routeLine, routerImage, routesOf, runCell, seedScript, sentinelConfig, sessionsOf, shippedClientRecord, shippedSource, sliceQuery, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, standaloneCommand, statusViewOf, stealShare, stopScript, subjectClient, subjectProblems, subscriptionsOf, substrateHere, substrateKey, substrateOf, substrateOn, targetImageOf, targetSourceOf, templated, toolJsonOf, topologyOf, toxicBody, unlockedManifests, unpinnedBases, unpoolable, verdictOf, versionsOf, viewOf, writesOf };
+export { AWS_FROM, CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, DAEMONS, DEBUG, DIRTY, DROP_HANDLING, FABRIC_CLIENT, FABRIC_DEVICE, FABRIC_TARGET, FRR_IMAGE, GIVES_AWAY, HASH_SCRIPT, INJECT, INSERT, KAFKA_TAIL_MS, KEPT, LINGER_MS, LISTING, LOCK_NAMES, LOSS_MARGIN, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, RESTART, ROUTERS, SEED_DOCUMENT, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STANDBY_READY, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, basesOf, bootstrapOf, brokerEnv, buildOf, buildRecordOf, callOf, cancelsOf, cargoVersions, clockOffsetOf, closeOrExit, cloudOf, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, commitProblems, comparable, controllerEnv, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, drillOnce, drillSpan, dropFinding, dropOf, endpointAfter, envOf, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, frrConf, goHttpImage, goVersions, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, harnessRecord, harnessVersion, hashOf, highestHeld, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isHopPlace, jarVersions, jitterOf, jitterOfLines, judge, kafkaArgs, kafkaClientImage, kafkaCutOf, kafkaOnce, kafkaViewOf, lagsOf, laterMove, latestEndOf, leafLacks, leftAfter, locksOf, logNameOf, manifestProblems, matches, measure, measureDriver, measureGroup, measurePorts, measureQueue, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicAt, monotonicOf, nodeClientImage, nodeCommand, nodeHttpImage, nodeIdOf, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, npmVersions, numbersOf, onMonotonic, openClusterLab, openFabric, openGroupLab, openKafkaLab, openLab, openReplicaSetLab, openStandby, pairArgs, pairOnce, pairingOf, pairingsOf, peersOf, pgClientImage, pingsOf, plainOf, planOf, programLines, programOf, quantileOf, readCpu, readHost, readOplog, reconnectFinding, reconnectOf, recordProblems, recoveryFindings, recoveryOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routeLine, routerImage, routesOf, runCell, seedScript, sentinelConfig, sessionsOf, shippedClientRecord, shippedSource, sliceQuery, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, standaloneCommand, standbyCommand, statusViewOf, stealShare, stopScript, subjectClient, subjectProblems, subscriptionsOf, substrateHere, substrateKey, substrateOf, substrateOn, targetImageOf, targetSourceOf, templated, toolJsonOf, topicOf, topologyOf, toxicBody, unlockedManifests, unpinnedBases, unpoolable, verdictOf, versionsOf, viewOf, writesOf };
 
 //# sourceMappingURL=index.js.map