@mark1russell7/harness 0.14.2 → 0.14.4

package/build/index.js

@@ -1,10 +1,12 @@
-import { C as targetImageOf, E as unpinnedBases, S as subjectProblems, T as unlockedManifests, _ as pgClientImage, a as SHIPPED_NAMES, b as routerImage, c as grpcClientImage, d as imageOf, f as imageProblems, g as nodeHttpImage, h as nodeClientImage, i as SHIPPED_CLIENTS, l as harnessImage, m as mongoClientImage, n as PINNED, o as amqpClientImage, p as inContext, r as REPOSITORY, s as goHttpImage, t as CLIENT_NAMES, u as hikariClientImage, v as planOf, w as targetSourceOf, x as subjectClient, y as programOf } from "./image-Lg8LL6C3.js";
+import { A as unlockedManifests, C as programOf, D as subjectProblems, E as subjectClient, F as jarVersions, I as locksOf, L as npmVersions, M as LOCK_NAMES, N as cargoVersions, O as targetImageOf, P as goVersions, R as versionsOf, S as planOf, T as shippedSource, _ as kafkaClientImage, a as SHIPPED_NAMES, b as nodeHttpImage, c as buildRecordOf, d as harnessImage, f as harnessVersion, g as inContext, h as imageProblems, i as SHIPPED_CLIENTS, j as unpinnedBases, k as targetSourceOf, l as goHttpImage, m as imageOf, n as PINNED, o as amqpClientImage, p as hikariClientImage, r as REPOSITORY, s as basesOf, t as CLIENT_NAMES, u as grpcClientImage, v as mongoClientImage, w as routerImage, x as pgClientImage, y as nodeClientImage } from "./image-EZ8ioZ3L.js";
 import { quantity, valueIn } from "@mark1russell7/quantity";
 import { median, medianInterval as medianInterval$1, quantileSorted } from "@mark1russell7/rigor-statistics";
-import { createHash } from "node:crypto";
-import { existsSync, readFileSync } from "node:fs";
-import { release } from "node:os";
+import { createHash, randomBytes } from "node:crypto";
+import { execFile } from "node:child_process";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { release, tmpdir } from "node:os";
 import { join } from "node:path";
+import { existsSync, readFileSync } from "node:fs";
 //#region src/cell.ts
 /** This function tells if a fault place is a hop of a topology. */
 const isHopPlace = (place) => typeof place === "object" && "hop" in place;
@@ -286,6 +288,10 @@ const groupVerdictOf = (prediction, counted, delta = NOISE) => {
 	const errors = /* @__PURE__ */ new Map();
 	for (const run of counted) for (const error of run.group?.errors ?? []) errors.set(error.name, [...errors.get(error.name) ?? [], valueIn(error.first, "s")]);
 	const lags = counted.flatMap((run) => run.group?.lags ?? []);
+	const copyCounts = counted.flatMap((run) => {
+		const copies = run.group?.writes.duplicates;
+		return copies === null || copies === void 0 ? [] : [copies.length];
+	});
 	return {
 		stages,
 		medians,
@@ -308,7 +314,13 @@ const groupVerdictOf = (prediction, counted, delta = NOISE) => {
 			runs: firsts.length,
 			first: s(median(firsts.sort((a, b) => a - b)))
 		})),
-		...lags.length === 0 ? {} : { lags: spreadOf(lags) }
+		...lags.length === 0 ? {} : { lags: spreadOf(lags) },
+		...prediction.group?.duplicates === void 0 ? {} : { duplicates: {
+			runs: copyCounts.length,
+			total: copyCounts.reduce((sum, n) => sum + n, 0),
+			median: medianCount(copyCounts),
+			max: copyCounts.length === 0 ? null : Math.max(...copyCounts)
+		} }
 	};
 };
 /** This function gives the verdict of the band of a cell: its containment and the equivalence of its median. */
@@ -333,6 +345,16 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
 		why: "no run counted"
 	};
 	if (prediction.detects === null) {
+		if (prediction.queue !== void 0 || prediction.ports !== void 0) {
+			const off = new Set(outside.map((each) => each.run)).size;
+			return {
+				...empty,
+				decision: off === 0 ? "holds" : "fails",
+				band: null,
+				outside,
+				why: `${counted.length - off} of ${counted.length} runs hold the law of the ${prediction.queue === void 0 ? "port range" : "queue"}; ${prediction.why}`
+			};
+		}
 		const crossed = counted.flatMap((run) => run.crossings === void 0 ? [] : [run.crossings]);
 		if (crossed.length > 0) {
 			const hop = crossed[0].hop;
@@ -361,16 +383,30 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
 			}
 			const outcomes = [...kinds].map(([kind, n]) => `${n} ${kind}`).join(", ");
 			const timed = times.length === 0 ? "no time" : `median ${s(median(times)).value} s from ${prediction.from}, range [${s(times[0]).value}, ${s(times.at(-1)).value}] s`;
-			const counts = prediction.group?.lost;
+			const { counts } = prediction;
 			if (counts !== void 0) {
+				const exact = counts.map((each) => `${each.what} ${each.count}`).join(", ");
+				return {
+					...empty,
+					decision: outside.length === 0 ? "holds" : "fails",
+					band: null,
+					outside,
+					median: times.length === 0 ? null : s(median(times)),
+					why: `${counted.length - outside.length} of ${counted.length} runs gave the exact counts (${exact}); ${outcomes}; ${timed}. ${prediction.why}`
+				};
+			}
+			const lost = prediction.group?.lost;
+			const copies = prediction.group?.duplicates;
+			if (lost !== void 0 || copies !== void 0) {
 				const off = new Set(outside.map((each) => each.run)).size;
+				const bands = [...lost === void 0 ? [] : [`lost writes [${lost.low}, ${lost.high}]`], ...copies === void 0 ? [] : [`extra copies [${copies.low}, ${copies.high}]`]];
 				return {
 					...empty,
 					decision: off === 0 ? "holds" : "fails",
 					band: null,
 					outside,
 					median: times.length === 0 ? null : s(median(times)),
-					why: `${counted.length - off} of ${counted.length} runs inside the band of lost writes [${counts.low}, ${counts.high}]; ${outcomes}; ${timed}. ${prediction.why}`
+					why: `${counted.length - off} of ${counted.length} runs inside the band of ${bands.join(" and the band of ")}; ${outcomes}; ${timed}. ${prediction.why}`
 				};
 			}
 			const missed = outside.filter((each) => counted.some((run) => name(run) === each.run && run.reconnect !== void 0));
@@ -404,6 +440,7 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
 			band: null,
 			outside: [],
 			median: samples.length === 0 ? null : s(median(samples)),
+			...samples.length === 0 ? {} : { slowest: s(samples.at(-1)) },
 			...reordering === void 0 ? {} : { reordering },
 			why
 		};
@@ -526,7 +563,7 @@ const DROP_HANDLING = quantity(1.5, "ms");
 /** This function gives the window of the act of a drop, in seconds: `DROP_HANDLING`, or the pairing window (`delta`) when it is longer. */
 const dropWindow = (delta) => Math.max(valueIn(DROP_HANDLING, "s"), valueIn(delta, "s"));
 /** This function gives the interval between two instants on the one clock, in seconds to the microsecond. */
-const between$3 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
+const between$4 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
 /** This function gives the state of a state record after the change, and before it. TCP_ESTABLISHED is 1, TCP_SYN_SENT 2. */
 const newState$2 = (record) => record.new;
 const oldState$2 = (record) => record.old;
@@ -568,7 +605,7 @@ const eventOf = (socket, line, matches, delta = NOISE) => {
 */
 const leftAfter = (socket, line) => {
 	const left = socket.find((record) => record.ev === "state" && oldState$2(record) === 1 && (newState$2(record) === 4 || newState$2(record) === 7) && record.mono >= line.t);
-	return left === void 0 ? null : between$3(line.t, left.mono);
+	return left === void 0 ? null : between$4(line.t, left.mono);
 };
 /**
 * This function gives the end of the client process in a run, as the kernel stamps it. The process
@@ -660,19 +697,19 @@ const reconnectOf = (run, detected, cookie, from = detected) => {
 	const waits = starts.flatMap((start, at) => {
 		const next = starts[at + 1]?.t ?? Number.POSITIVE_INFINITY;
 		const dial = dials.find((each) => each.t >= start.t && each.t < next);
-		return dial === void 0 ? [] : [between$3(start.t, dial.t)];
+		return dial === void 0 ? [] : [between$4(start.t, dial.t)];
 	});
 	const fresh = run.kernel.filter((record) => record.netns === run.netns && record.cookie !== cookie && record.ev === "state" && record.mono > detected);
 	const syns = fresh.filter((record) => newState$2(record) === 2);
 	const established = fresh.find((record) => newState$2(record) === 1);
 	const connects = dials.flatMap((dial) => {
 		const syn = syns.find((record) => record.mono >= dial.t);
-		return syn === void 0 ? [] : [between$3(dial.t, syn.mono)];
+		return syn === void 0 ? [] : [between$4(dial.t, syn.mono)];
 	});
 	return {
 		waits,
-		syn: syns[0] === void 0 ? null : between$3(detected, syns[0].mono),
-		established: established === void 0 ? null : between$3(detected, established.mono),
+		syn: syns[0] === void 0 ? null : between$4(detected, syns[0].mono),
+		established: established === void 0 ? null : between$4(detected, established.mono),
 		connects
 	};
 };
@@ -699,7 +736,7 @@ const callOf = (events, request, detected) => {
 	};
 	return {
 		outcome: end.ev === "data" ? "a reply" : String(end["message"] ?? end["code"] ?? "an error"),
-		after: detected === void 0 ? null : between$3(detected, end.t)
+		after: detected === void 0 ? null : between$4(detected, end.t)
 	};
 };
 //#endregion
@@ -714,12 +751,22 @@ const ERRNO$1 = {
 	111: "ECONNREFUSED",
 	113: "EHOSTUNREACH"
 };
-const between$2 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
+const between$3 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
 const round = (seconds) => Math.round(seconds * 1e6) / 1e6;
 /** This function gives the value of a field of a line by its name. A dotted name is a field in a field. */
 const fieldOf = (line, path) => path.split(".").reduce((value, key) => value !== null && typeof value === "object" ? value[key] : void 0, line);
 /** This function tells if a line of the client matches a line match: its `ev`, and each field that the match names. */
 const matches = (line, match) => line.ev === match.ev && Object.entries(match.where ?? {}).every(([path, value]) => fieldOf(line, path) === value);
+/** This function gives the count of the lines that match. */
+const countOf = (lines, match) => lines.filter((line) => matches(line, match)).length;
+/** This function gives the finding of the line counts of a run, or null when each count is the exact one. */
+const countFinding = (counts, lines) => {
+	const off = counts.flatMap((each) => {
+		const seen = countOf(lines, each.match);
+		return seen === each.count ? [] : [`${each.what}: ${seen} lines, and the law gives ${each.count}`];
+	});
+	return off.length === 0 ? null : off.join("; ");
+};
 /** This function says a line match in words, for a finding. */
 const wordsOf = (match) => {
 	const where = Object.entries(match.where ?? {}).map(([path, value]) => `${path} ${String(value)}`);
@@ -870,7 +917,7 @@ const ipOf$1 = (address) => address.slice(0, address.lastIndexOf(":"));
 /** This function gives the first line of the client that matches, after an instant. */
 const lineAfter = (events, match, after) => events.find((event) => event.t > after && matches(event, match));
 /** This function gives the name of the error of a line: the field that the match names, or `name`. */
-const nameOf = (line, match) => String(fieldOf(line, match.name ?? "name") ?? line.ev);
+const nameOf$1 = (line, match) => String(fieldOf(line, match.name ?? "name") ?? line.ev);
 /**
 * This function gives the report of the kernel on a socket after an instant: an error, or the end of
 * the file (CLOSE_WAIT). With neither, a close of the socket in SYN_SENT is the end. The program gave
@@ -1022,7 +1069,7 @@ const measureDriver = (cell, run) => {
 	const until = prediction.detects ? Infinity : after + valueIn(prediction.window, "s") * 1e9;
 	for (const match of driver.quiet ?? []) {
 		const line = events.find((event) => event.t > after && event.t <= until && matches(event, match));
-		if (line !== void 0) findings.push(`${wordsOf(match)} came ${between$2(after, line.t).value} s after the fault, where none should`);
+		if (line !== void 0) findings.push(`${wordsOf(match)} came ${between$3(after, line.t).value} s after the fault, where none should`);
 	}
 	const expected = prediction.detects ? void 0 : prediction.middlebox?.forgets;
 	if (!prediction.detects && (run.middlebox !== void 0 || expected !== void 0)) {
@@ -1032,14 +1079,14 @@ const measureDriver = (cell, run) => {
 		for (const socket of socketsOf(events)) {
 			const forgot = middlebox.find((line) => line.ev === "ct" && line["type"] === "DESTROY" && String(line["orig"] ?? "").startsWith(`${socket.local}>`) && line.t > socket.opened && (socket.ended === null || line.t < socket.ended) && line.t <= until);
 			if (expected === void 0 || socket.local !== own) {
-				if (forgot !== void 0) findings.push(`a router forgot the flow of the ${socket.role} socket ${between$2(after, forgot.t).value} s after the fault, where it should keep it`);
+				if (forgot !== void 0) findings.push(`a router forgot the flow of the ${socket.role} socket ${between$3(after, forgot.t).value} s after the fault, where it should keep it`);
 				continue;
 			}
 			const last = referenceOf(run, "last-received", driver.socket, false);
 			if (forgot === void 0) findings.push(`no router forgot the flow of the ${socket.role} socket in the window, where it should forget it`);
 			else if (last === void 0) findings.push(`the ${socket.role} socket got no segment before the fault`);
 			else {
-				const at = between$2(last, forgot.t).value;
+				const at = between$3(last, forgot.t).value;
 				const [low, high] = [round(valueIn(expected.low, "s") - noise), round(valueIn(expected.high, "s") + noise)];
 				if (at < low || at > high) findings.push(`a router forgot the flow of the ${socket.role} socket ${at} s after its last segment, outside [${low}, ${high}] s`);
 			}
@@ -1052,12 +1099,12 @@ const measureDriver = (cell, run) => {
 			findings.push(line === void 0 ? `${step.what}: ${wordsOf(step.ends)} never came` : `${step.what}: its reference (${step.from}) never came`);
 			return {
 				what: step.what,
-				name: line === void 0 ? null : nameOf(line, step.ends),
+				name: line === void 0 ? null : nameOf$1(line, step.ends),
 				elapsed: null
 			};
 		}
-		const elapsed = between$2(from, line.t);
-		const name = nameOf(line, step.ends);
+		const elapsed = between$3(from, line.t);
+		const name = nameOf$1(line, step.ends);
 		if (step.name !== "OK" && name !== step.name) findings.push(`${step.what}: it ended with ${name}, not ${step.name}`);
 		const [low, high] = [round(valueIn(step.low, "s") - noise), round(valueIn(step.high, "s") + noise)];
 		if (elapsed.value < low || elapsed.value > high) findings.push(`${step.what}: ${elapsed.value} s is outside [${low}, ${high}] s`);
@@ -1090,7 +1137,7 @@ const measureDriver = (cell, run) => {
 			...nothing,
 			...stepped,
 			outcome: { kind: "answered" },
-			elapsed: between$2(from, line.t),
+			elapsed: between$3(from, line.t),
 			void: voided,
 			finding: findings[0] ?? null
 		};
@@ -1107,7 +1154,7 @@ const measureDriver = (cell, run) => {
 	const ended = line?.t ?? report?.at;
 	const outcome = line !== void 0 && driver.ends !== void 0 ? {
 		kind: "error",
-		errno: nameOf(line, driver.ends)
+		errno: nameOf$1(line, driver.ends)
 	} : report?.outcome ?? { kind: "silent" };
 	if (ended === void 0) return {
 		...base,
@@ -1133,15 +1180,431 @@ const measureDriver = (cell, run) => {
 		...base,
 		...stepped,
 		outcome,
-		elapsed: between$2(reference, ended),
-		socket: seen === void 0 ? null : between$2(reference, seen.t),
-		noticed: report === void 0 || seen === void 0 ? null : between$2(report.at, seen.t),
-		steps: (prediction.from === "last-received" ? records.filter((record) => record.ev === "probe" && record.what === "keepalive" && record.mono > reference) : []).map((record) => between$2(reference, record.mono)),
+		elapsed: between$3(reference, ended),
+		socket: seen === void 0 ? null : between$3(reference, seen.t),
+		noticed: report === void 0 || seen === void 0 ? null : between$3(report.at, seen.t),
+		steps: (prediction.from === "last-received" ? records.filter((record) => record.ev === "probe" && record.what === "keepalive" && record.mono > reference) : []).map((record) => between$3(reference, record.mono)),
 		void: voided,
 		finding: findings[0] ?? null
 	};
 };
 //#endregion
+//#region src/queue.ts
+/** This bit is the SYN of the flags of an `rx` record of the prober. */
+const SYN = 2;
+/** This function gives the time between two stamps of CLOCK_MONOTONIC. */
+const between$2 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
+const numberOf$1 = (line, field) => {
+	const value = line[field];
+	return typeof value === "number" ? value : 0;
+};
+/** This function says if a time is inside a band. */
+const inside = (time, band) => {
+	const seconds = valueIn(time, "s");
+	return seconds >= valueIn(band.low, "s") && seconds <= valueIn(band.high, "s");
+};
+/** This function gives the words of a band. */
+const bandWords = (band) => `[${valueIn(band.low, "s")}, ${valueIn(band.high, "s")}] s`;
+const waitsOf = (from, stamps) => stamps.map((stamp, at) => between$2(at === 0 ? from : stamps[at - 1], stamp));
+/** This function finds the records of the sockets of the client that had this local address. */
+const socketsAt = (run, local) => {
+	const cookies = new Set(run.kernel.filter((record) => record.netns === run.netns && record.src === local).map((record) => record.cookie));
+	return run.kernel.filter((record) => record.netns === run.netns && cookies.has(record.cookie));
+};
+const readConnection = (name, run, expected) => {
+	const lineOf = (ev) => run.events.find((event) => event.ev === ev && event["name"] === name);
+	const dial = lineOf("dial");
+	const connected = lineOf("connected");
+	const failed = lineOf("failed");
+	const state = lineOf("state");
+	const socket = dial === void 0 ? [] : socketsAt(run, dial["local"]);
+	const sent = socket.filter((record) => record.ev === "retrans").map((record) => record.mono);
+	const rx = socket.filter((record) => record.ev === "rx" && ((record.seg?.flags ?? 0) & SYN) !== 0);
+	const from = dial?.t ?? 0;
+	const start = expected?.ends === null || expected?.ends === void 0 ? void 0 : lineOf(expected.ends.from);
+	const last = sent.at(-1);
+	return {
+		name,
+		connected: connected !== void 0,
+		syns: waitsOf(from, connected === void 0 ? sent : sent.filter((stamp) => stamp < connected.t)),
+		synAcks: waitsOf(from, rx.map((record) => record.mono)),
+		retransmissions: sent.length,
+		errno: failed === void 0 ? null : String(failed["errno"]),
+		ended: failed === void 0 || start === void 0 ? null : between$2(start.t, failed.t),
+		afterLast: failed === void 0 || last === void 0 ? null : between$2(last, failed.t),
+		unacked: state === void 0 ? null : numberOf$1(state, "unacked")
+	};
+};
+/** This function reads one run against the band of each wait in a list, and gives the words of each miss. */
+const waitFindings = (what, seen, bands) => {
+	if (seen.length !== bands.length) return [`${what}: ${seen.length} waits, not ${bands.length}`];
+	return seen.flatMap((wait, at) => inside(wait, bands[at]) ? [] : [`${what}: wait ${at} of ${valueIn(wait, "s")} s, outside ${bandWords(bands[at])}`]);
+};
+const connectionFindings = (expected, read, expiredAt, stamps) => {
+	const { name, ends } = expected;
+	const found = [];
+	if (read.connected !== expected.connects) found.push(`${name}: ${read.connected ? "connected" : "did not connect"}, against the prediction`);
+	if (expected.syns !== void 0) found.push(...waitFindings(`${name} SYN`, read.syns, expected.syns));
+	if (expected.synAcks !== void 0) found.push(...waitFindings(`${name} SYN-ACK`, read.synAcks, expected.synAcks));
+	if (expected.unacked !== void 0 && read.unacked !== expected.unacked) found.push(`${name}: ${read.unacked ?? "no"} segments without an ACK, not ${expected.unacked}`);
+	if (ends === null) {
+		if (read.errno !== null) found.push(`${name}: ended with ${read.errno}, and the prediction has no end`);
+		return found;
+	}
+	if (read.errno !== ends.errno) found.push(`${name}: ended with ${read.errno ?? "nothing"}, not ${ends.errno}`);
+	if (read.ended === null) return [...found, `${name}: no time for its end`];
+	if (!inside(read.ended, ends.within)) found.push(`${name}: its end came at ${valueIn(read.ended, "s")} s, outside ${bandWords(ends.within)}`);
+	if (ends.retransmitted === void 0) return found;
+	if (read.afterLast === null || !inside(read.afterLast, ends.retransmitted)) found.push(`${name}: its end came ${read.afterLast === null ? "with no retransmission" : `${valueIn(read.afterLast, "s")} s after its last retransmission`}, outside ${bandWords(ends.retransmitted)}`);
+	if (expiredAt === null) return found;
+	const before = stamps.at(-2);
+	const last = stamps.at(-1);
+	if (last === void 0 || last < expiredAt || before !== void 0 && before > expiredAt) found.push(`${name}: its last two retransmissions do not span the expiry`);
+	return found;
+};
+/**
+* This function reads one run of a queue cell. The client lines give the connections and the
+* counters at the two ends of the run. The target lines give the count of the queue. The records of
+* the prober give the waits of each socket.
+*/
+const measureQueue = (run, expected) => {
+	const { events, kernel } = run;
+	const lineOf = (ev) => events.find((event) => event.ev === ev);
+	const base = {
+		environment: run.environment,
+		index: run.index,
+		outcome: { kind: "answered" },
+		elapsed: null,
+		socket: null,
+		noticed: null,
+		steps: [],
+		probes: 0,
+		retransmissions: kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length,
+		acks: 0
+	};
+	const voided = (why) => ({
+		...base,
+		void: why,
+		finding: null
+	});
+	const opened = lineOf("opened");
+	const stats = lineOf("stats");
+	if (opened === void 0) return voided("the target did not open its listener");
+	if (stats === void 0 || lineOf("done") === void 0) return voided("the client did not reach its end");
+	if (run.lost === null) return voided("the prober wrote no clock line after the run, so nothing says what it dropped");
+	if (run.lost > 0) return voided(`the prober dropped ${run.lost} records`);
+	const lines = (run.server ?? []).filter((line) => line.ev === "queue" && line.t >= opened.t);
+	if (lines.length === 0) return voided("the target wrote no queue line in the run");
+	const named = [...new Set(events.filter((event) => event.ev === "dial").map((event) => String(event["name"])))];
+	const predicted = new Map(expected.connections.map((each) => [each.name, each]));
+	const connections = named.map((name) => readConnection(name, run, predicted.get(name)));
+	const admitted = expected.admitted === void 0 ? void 0 : connections.find((each) => each.name === expected.admitted);
+	if (expected.admitted !== void 0 && (admitted === void 0 || !admitted.connected || admitted.unacked !== 0)) return voided(`${expected.admitted} did not take the last place in the queue: the burst did not land in its sequence`);
+	const change = (field) => numberOf$1(stats, field) - numberOf$1(opened, field);
+	const burst = events.find((event) => event.ev === "dial" && event["name"] === "in")?.t ?? opened.t;
+	const counted = expected.expiry?.timeouts;
+	const expired = counted === void 0 ? void 0 : lines.find((line) => numberOf$1(line, "timeouts") - numberOf$1(opened, "timeouts") >= counted);
+	const accepted = events.find((event) => event.ev === "accepted");
+	const rise = expected.rise?.count ?? 0;
+	const low = accepted === void 0 ? -1 : lines.findIndex((line) => line.t >= accepted.t && numberOf$1(line, "count") < rise);
+	const risen = low < 0 ? void 0 : lines.slice(low).find((line) => numberOf$1(line, "count") >= rise);
+	const drained = events.filter((event) => event.ev === "accepted").reduce((sum, event) => sum + (Array.isArray(event["peers"]) ? event["peers"].length : 0), 0);
+	const queue = {
+		max: numberOf$1(opened, "max"),
+		held: Math.max(...lines.map((line) => numberOf$1(line, "count"))),
+		connected: connections.filter((each) => each.connected).length,
+		overflows: change("overflows"),
+		drops: change("drops"),
+		cookies: change("cookies"),
+		timeouts: change("timeouts"),
+		resets: change("resets"),
+		drained,
+		expiry: expired === void 0 ? null : between$2(burst, expired.t),
+		rise: risen === void 0 ? null : between$2(burst, risen.t),
+		connections
+	};
+	const found = [];
+	if (queue.max !== expected.max) found.push(`the limit is ${queue.max}, not ${expected.max}`);
+	if (queue.held !== expected.held) found.push(`the queue held ${queue.held}, not ${expected.held}`);
+	if (queue.connected !== expected.connected) found.push(`${queue.connected} connected, not ${expected.connected}`);
+	if (queue.overflows < expected.overflows.low || queue.overflows > expected.overflows.high) found.push(`${queue.overflows} overflows, outside [${expected.overflows.low}, ${expected.overflows.high}]`);
+	if (queue.resets !== expected.resets) found.push(`${queue.resets} resets, not ${expected.resets}`);
+	if (queue.cookies !== 0) found.push(`${queue.cookies} SYN cookies, and the law has none`);
+	if (expected.drained !== void 0 && queue.drained !== expected.drained) found.push(`${queue.drained} accepted, not ${expected.drained}`);
+	if (expected.expiry !== void 0 && (queue.expiry === null || !inside(queue.expiry, expected.expiry.within))) found.push(`the expiry came at ${queue.expiry === null ? "no time" : `${valueIn(queue.expiry, "s")} s`}, outside ${bandWords(expected.expiry.within)}`);
+	if (expected.rise !== void 0 && (queue.rise === null || !inside(queue.rise, expected.rise.within))) found.push(`the count got back to ${expected.rise.count} at ${queue.rise === null ? "no time" : `${valueIn(queue.rise, "s")} s`}, outside ${bandWords(expected.rise.within)}`);
+	for (const each of expected.connections) {
+		const read = connections.find((connection) => connection.name === each.name);
+		if (read === void 0) {
+			found.push(`${each.name}: the client did not dial it`);
+			continue;
+		}
+		const dial = events.find((event) => event.ev === "dial" && event["name"] === each.name);
+		const stamps = socketsAt(run, dial?.["local"]).filter((record) => record.ev === "retrans").map((record) => record.mono);
+		found.push(...connectionFindings(each, read, expired?.t ?? null, stamps));
+	}
+	return {
+		...base,
+		void: null,
+		finding: found.length === 0 ? null : found.join(". "),
+		queue
+	};
+};
+//#endregion
+//#region src/ports.ts
+/** This errno ends a search that found no port. */
+const NO_PORT = "EADDRNOTAVAIL";
+const portsOf = (line) => Array.isArray(line["ports"]) ? line["ports"].filter((port) => typeof port === "number") : [];
+/** This function says if the ports of the parity of the low port come first in one search. */
+const lowFirst = (ports, low) => {
+	const odd = ports.findIndex((port) => (port - low) % 2 !== 0);
+	return odd < 0 || ports.slice(odd).every((port) => (port - low) % 2 !== 0);
+};
+/**
+* This function reads one run of a port cell. The `round` lines give each search. The `closing` and
+* `freed` lines give when each held port comes back. The records of the prober give the
+* retransmissions.
+*/
+const measurePorts = (run, expected) => {
+	const { events, kernel } = run;
+	const retransmissions = kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length;
+	const base = {
+		environment: run.environment,
+		index: run.index,
+		outcome: { kind: "answered" },
+		elapsed: null,
+		socket: null,
+		noticed: null,
+		steps: [],
+		probes: 0,
+		retransmissions,
+		acks: 0
+	};
+	const voided = (why) => ({
+		...base,
+		void: why,
+		finding: null
+	});
+	if (events.find((event) => event.ev === "done") === void 0) return voided("the client did not reach its end");
+	if (run.lost === null) return voided("the prober wrote no clock line after the run, so nothing says what it dropped");
+	if (run.lost > 0) return voided(`the prober dropped ${run.lost} records`);
+	const rounds = events.filter((event) => event.ev === "round");
+	const closing = events.find((event) => event.ev === "closing");
+	const freedLines = events.filter((event) => event.ev === "freed");
+	const counts = {};
+	for (const round of rounds) {
+		const count = String(portsOf(round).length);
+		counts[count] = (counts[count] ?? 0) + 1;
+	}
+	const seen = [...new Set(rounds.flatMap((round) => portsOf(round).map((port) => port - expected.low)))].sort((a, b) => a - b);
+	const refusal = Math.max(0, ...rounds.map((round) => typeof round["refusal"] === "number" ? round["refusal"] : 0));
+	const first = freedLines[0];
+	const last = freedLines.at(-1);
+	const ports = {
+		rounds: rounds.length,
+		counts,
+		seen,
+		errnos: [...new Set(rounds.map((round) => String(round["errno"])))],
+		refusal: quantity(Math.round(refusal / 1e3) / 1e6, "s"),
+		firstFreed: closing === void 0 || first === void 0 ? null : between$2(closing.t, first.t),
+		lastFreed: closing === void 0 || last === void 0 ? null : between$2(closing.t, last.t),
+		freed: freedLines.length,
+		tries: last === void 0 || typeof last["tries"] !== "number" ? 0 : last["tries"]
+	};
+	const found = [];
+	const searches = expected.rounds * expected.destinations;
+	if (ports.rounds !== searches) found.push(`${ports.rounds} searches, not ${searches}`);
+	const stray = Object.keys(counts).filter((count) => !expected.counts.includes(Number(count)));
+	if (stray.length > 0) found.push(`a search gave ${stray.join(" or ")} connects, outside ${expected.counts.join(" or ")}`);
+	if (seen.join() !== expected.seen.join()) found.push(`the ports at offsets ${seen.join(" ")} were used, not ${expected.seen.join(" ")}`);
+	if (ports.errnos.join() !== NO_PORT) found.push(`the searches ended with ${ports.errnos.join(" and ")}, not ${NO_PORT}`);
+	if (valueIn(ports.refusal, "s") > valueIn(expected.refusedWithin, "s")) found.push(`a refused connect took ${valueIn(ports.refusal, "s")} s, above ${valueIn(expected.refusedWithin, "s")} s`);
+	if (expected.share !== void 0) {
+		const { count, low, high } = expected.share;
+		const share = (counts[String(count)] ?? 0) / Math.max(1, ports.rounds);
+		if (share < low || share > high) found.push(`the share of ${count} connects is ${share}, outside [${low}, ${high}]`);
+	}
+	if (expected.lowFirst === true && !rounds.every((round) => lowFirst(portsOf(round), expected.low))) found.push("a search got a port of the other parity before each port of the parity of the low port");
+	if (expected.retransmissions !== void 0 && retransmissions !== expected.retransmissions) found.push(`${retransmissions} retransmissions, not ${expected.retransmissions}`);
+	if (expected.freed !== void 0) {
+		const held = Math.max(...expected.counts);
+		if (ports.freed !== held) found.push(`${ports.freed} ports came back, not ${held}`);
+		for (const [name, time] of [["first", ports.firstFreed], ["last", ports.lastFreed]]) if (time === null || !inside(time, expected.freed)) found.push(`the ${name} port came back at ${time === null ? "no time" : `${valueIn(time, "s")} s`}, outside ${bandWords(expected.freed)}`);
+		if (ports.tries === 0) found.push("no connect was refused before the ports came back");
+	}
+	return {
+		...base,
+		void: null,
+		finding: found.length === 0 ? null : found.join(". "),
+		ports
+	};
+};
+//#endregion
+//#region src/recovery.ts
+/**
+* This module holds the pure parts of a recovery drill. These are the scripts of the shell, the reader
+* of the output of `bsondump`, and the set check of a recovery. It has no container, thus unit tests
+* run it. `drill.ts` does the drill on a lab.
+*
+* A dump with `--oplog` reads the oplog from the start of the dump to its end. A slice reads the
+* oplog after the last entry of the backup before it. Thus the backups together hold each write up
+* to the last entry of the last backup, and no write after it. One writer sends the writes one at a
+* time, thus the highest sequence number is also the latest write.
+*/
+/** This size is one document of generated data. A tenant holds as many documents as its size needs. */
+const SEED_DOCUMENT = 65536;
+/** This count is how far the lost writes can be from the writes after the last backup: the writes in flight at the fault. */
+const LOSS_MARGIN = 2;
+/** This time is how long a script waits for a server to stop, in tenths of a second. */
+const STOP_TENTHS = 600;
+/** This function gives the time of a drill from the start of the backups to the fault, in seconds. */
+const drillSpan = (drill) => drill.slices * valueIn(drill.sliceEvery, "s") + valueIn(drill.age, "s");
+/**
+* This script writes the generated data of each tenant into the collection `tenants`. Each write
+* waits for `copies` members, thus each member holds the data when the script ends.
+*/
+const seedScript = (drill, copies) => {
+	const documents = Math.ceil(valueIn(drill.perTenant, "B") / SEED_DOCUMENT);
+	const blob = `BinData(0, require("crypto").randomBytes(${SEED_DOCUMENT}).toString("base64"))`;
+	return `const c = db.getSiblingDB("cells"); c.tenants.drop({ writeConcern: { w: ${copies} } }); for (let t = 1; t <= ${drill.tenants}; t += 1) { const docs = []; for (let d = 1; d <= ${documents}; d += 1) docs.push({ _id: "t" + t + ".d" + d, tenant: t, blob: ${blob} }); c.tenants.insertMany(docs, { writeConcern: { w: ${copies}, wtimeout: 60000 } }); } print(JSON.stringify({ ok: true }))`;
+};
+/** This script prints the hash of the collection `tenants`. It takes locks, thus a run reads it with no writer. */
+const HASH_SCRIPT = "print(JSON.stringify({ hash: db.getSiblingDB(\"cells\").runCommand({ dbHash: 1, collections: [\"tenants\"] }).collections.tenants }))";
+/** This function gives the hash from the output of `HASH_SCRIPT`. */
+const hashOf = (text) => {
+	const line = text.split("\n").filter((each) => each.trim() !== "").at(-1) ?? "";
+	const { hash } = JSON.parse(line);
+	if (typeof hash !== "string" || hash === "") throw new Error(`no hash of the tenants in: ${line}`);
+	return hash;
+};
+/** This function gives the command of a server with no set, on its own port. A client of the set cannot reach that server, and a restore writes to it. */
+const standaloneCommand = (set, port) => `${set.programs.server} --bind_ip_all --port ${port} ${(set.args ?? []).join(" ")}`.trim();
+/**
+* This script waits for the end of the server of a box, and it can stop the server first. `kill -0`
+* also succeeds on a zombie, and the shell of the box can leave its child as a zombie for a long
+* time. Thus the script reads the state of the process, and a zombie has ended.
+*/
+const stopScript = (terminate) => "pid=$(cat /tmp/server.pid); " + (terminate ? "kill -TERM \"$pid\"; " : "") + `for i in $(seq 1 ${STOP_TENTHS}); do s=$(cut -d' ' -f3 /proc/$pid/stat 2>/dev/null); if [ -z "\$s" ] || [ "\$s" = Z ]; then exit 0; fi; sleep 0.1; done; exit 1`;
+/** This function gives a number from Extended JSON, in its relaxed form or in its canonical form, or null. */
+const numberOf = (value) => {
+	if (typeof value === "number") return value;
+	if (typeof value !== "object" || value === null) return null;
+	const wrapped = value;
+	const text = wrapped.$numberInt ?? wrapped.$numberLong ?? wrapped.$numberDouble;
+	return typeof text === "string" && text !== "" && Number.isFinite(Number(text)) ? Number(text) : null;
+};
+/** This function gives each document of the output of `bsondump`. A line that is not a document does not count. */
+const documentsOf = (text) => text.split("\n").flatMap((line) => {
+	if (!line.trim().startsWith("{")) return [];
+	try {
+		return [JSON.parse(line)];
+	} catch {
+		return [];
+	}
+});
+/** This function gives the `ts` of an oplog entry, or null. */
+const timeOf = (entry) => {
+	const stamp = entry["ts"]?.$timestamp;
+	const t = numberOf(stamp?.t);
+	const i = numberOf(stamp?.i);
+	return t === null || i === null ? null : {
+		t,
+		i
+	};
+};
+/** This function reads an oplog file from the output of `bsondump`. Only an insert into `cells.w` is a write of a client. */
+const readOplog = (text) => {
+	let last = null;
+	let highest = null;
+	for (const entry of documentsOf(text)) {
+		const time = timeOf(entry);
+		if (time !== null && (last === null || time.t > last.t || time.t === last.t && time.i > last.i)) last = time;
+		if (entry["op"] !== "i" || entry["ns"] !== "cells.w") continue;
+		const n = numberOf(entry["o"]?.n);
+		if (n !== null && (highest === null || n > highest)) highest = n;
+	}
+	return {
+		last,
+		highest
+	};
+};
+/** This function gives the highest sequence number of the file of the writes in a dump, from the output of `bsondump`, or null. */
+const highestHeld = (text) => {
+	const held = documentsOf(text).flatMap((document) => {
+		const n = numberOf(document["n"]);
+		return n === null ? [] : [n];
+	});
+	return held.length === 0 ? null : Math.max(...held);
+};
+/** This function gives the query of a slice of the oplog: each entry after an oplog time, in Extended JSON. */
+const sliceQuery = (after) => JSON.stringify({ ts: { $gt: { $timestamp: {
+	t: after.t,
+	i: after.i
+} } } });
+const rounded = (value) => Math.round(value * 1e6) / 1e6;
+/** This function gives the acknowledgement time of each accepted write of the first client of a run. */
+const firstAcks = (events, second) => {
+	const acks = /* @__PURE__ */ new Map();
+	for (const event of events) {
+		const n = Number(event["n"]);
+		if (event.ev === "ack" && (second === null || n < second)) acks.set(n, event.t);
+	}
+	return acks;
+};
+/** This function gives the measurement of a drill, from its run. `t0` is the fault. */
+const recoveryOf = (drill, run, t0) => {
+	const record = run.group?.drill;
+	const sameData = record === void 0 || record.after === null ? null : record.after === record.before;
+	if (drill.recovery === "election" || record === void 0) return {
+		rpo: null,
+		excess: null,
+		holes: [],
+		afterBackup: null,
+		sameData
+	};
+	const acks = firstAcks(run.events, record.second);
+	const accepted = [...acks.keys()].sort((a, b) => a - b);
+	const cutoff = record.backups.at(-1)?.last?.n ?? 0;
+	const afterBackup = accepted.filter((n) => n > cutoff).length;
+	const holds = run.group?.holds ?? null;
+	if (holds === null) return {
+		rpo: null,
+		excess: null,
+		holes: [],
+		afterBackup,
+		sameData
+	};
+	const kept = new Set(holds);
+	const ownKept = holds.filter((n) => record.second === null || n < record.second);
+	const highest = ownKept.length === 0 ? 0 : Math.max(...ownKept);
+	const holes = accepted.filter((n) => n < highest && !kept.has(n));
+	const lastKept = accepted.filter((n) => kept.has(n)).at(-1);
+	const rpo = lastKept === void 0 ? null : rounded((t0 - acks.get(lastKept)) / 1e9);
+	return {
+		rpo: rpo === null ? null : {
+			value: rpo,
+			unit: "s"
+		},
+		excess: rpo === null || drill.quiet === true ? null : {
+			value: rounded(rpo - valueIn(drill.age, "s")),
+			unit: "s"
+		},
+		holes,
+		afterBackup,
+		sameData
+	};
+};
+/** This function gives the findings of a drill: each rule of a recovery that its run broke. */
+const recoveryFindings = (measured, writes) => {
+	const findings = [];
+	if (measured.sameData === null) findings.push("the set did not converge, thus nothing compared the data of the tenants");
+	else if (!measured.sameData) findings.push("the data of the tenants changed in the recovery");
+	if (measured.holes.length > 0) findings.push(`the restore lost ${measured.holes.length} accepted writes below a write that it kept, the first at ${measured.holes[0]}`);
+	const lost = writes.lost?.length ?? null;
+	if (measured.afterBackup !== null && lost !== null && Math.abs(lost - measured.afterBackup) > 2) findings.push(`the restore lost ${lost} accepted writes, and ${measured.afterBackup} came after the last backup`);
+	return findings;
+};
+//#endregion
 //#region src/timeline.ts
 /**
 * This module measures a run of a group cell. It gives the first write that a primary accepted
@@ -1348,12 +1811,32 @@ const writesOf = (run, after, primary) => {
 			held: kept === null ? null : kept.has(n)
 		};
 	});
+	const duplicates = holds === null ? null : [...holds].sort((a, b) => a - b).filter((n, at, all) => all[at - 1] === n);
+	const oldHolds = run.group?.oldHolds;
 	return {
 		sent: sent.length,
 		acked: acks.size,
 		old,
 		lost,
-		unknown
+		unknown,
+		duplicates,
+		...oldHolds === void 0 ? {} : { sets: setsOf(run, acks.keys(), unknown, new Set(oldHolds)) }
+	};
+};
+/**
+* This function splits the writes of a pair by its two final reads. A write with no reply is `kept`
+* on the new primary. It is `stranded` on the old primary alone, or `absent` from each server.
+*/
+const setsOf = (run, acked, unknown, old) => {
+	const kept = unknown.filter((each) => each.held === true).length;
+	const stranded = unknown.filter((each) => each.held !== true && old.has(each.n)).length;
+	return {
+		unknown: unknown.length,
+		kept,
+		stranded,
+		absent: unknown.length - kept - stranded,
+		missing: [...acked].filter((n) => !old.has(n)).length,
+		warnings: run.events.filter((event) => event.ev === "notice" && event["severity"] === "WARNING").length
 	};
 };
 /** This function gives the errors that the client met after an instant: each kind, its first time, and its count. */
@@ -1464,9 +1947,10 @@ const measureGroup = (cell, run) => {
 	const heal = run.heal ?? null;
 	const lands = expected?.lands ?? "new";
 	const cluster = cell.cluster !== void 0;
-	const replica = cell.replicaSet !== void 0 || cluster;
+	const kafka = cell.kafka !== void 0;
+	const replica = cell.replicaSet !== void 0 || cluster || kafka;
 	const stall = prediction.detects === false && prediction.recovers !== void 0;
-	const stages = cluster ? clusterStagesOf(run, t0, run.fault.t1) : replica ? replicaSetStagesOf(run, t0, run.fault.t1, lands) : stagesOf(timeline, t0);
+	const stages = kafka ? {} : cluster ? clusterStagesOf(run, t0, run.fault.t1) : replica ? replicaSetStagesOf(run, t0, run.fault.t1, lands) : stagesOf(timeline, t0);
 	const after = lands === "same" && heal !== null && (!replica || stall) ? heal.t0 : t0;
 	const sentAt = /* @__PURE__ */ new Map();
 	for (const event of events) if (event.ev === "sent") sentAt.set(Number(event["n"]), event.t);
@@ -1487,7 +1971,8 @@ const measureGroup = (cell, run) => {
 		stages,
 		writes: writesOf(run, t0, cluster ? "" : primary),
 		errors: errorsOf(events, t0),
-		...cell.calibrates === "sentinels" ? { lags: lagsOf(run, kernel) } : {}
+		...cell.calibrates === "sentinels" ? { lags: lagsOf(run, kernel) } : {},
+		...cell.drill === void 0 ? {} : { recovery: recoveryOf(cell.drill, run, t0) }
 	};
 	let voided = null;
 	const tilt = timeline.find((line) => line.ev === "sentinel" && line.event === "+tilt" && line.t >= t0);
@@ -1497,14 +1982,14 @@ const measureGroup = (cell, run) => {
 	if (voided === null && split !== void 0 && (elected === void 0 || elected.t > split.t)) voided = "the election split its votes and runs again after 2 × failover-timeout, which the band does not bound";
 	const odown = stages["+odown"];
 	if (voided === null && odown !== void 0 && stages["+try-failover"] === void 0) voided = "no Sentinel started a failover after +odown: each was in the hold-off of 2 × failover-timeout after an earlier attempt";
-	if (voided === null && cell.hold !== void 0 && prediction.detects !== null) {
+	if (voided === null && cell.hold !== void 0 && (prediction.detects !== null || kafka)) {
 		if (heal === null) voided = "the fault never lifted";
 		else {
 			const late = (heal.t0 - t0) / 1e9 - valueIn(cell.hold, "s");
 			if (late > valueIn(INJECT, "s")) voided = `the lift came ${Math.round(late * 1e4) / 10} ms after its hold: the injector was late, not the group`;
 		}
 	}
-	const firstAddress = cluster ? String(events.find((event) => event.ev === "connected")?.["at"] ?? "") : `${run.group?.members[primary]?.ip}:${(cell.group ?? cell.replicaSet)?.port}`;
+	const firstAddress = cluster ? String(events.find((event) => event.ev === "connected")?.["at"] ?? "") : `${run.group?.members[primary]?.ip}:${(cell.group ?? cell.replicaSet ?? cell.kafka)?.port}`;
 	const assumed = prediction.detects === null ? void 0 : prediction.rtt;
 	const first = kernel.find((record) => record.netns === run.netns && record.dst === firstAddress && record.tcp?.srtt !== void 0);
 	const rtt = first?.tcp?.srtt === void 0 ? null : valueIn(first.tcp.srtt, "us") / 1e3;
@@ -1524,11 +2009,13 @@ const measureGroup = (cell, run) => {
 	const findings = [];
 	const switched = replica ? stages["elected"] : stages["+switch-master"];
 	if (expected?.failover === false && switched !== void 0) findings.push(`the group failed over ${switched.value} s after the fault, where it should not`);
+	const leaderAfter = run.group?.after ?? null;
+	if (kafka && expected !== void 0 && leaderAfter !== null && expected.failover !== (leaderAfter !== primary)) findings.push(expected.failover ? `the leader stayed on ${primary}, where it should change` : `the leader moved to ${leaderAfter}, where it should stay on ${primary}`);
 	for (const stage of expected?.quiet ?? []) {
 		const at = stages[stage];
 		if (at !== void 0) findings.push(`${stage} came ${at.value} s after the fault, where it should not`);
 	}
-	if (accepted !== void 0 && lands === "same" && !cluster && to !== primary) findings.push(`the client wrote to ${to ?? String(accepted["at"])}, where it should stay on ${primary}`);
+	if (accepted !== void 0 && lands === "same" && !cluster && !kafka && to !== primary) findings.push(`the client wrote to ${to ?? String(accepted["at"])}, where it should stay on ${primary}`);
 	for (const error of expected?.errors ?? []) {
 		const met = group.errors.find((each) => each.name === error.name);
 		const noise = valueIn(noiseOf(cell), "s");
@@ -1541,8 +2028,22 @@ const measureGroup = (cell, run) => {
 		if (lost === null) findings.push("the group did not converge, thus the count of its lost writes is not known");
 		else if (lost.length < low || lost.length > high) findings.push(`the group lost ${lost.length} accepted writes, outside [${low}, ${high}]`);
 	}
+	if (expected?.duplicates !== void 0) {
+		const { low, high } = expected.duplicates;
+		const copies = group.writes.duplicates ?? null;
+		if (copies === null) findings.push("the group did not converge, thus the count of its extra copies is not known");
+		else if (run.group?.holds?.length === 0) findings.push("the group holds no write, thus the count of its extra copies says nothing");
+		else if (copies.length < low || copies.length > high) findings.push(`the group holds ${copies.length} extra copies, outside [${low}, ${high}]`);
+	}
+	const { sets } = group.writes;
+	for (const [name, band] of Object.entries(expected?.outcomes ?? {})) {
+		const count = sets?.[name];
+		if (count === void 0) findings.push(`the run has no count of ${name}`);
+		else if (count < band.low || count > band.high) findings.push(`the count of ${name} is ${count}, outside [${band.low}, ${band.high}]`);
+	}
+	if (group.recovery !== void 0) findings.push(...recoveryFindings(group.recovery, group.writes));
 	if (accepted === void 0) {
-		if (prediction.detects !== null) findings.push(lands === "new" ? "the client never wrote to a new primary in the window" : "the client never wrote again in the window");
+		if (prediction.detects !== null || kafka) findings.push(lands === "new" ? "the client never wrote to a new primary in the window" : "the client never wrote again in the window");
 		return {
 			...silent,
 			void: voided,
@@ -1811,7 +2312,7 @@ const requestOf = (events) => {
 };
 /** This function reads the socket of one run against its cell, as `measure` does without the second request of an HTTP client. */
 const measureSocket = (cell, run) => {
-	if (cell.group !== void 0 || cell.replicaSet !== void 0 || cell.cluster !== void 0) {
+	if (cell.group !== void 0 || cell.replicaSet !== void 0 || cell.cluster !== void 0 || cell.pair !== void 0 || cell.kafka !== void 0) {
 		const read = measureGroup(cell, run);
 		return {
 			environment: run.environment,
@@ -1829,6 +2330,8 @@ const measureSocket = (cell, run) => {
 			group: read.group
 		};
 	}
+	if (cell.prediction.detects === null && cell.prediction.queue !== void 0) return measureQueue(run, cell.prediction.queue);
+	if (cell.prediction.detects === null && cell.prediction.ports !== void 0) return measurePorts(run, cell.prediction.ports);
 	if (cell.prediction.detects !== null && cell.prediction.driver !== void 0) return measureDriver(cell, run);
 	const { events, kernel } = run;
 	const ready = events.find((event) => event.ev === "ready");
@@ -1985,12 +2488,13 @@ const measureSocket = (cell, run) => {
 		}, run)) };
 		const reconnect = prediction.reconnects === void 0 || ended === void 0 ? void 0 : reconnectOf(run, ended, cookie, drop?.line.t ?? ended);
 		const late = reconnect === void 0 || prediction.reconnects === void 0 ? null : reconnectFinding(prediction.reconnects, reconnect, noiseOf(cell));
+		const counted = prediction.counts === void 0 ? null : countFinding(prediction.counts, run.server ?? []);
 		return {
 			...base,
 			...nothing,
 			...chain,
 			...reconnect === void 0 ? {} : { reconnect },
-			finding: late,
+			finding: late ?? counted,
 			outcome: ended !== void 0 ? outcome : { kind: "answered" },
 			...from === void 0 || until === void 0 ? {} : { elapsed: between(from, until) },
 			noticed: prediction.from === void 0 ? null : noticed,
@@ -2169,6 +2673,440 @@ const measureSocket = (cell, run) => {
 	};
 };
 //#endregion
+//#region src/fabric.ts
+/**
+* A fabric is a routed network between the client and the target, on layer E3 (docs/S7.md). Four
+* routers of FRRouting, two leaves and two spines, run real BGP over veth pairs that containerlab
+* makes. The client hangs off leaf1 and the target off leaf2. The path between them goes through
+* spine1 by a weight, and spine2 holds the other path. A cell silences spine1, and the routes of
+* the leaves move to spine2 when a detector of the routers fires.
+*
+* This file holds the plan of the fabric as pure functions, and `openFabric` that starts it. A lab
+* opens a fabric with `Topology.fabric`. Thus the client, the target, the prober and the verdict
+* stay the same as on layer E1. The fabric needs a Linux host with containerlab and sudo.
+*/
+/** This image is FRRouting 10.7.1, pinned by digest. */
+const FRR_IMAGE = "quay.io/frrouting/frr:10.7.1@sha256:e995beaa50fdc9edb35eadcfefa29b7f062cc06f2b812613789b68fa541554d2";
+/** This device is the side of the fabric in the client and in the target. Docker keeps eth0. */
+const FABRIC_DEVICE = "eth1";
+/** This address is the client on its network behind leaf1. */
+const FABRIC_CLIENT = "10.1.0.2";
+/** This address is the target on its network behind leaf2. */
+const FABRIC_TARGET = "10.2.0.2";
+/** This device of a leaf goes to spine1, thus a route on it is on the preferred path. */
+const PREFERRED_DEVICE = "eth1";
+/** This limit is the time that the sessions and the routes get to come back before a run. */
+const READY_MS = 9e4;
+const READY_POLL_MS = 250;
+const CLIENT_NET = "10.1.0.0/24";
+const TARGET_NET = "10.2.0.0/24";
+/** These are the routers. Each leaf prefers spine1 by a weight of 100, thus the path of each run is known. */
+const ROUTERS = [
+	{
+		name: "leaf1",
+		as: 65001,
+		id: "10.255.0.1",
+		interfaces: {
+			eth1: "10.0.1.0/31",
+			eth2: "10.0.2.0/31",
+			eth3: "10.1.0.1/24"
+		},
+		neighbors: [{
+			address: "10.0.1.1",
+			as: 65101,
+			weight: 100
+		}, {
+			address: "10.0.2.1",
+			as: 65102
+		}],
+		network: CLIENT_NET,
+		remote: TARGET_NET
+	},
+	{
+		name: "leaf2",
+		as: 65002,
+		id: "10.255.0.2",
+		interfaces: {
+			eth1: "10.0.3.0/31",
+			eth2: "10.0.4.0/31",
+			eth3: "10.2.0.1/24"
+		},
+		neighbors: [{
+			address: "10.0.3.1",
+			as: 65101,
+			weight: 100
+		}, {
+			address: "10.0.4.1",
+			as: 65102
+		}],
+		network: TARGET_NET,
+		remote: CLIENT_NET
+	},
+	{
+		name: "spine1",
+		as: 65101,
+		id: "10.255.1.1",
+		interfaces: {
+			eth1: "10.0.1.1/31",
+			eth2: "10.0.3.1/31"
+		},
+		neighbors: [{
+			address: "10.0.1.0",
+			as: 65001
+		}, {
+			address: "10.0.3.0",
+			as: 65002
+		}]
+	},
+	{
+		name: "spine2",
+		as: 65102,
+		id: "10.255.1.2",
+		interfaces: {
+			eth1: "10.0.2.1/31",
+			eth2: "10.0.4.1/31"
+		},
+		neighbors: [{
+			address: "10.0.2.0",
+			as: 65001
+		}, {
+			address: "10.0.4.0",
+			as: 65002
+		}]
+	}
+];
+const LEAVES = ROUTERS.filter((router) => router.remote !== void 0);
+/** These are the links between the routers. The links to the client and the target come from `topologyOf`. */
+const LINKS = [
+	["leaf1:eth1", "spine1:eth1"],
+	["leaf1:eth2", "spine2:eth1"],
+	["leaf2:eth1", "spine1:eth2"],
+	["leaf2:eth2", "spine2:eth2"]
+];
+/** This text is the `daemons` file of each router: BGP and BFD, on top of what FRRouting always starts. */
+const DAEMONS = [
+	"bgpd=yes",
+	"bfdd=yes",
+	"vtysh_enable=yes",
+	"zebra_options=\"  -A 127.0.0.1 -s 90000000\"",
+	"mgmtd_options=\"  -A 127.0.0.1\"",
+	"bgpd_options=\"   -A 127.0.0.1\"",
+	"bfdd_options=\"   -A 127.0.0.1\"",
+	"staticd_options=\"-A 127.0.0.1\"",
+	""
+].join("\n");
+/**
+* This function gives the configuration of a router. Graceful restart is off: after a hold expiry,
+* a helper can keep the stale routes, and the move that a cell measures does not come.
+* `maximum-paths 1` keeps one path, thus a move is one change of one route.
+*/
+const frrConf = (router, fabric) => {
+	const keepalive = valueIn(fabric.keepalive, "s");
+	const hold = valueIn(fabric.hold, "s");
+	const out = [
+		"frr defaults traditional",
+		`hostname ${router.name}`,
+		"ip forwarding",
+		"!"
+	];
+	for (const [name, address] of Object.entries(router.interfaces)) out.push(`interface ${name}`, ` ip address ${address}`, "!");
+	if (fabric.bfd !== null) {
+		const interval = valueIn(fabric.bfd.interval, "ms");
+		out.push("bfd", " profile fast", `  receive-interval ${interval}`, `  transmit-interval ${interval}`, `  detect-multiplier ${fabric.bfd.multiplier}`, " exit", "exit", "!");
+	}
+	out.push(`router bgp ${router.as}`, ` bgp router-id ${router.id}`, " no bgp ebgp-requires-policy", " no bgp network import-check", " bgp graceful-restart-disable");
+	for (const neighbor of router.neighbors) {
+		out.push(` neighbor ${neighbor.address} remote-as ${neighbor.as}`, ` neighbor ${neighbor.address} timers ${keepalive} ${hold}`, ` neighbor ${neighbor.address} timers connect 1`, ` neighbor ${neighbor.address} advertisement-interval 0`);
+		if (fabric.bfd !== null) out.push(` neighbor ${neighbor.address} bfd profile fast`);
+	}
+	out.push(" !", " address-family ipv4 unicast");
+	if (router.network !== void 0) out.push(`  network ${router.network}`);
+	out.push("  maximum-paths 1");
+	for (const neighbor of router.neighbors) if (neighbor.weight !== void 0) out.push(`  neighbor ${neighbor.address} weight ${neighbor.weight}`);
+	out.push(" exit-address-family", "exit", "");
+	return out.join("\n");
+};
+/**
+* This function gives the topology file of containerlab. The client and the target are containers
+* of the lab, and the fabric adds a device to each. Each link has the MTU of a Docker network, thus
+* a segment is as long as on layer E1. The default of containerlab is 9500. The management network
+* belongs to the lab, and Docker chooses its subnet.
+*/
+const topologyOf = (lab, dir, client, target) => {
+	const nodes = ROUTERS.flatMap((router) => [
+		`    ${router.name}:`,
+		"      kind: linux",
+		`      image: ${FRR_IMAGE}`,
+		"      binds:",
+		`        - ${dir}/${router.name}.conf:/etc/frr/frr.conf`,
+		`        - ${dir}/daemons:/etc/frr/daemons`,
+		"      sysctls:",
+		"        net.ipv4.ip_forward: 1",
+		"        net.ipv4.conf.all.rp_filter: 0",
+		"        net.ipv4.conf.default.rp_filter: 0"
+	]);
+	const links = [
+		...LINKS,
+		["leaf1:eth3", `${client}:${FABRIC_DEVICE}`],
+		["leaf2:eth3", `${target}:${FABRIC_DEVICE}`]
+	];
+	return [
+		`name: ${lab}`,
+		"mgmt:",
+		`  network: clab-${lab}`,
+		"  ipv4-subnet: auto",
+		"topology:",
+		"  nodes:",
+		...nodes,
+		`    ${client}:`,
+		"      kind: ext-container",
+		`    ${target}:`,
+		"      kind: ext-container",
+		"  links:",
+		...links.flatMap(([a, b]) => [`    - endpoints: ["${a}", "${b}"]`, "      mtu: 1500"]),
+		""
+	].join("\n");
+};
+/** This function reads a line of `ip -ts -o monitor route`, or it gives null for another line. The clock of a container is UTC. */
+const routeLine = (text) => {
+	const line = text.trim();
+	const found = /^\[(\d{4})-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)\.(\d{6})\] /.exec(line);
+	if (found === null) return null;
+	const [stamp, year, month, day, hour, minute, second, micro] = found;
+	const words = line.slice(stamp.length).split(" ").filter((word) => word !== "");
+	const change = words[0] === "Deleted" || words[0] === "Replaced" ? words[0] : null;
+	const prefix = words[change === null ? 0 : 1];
+	if (prefix === void 0) return null;
+	const after = (name) => {
+		const at = words.indexOf(name);
+		return at === -1 ? null : words[at + 1] ?? null;
+	};
+	return {
+		wall: Date.UTC(Number(year), Number(month) - 1, Number(day), Number(hour), Number(minute), Number(second)) + Number(micro) / 1e3,
+		deleted: change === "Deleted",
+		prefix,
+		via: after("via"),
+		dev: after("dev")
+	};
+};
+/**
+* This function takes a time on the wall clock to the monotonic clock. The harness runs on the host
+* of the containers, thus `process.hrtime` reads the CLOCK_MONOTONIC of the injector and the
+* prober. The error of the pair is approximately 1 ms, the step of `Date.now`.
+*/
+const monotonicAt = (wall, pair) => pair.mono + (wall - pair.wall) * 1e6;
+/** This function gives the state of the BGP sessions from `show bgp neighbors json`. */
+const sessionsOf = (json) => {
+	const views = Object.values(JSON.parse(json));
+	return {
+		up: views.filter((view) => view.bgpState === "Established").length,
+		established: views.reduce((sum, view) => sum + (view.connectionsEstablished ?? 0), 0)
+	};
+};
+/**
+* This function tells what a leaf still lacks before a run, or it gives null. The two sessions must
+* be up and the route must be on spine1. Spine2 must hold the other path. Without these, the fault
+* of a run acts on another path, or nothing can take over.
+*/
+const leafLacks = (views) => {
+	const { up } = sessionsOf(views.neighbors);
+	if (up < 2) return `${up} of 2 BGP sessions are up`;
+	if (!views.route.includes(` dev ${PREFERRED_DEVICE} `)) return `the route is not on spine1: ${views.route || "no route"}`;
+	const paths = JSON.parse(views.paths).paths?.length ?? 0;
+	if (paths < 2) return `BGP holds ${paths} of 2 paths`;
+	if (views.bfd !== null) {
+		const live = JSON.parse(views.bfd).filter((peer) => peer.status === "up").length;
+		if (live < 2) return `${live} of 2 BFD peers are up`;
+	}
+	return null;
+};
+/**
+* This function gives the later move of the leaves after an instant: for each leaf, the first route
+* to its far network that leaves spine1. It gives null when a leaf has no move. The later move ends
+* the outage in the two directions.
+*/
+const laterMove = (routes, after) => {
+	const firsts = LEAVES.map((leaf) => routes.find((route) => route.leaf === leaf.name && route.prefix === leaf.remote && route.t > after && !route.deleted && route.dev !== PREFERRED_DEVICE)?.t);
+	return firsts.every((t) => t !== void 0) ? Math.max(...firsts) : null;
+};
+/** This function runs a program on the host and gives its output. Its error holds the stderr of the program. */
+const host = (program, args) => new Promise((resolve, reject) => {
+	execFile(program, [...args], { maxBuffer: 16777216 }, (error, stdout, stderr) => {
+		if (error === null) resolve(stdout);
+		else reject(/* @__PURE__ */ new Error(`${program} ${args.join(" ")}: ${error.message}\n${stderr}`));
+	});
+});
+const nameOf = (container) => container.getName().replace(/^\//, "");
+/** This function gives a clock pair at this instant. */
+const clockPair = () => ({
+	mono: Number(process.hrtime.bigint()),
+	wall: Date.now()
+});
+/**
+* This function deploys the routers of a lab and joins the client and the target to them. It also
+* starts the sidecars: the injector of spine1 and a route monitor on each leaf. The caller stops the
+* sidecars with the other containers of the lab, and it calls `close` first.
+*/
+const openFabric = async (parts) => {
+	const { fabric, image, admin, injector, opened, output } = parts;
+	const { GenericContainer } = await import("testcontainers");
+	const lab = `e3-${randomBytes(4).toString("hex")}`;
+	const dir = await mkdtemp(join(tmpdir(), "fabric-"));
+	const file = join(dir, "lab.clab.yml");
+	await writeFile(join(dir, "daemons"), DAEMONS);
+	for (const router of ROUTERS) await writeFile(join(dir, `${router.name}.conf`), frrConf(router, fabric));
+	await writeFile(file, topologyOf(lab, dir, nameOf(parts.client), nameOf(parts.server)));
+	const close = async () => {
+		await host("sudo", [
+			"containerlab",
+			"destroy",
+			"-t",
+			file,
+			"--cleanup"
+		]).catch(() => void 0);
+		await rm(dir, {
+			recursive: true,
+			force: true
+		}).catch(() => void 0);
+	};
+	const node = (name) => `clab-${lab}-${name}`;
+	try {
+		await host("sudo", [
+			"containerlab",
+			"deploy",
+			"-t",
+			file
+		]);
+		const version = /version:\s*(\S+)/.exec(await host("containerlab", ["version"]))?.[1] ?? "unknown";
+		await output(admin, [
+			"ip",
+			"addr",
+			"add",
+			`${FABRIC_CLIENT}/24`,
+			"dev",
+			FABRIC_DEVICE
+		]);
+		await output(admin, [
+			"ip",
+			"route",
+			"replace",
+			TARGET_NET,
+			"via",
+			"10.1.0.1",
+			"dev",
+			FABRIC_DEVICE
+		]);
+		await output(injector, [
+			"ip",
+			"addr",
+			"add",
+			`${FABRIC_TARGET}/24`,
+			"dev",
+			FABRIC_DEVICE
+		]);
+		await output(injector, [
+			"ip",
+			"route",
+			"replace",
+			CLIENT_NET,
+			"via",
+			"10.2.0.1",
+			"dev",
+			FABRIC_DEVICE
+		]);
+		const silencer = await new GenericContainer(image).withNetworkMode(`container:${node("spine1")}`).withAddedCapabilities("NET_ADMIN", "NET_RAW").start();
+		opened.push(silencer);
+		const routes = [];
+		const monitors = /* @__PURE__ */ new Map();
+		for (const leaf of LEAVES) {
+			const monitor = await new GenericContainer(image).withNetworkMode(`container:${node(leaf.name)}`).withCommand([
+				"ip",
+				"-ts",
+				"-o",
+				"monitor",
+				"route"
+			]).withLogConsumer((stream) => {
+				let rest = "";
+				stream.on("data", (chunk) => {
+					const text = (rest + chunk.toString()).split("\n");
+					rest = text.pop() ?? "";
+					for (const each of text) {
+						const line = routeLine(each);
+						if (line === null) continue;
+						const { wall, ...route } = line;
+						routes.push({
+							...route,
+							leaf: leaf.name,
+							t: monotonicAt(wall, clockPair())
+						});
+					}
+				});
+			}).start();
+			opened.push(monitor);
+			monitors.set(leaf.name, monitor);
+		}
+		const stamp = async (action) => {
+			const last = (await output(silencer, ["/opt/harness/inject", action])).split("\n").at(-1);
+			const { t0, t1 } = JSON.parse(last);
+			return {
+				t0,
+				t1
+			};
+		};
+		const vtysh = (leaf, view) => host("docker", [
+			"exec",
+			node(leaf),
+			"vtysh",
+			"-c",
+			view
+		]);
+		return {
+			routes,
+			silence: () => stamp("silence"),
+			clear: () => stamp("clear-silence"),
+			async ready() {
+				const deadline = Date.now() + READY_MS;
+				for (;;) {
+					const lacks = [];
+					for (const leaf of LEAVES) {
+						const prefix = leaf.remote;
+						const lacking = await (async () => leafLacks({
+							neighbors: await vtysh(leaf.name, "show bgp neighbors json"),
+							route: `${await output(monitors.get(leaf.name), [
+								"ip",
+								"-o",
+								"route",
+								"show",
+								prefix
+							])} `,
+							paths: await vtysh(leaf.name, `show bgp ipv4 unicast ${prefix} json`),
+							bfd: fabric.bfd === null ? null : await vtysh(leaf.name, "show bfd peers json")
+						}))().catch((error) => String(error));
+						if (lacking !== null) lacks.push(`${leaf.name}: ${lacking}`);
+					}
+					if (lacks.length === 0) return;
+					if (Date.now() > deadline) throw new Error(`the fabric is not ready: ${lacks.join("; ")}`);
+					await new Promise((resolve) => setTimeout(resolve, READY_POLL_MS));
+				}
+			},
+			async sessions() {
+				const out = {};
+				for (const leaf of LEAVES) out[leaf.name] = sessionsOf(await vtysh(leaf.name, "show bgp neighbors json")).established;
+				return out;
+			},
+			manifest: () => ({
+				...fabric,
+				image: FRR_IMAGE,
+				containerlab: version
+			}),
+			close
+		};
+	} catch (error) {
+		await close();
+		throw error;
+	}
+};
+//#endregion
 //#region src/substrate.ts
 /** GitHub announced Linux jobs on AWS on this date. Each earlier hosted job ran on Azure. */
 const AWS_FROM = "2026-10-28";
@@ -2207,6 +3145,41 @@ const substrateOn = (layer, host = readHost()) => ({
 	...host
 });
 /**
+* This function gives the substrate of a lab at this place. A guest cannot read its own runner: its
+* DMI vendor is QEMU, and `ImageOS` is not set. Thus the runner writes its host to `LAB_RUNNER`, and
+* the guest keeps only its own kernel and HZ. A value of that variable makes the layer E4.
+*/
+const substrateHere = (layer, host = readHost(), env = process.env) => {
+	const runner = env["LAB_RUNNER"];
+	if (runner === void 0 || runner === "") return substrateOn(layer, host);
+	return substrateOn("E4", {
+		...JSON.parse(runner),
+		kernel: host.kernel,
+		hz: host.hz
+	});
+};
+/** This function reads the CPU time of this machine, or gives null without `/proc/stat`. `root` is for tests. */
+const readCpu = (root = "/") => {
+	const line = textOf(join(root, "proc", "stat"))?.split("\n")[0];
+	if (line === void 0 || !line.startsWith("cpu ")) return null;
+	const ticks = line.trim().split(/\s+/).slice(1, 9).map(Number);
+	const steal = ticks[7];
+	if (steal === void 0 || ticks.some(Number.isNaN)) return null;
+	return {
+		steal,
+		total: ticks.reduce((sum, tick) => sum + tick, 0)
+	};
+};
+/**
+* This function gives the share of the CPU time between two readings that the host took. The share
+* goes from zero to one. It is null if no tick passed. A guest sees only the host above it. On a hosted
+* runner that host is itself a guest, and this number does not show the contention of the layer below it.
+*/
+const stealShare = (before, after) => {
+	const total = after.total - before.total;
+	return total <= 0 ? null : (after.steal - before.steal) / total;
+};
+/**
 * This function reads the substrate of a record, the upgrade of a record from before this block.
 * Nothing rewrites such a record. It reads as the given layer. Its cloud is Azure if the record is
 * from before `AWS_FROM`, and unknown after that date, because its runner did not record the cloud.
@@ -2279,7 +3252,7 @@ const TAIL_LINES = 60;
 const OUTPUT_LINES = 2e3;
 /** This time is how long a write of a hop waits for the line that confirms it, in milliseconds. */
 const CONFIRM_MS = 1e4;
-const sleep$4 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const sleep$6 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** This function gives the arguments of the injector for the options of a fault. */
 const optionArgs = (options) => [
 	...options.direction === void 0 ? [] : ["--direction", options.direction],
@@ -2442,6 +3415,25 @@ const netnsOf = async (container) => {
 	if (inode === void 0) throw new Error(`not a network namespace: ${link}`);
 	return Number(inode);
 };
+/** This function gives the build record of a source as a part of a manifest: `{ build }`, or nothing for an image that the harness pulls. */
+const buildOf = (source) => {
+	const build = buildRecordOf(source);
+	return build === void 0 ? {} : { build };
+};
+/** This function gives the version of the harness and the build of its image, as a part of the harness record of a manifest. */
+const harnessRecord = () => ({
+	version: harnessVersion(),
+	...buildOf(shippedSource("harness"))
+});
+/**
+* This function gives the record of the client of a run that the harness ships, for a manifest:
+* its kind, its image, and its build.
+*/
+const shippedClientRecord = (kind, image) => ({
+	kind,
+	image,
+	...buildOf(SHIPPED_CLIENTS[kind].image)
+});
 /**
 * This function gives the image and the command of the client of each run, its kind for the
 * manifest, and the source of its image. A client by its name is a client that the harness ships
@@ -2492,7 +3484,9 @@ const openLab = async (target, kind = "tcp", topology) => {
 	const opened = [];
 	const networks = [];
 	const serverLines = lines();
+	let fabric;
 	const close = async () => {
+		await fabric?.close();
 		for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
 		for (const network of networks.reverse()) await network.stop().catch(() => void 0);
 	};
@@ -2633,7 +3627,18 @@ const openLab = async (target, kind = "tcp", topology) => {
 		opened.push(admin);
 		const injector = await new GenericContainer(image).withNetworkMode(`container:${server.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW").start();
 		opened.push(injector);
-		const clientIp = await ipOn(client, networks[0]);
+		if (topology?.fabric !== void 0) fabric = await openFabric({
+			fabric: topology.fabric,
+			image,
+			client,
+			server,
+			admin,
+			injector,
+			opened,
+			output
+		});
+		const device = fabric === void 0 ? "eth0" : FABRIC_DEVICE;
+		const clientIp = fabric === void 0 ? await ipOn(client, networks[0]) : FABRIC_CLIENT;
 		for (let at = 0; at < inChain.length; at += 1) {
 			const before = at === 0 ? clientIp : inChain[at - 1].ip.target;
 			inChain[at] = {
@@ -2716,7 +3721,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 		const begun = await records.until(["begin", "fail"], 3e4, "prober");
 		if (begun.ev === "fail") throw new Error(`the prober failed: ${String(begun["why"])}`);
 		const first = endpointAfter(hops, 0);
-		const targetIp = first === hops.length + 1 ? serverIp : inChain[first - 1].ip.client;
+		const targetIp = fabric !== void 0 ? FABRIC_TARGET : first === hops.length + 1 ? serverIp : inChain[first - 1].ip.client;
 		const targetPort = first === hops.length + 1 ? target.port : inChain[first - 1].hop.port;
 		const targetPeer = inChain.length === 0 ? clientIp : inChain.at(-1).ip.target;
 		const placed = inChain.length === 0 ? null : {
@@ -2769,8 +3774,8 @@ const openLab = async (target, kind = "tcp", topology) => {
 		};
 		/** This function gives the namespace and the device of netem at a place: an end, or the device of a hop toward a side. */
 		const deviceAt = (place) => {
-			if (place === "client") return [admin, "eth0"];
-			if (place === "target") return [injector, "eth0"];
+			if (place === "client") return [admin, device];
+			if (place === "target") return [injector, device];
 			const at = hopPlace(place);
 			const hop = hopNamed(at.hop);
 			return [hop.injector, hop.devices[at.toward]];
@@ -2789,6 +3794,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 		};
 		const proxies = inChain.filter((hop) => hop.hop.kind === "proxy");
 		return {
+			serverIp,
 			targetIp,
 			targetPort,
 			clientIp,
@@ -2797,6 +3803,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 			middlebox,
 			server: serverLines.seen,
 			kernel: records.seen,
+			...fabric === void 0 ? {} : { fabric },
 			async sysctl(values) {
 				const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
 				if (settings.length > 0) await output(admin, [
@@ -2852,7 +3859,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 				return stamped(where, [
 					"/opt/harness/inject",
 					kind,
-					peer,
+					options.peers?.join(",") ?? peer,
 					...optionArgs(options),
 					...via
 				]);
@@ -2862,7 +3869,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 				await output(where, [
 					"/opt/harness/inject",
 					LIFT[kind],
-					peer,
+					options.peers?.join(",") ?? peer,
 					...optionArgs(options),
 					...via
 				]);
@@ -2914,6 +3921,18 @@ const openLab = async (target, kind = "tcp", topology) => {
 				if (target.close === void 0) throw new Error(`${target.image} names no way to close its clients`);
 				await output(server, [...target.close]);
 			},
+			async exec(argv) {
+				return output(server, [...argv]);
+			},
+			async sibling(siblingImage, argv, ready) {
+				const started = await new GenericContainer(siblingImage).withNetwork(last).withWaitStrategy(Wait.forLogMessage(ready)).withCommand([...argv]).start();
+				opened.push(started);
+				return {
+					ip: await ipOn(started, last),
+					netns: await netnsOf(started),
+					exec: (command) => output(started, [...command])
+				};
+			},
 			async agent(agentImage, argv, options = {}) {
 				const Shared = shared(GenericContainer);
 				const seen = lines();
@@ -3054,7 +4073,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 						};
 						while (!confirmed()) {
 							if (Date.now() - started > CONFIRM_MS) throw new Error(`the hop ${event.hop} did not confirm the write of ${event.write.path}: no line "${confirm}" in ${CONFIRM_MS / 1e3} s`);
-							await sleep$4(50);
+							await sleep$6(50);
 						}
 					}
 					return stamp;
@@ -3146,7 +4165,8 @@ const openLab = async (target, kind = "tcp", topology) => {
 					})(),
 					boot_id: await output(client, ["cat", "/proc/sys/kernel/random/boot_id"]),
 					docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
-					substrate: substrateOn("E1"),
+					substrate: substrateHere(fabric === void 0 ? "E1" : "E3"),
+					...fabric === void 0 ? {} : { fabric: fabric.manifest() },
 					target: {
 						image: targetImage,
 						...target.subject === void 0 ? {} : {
@@ -3157,7 +4177,8 @@ const openLab = async (target, kind = "tcp", topology) => {
 						settings,
 						sysctls: targetSysctls,
 						...target.files === void 0 ? {} : { files: Object.fromEntries(Object.entries(target.files).map(([path, content]) => [path, createHash("sha256").update(content).digest("hex")])) },
-						...serverLines.seen[0] === void 0 ? {} : { first: serverLines.seen[0] }
+						...serverLines.seen[0] === void 0 ? {} : { first: serverLines.seen[0] },
+						...buildOf(targetSourceOf(target))
 					},
 					harness: {
 						image,
@@ -3166,12 +4187,16 @@ const openLab = async (target, kind = "tcp", topology) => {
 							"apk",
 							"info",
 							"-v"
-						])).split("\n").sort()
+						])).split("\n").sort(),
+						...harnessRecord()
 					},
 					client: {
 						kind: clientKind,
 						image: clientImage,
-						...source === void 0 ? {} : { source }
+						...source === void 0 ? {} : {
+							source,
+							...buildOf(source)
+						}
 					},
 					netns,
 					sysctls,
@@ -3186,46 +4211,316 @@ const openLab = async (target, kind = "tcp", topology) => {
 	}
 };
 //#endregion
-//#region src/group.ts
-/** This function gives the configuration file of a member: its port, its settings, and the primary that a replica follows. */
-const memberConfig = (group, primary) => [
-	`port ${group.port}`,
-	...Object.entries(group.memberSettings).map(([key, value]) => `${key} ${value}`),
-	...primary === null ? [] : [`replicaof ${primary} ${group.port}`],
-	""
-].join("\n");
-/** This function gives the configuration file of a Sentinel: its port, the group that it monitors, and the settings of the group. */
-const sentinelConfig = (group, primary) => [
-	`port ${group.sentinelPort}`,
-	`sentinel monitor ${group.name} ${primary} ${group.port} ${group.quorum}`,
-	...Object.entries(group.sentinelSettings).map(([key, value]) => `sentinel ${key} ${group.name} ${value}`),
-	""
-].join("\n");
+//#region src/standby.ts
 /**
-* This function gives the peers that a fault on a part cuts it from, by their addresses. `all` is
-* each other part and the client, `group` is the other parts, and `clients` is the client. `one`
-* and `one-way` are the first other member. The observer is not a peer, thus the timeline of the
-* group stays whole.
+* This module is the second server of a pair (`Cell.pair`). The lab of the cell has the primary, the
+* topology in front of it, the injector and the prober. This module adds one standby on the network
+* of the primary, and it does one run:
+*
+* 1. The client writes a stream of sequence numbers through the topology.
+* 2. The fault cuts the primary from the standby, or from each peer. The cut stays to the end.
+* 3. At the end of the hold, the injector stops each process of the primary with SIGKILL.
+* 4. The harness promotes the standby, and reads what it holds.
+* 5. The harness starts the old primary, and reads what its crash recovery gives.
+*
+* The standby has no way to the WAL of the old primary, because the primary stops under the cut. A
+* promotion cannot be undone, thus each run has its own environment (`Cell.fresh`).
 */
-const peersOf = (parts, client, part, cuts) => {
-	const others = [...Object.entries(parts.members), ...Object.entries(parts.sentinels)].filter(([name]) => name !== part).map(([, at]) => at.ip);
-	if (cuts === "clients") return [client];
-	if (cuts === "group") return others;
-	if (cuts === "one" || cuts === "one-way") return Object.entries(parts.members).filter(([name]) => name !== part).slice(0, 1).map(([, at]) => at.ip);
-	return [...others, client];
-};
-/** This function gives the direction of the rules of a cut: `one-way` drops only the segments from the part. */
-const directionOf = (cuts) => cuts === "one-way" ? ["--direction", "out"] : [];
-/** This function reads the flat reply of a command line client into its fields: name, value, name, value. */
-const fieldsOf = (text) => {
-	const out = [];
-	const words = text.split("\n").map((line) => line.replace(/\r$/, ""));
-	let current = null;
-	for (let at = 0; at + 1 < words.length; at += 2) {
-		const [key, value] = [words[at], words[at + 1]];
-		if (key === "name" || current === null) {
-			current = {};
-			out.push(current);
+/** This table holds one row for each write of the client. */
+const TABLE = "writes";
+/** This time bounds each wait of the setup of the pair, in ms. */
+const SETUP_MS = 6e4;
+/** This count is the acknowledged writes before the fault: the stream is steady. */
+const STEADY$4 = 20;
+/** This time is the wait after the stop of the primary, in ms: each writer sends its last write in it. */
+const LAST_WRITE_MS = 2e3;
+/** This time bounds the wait for the last line of the client, in ms. */
+const STOP_MS = 5e3;
+/** This text is the line of the log of a standby that serves reads (src/backend/postmaster/postmaster.c:3740). */
+const STANDBY_READY = "database system is ready to accept read-only connections";
+/** This text is the write of the client, with its sequence number. */
+const INSERT = `INSERT INTO ${TABLE} VALUES ($n)`;
+const sleep$5 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/** This function gives the argv of one statement in psql, on the server in the same container. */
+const psql = (statement) => [
+	"psql",
+	"-h",
+	"127.0.0.1",
+	"-U",
+	"postgres",
+	"-At",
+	"-c",
+	statement
+];
+/** This function gives the command of a standby: a base backup of the primary, then the server in recovery. */
+const standbyCommand = (primary) => [
+	"sh",
+	"-c",
+	`chown postgres:postgres "$PGDATA"; chmod 0700 "$PGDATA"; until gosu postgres pg_basebackup -h ${primary} -U postgres -D "$PGDATA" -X stream -c fast -R; do gosu postgres sh -c 'rm -rf "$PGDATA"/*'; sleep 0.5; done; exec gosu postgres postgres`
+];
+/** This command starts the old primary again, after the stop of each of its processes. Its crash recovery runs before the wait ends. */
+const RESTART = [
+	"sh",
+	"-c",
+	"exec gosu postgres pg_ctl -D \"$PGDATA\" -l /tmp/restart.log -w -t 60 start"
+];
+/** This function reads the output of a query for sequence numbers into a sorted list. */
+const numbersOf = (text) => text.split("\n").map((line) => line.trim()).filter((line) => line !== "").map(Number).sort((a, b) => a - b);
+/** This function gives the arguments of the client of a run. */
+const pairArgs = (cell, lab) => {
+	const pair = cell.pair;
+	const hold = cell.hold === void 0 ? 0 : valueIn(cell.hold, "s");
+	return [
+		lab.targetIp,
+		String(lab.targetPort),
+		"--mode",
+		"stream",
+		"--interval-ms",
+		String(valueIn(pair.interval, "ms")),
+		"--writers",
+		String(pair.writers),
+		"--query",
+		INSERT,
+		"--max-s",
+		String(Math.ceil(hold) + 60),
+		...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
+	];
+};
+/** This function gives the peers of the cut of a pair: the standby, or the standby and the peer of the primary in the topology. */
+const peersOf$1 = (pair, standby, lab) => pair.cuts === "standby" ? [standby] : [standby, lab.topology?.targetPeer ?? lab.clientIp];
+/** This function gives the instants of the cancels of a hold, in ns from the fault: equal parts of the hold. */
+const cancelsOf = (pair, holdNs) => {
+	const count = pair.cancels ?? 0;
+	return Array.from({ length: count }, (_, at) => (at + 1) * holdNs / (count + 1));
+};
+/** This function asks until the primary reports the standby in a state of replication. */
+const untilStreaming = async (lab, state) => {
+	const started = Date.now();
+	for (;;) {
+		const seen = await lab.exec(psql("SELECT sync_state FROM pg_stat_replication WHERE state = 'streaming'"));
+		if (seen === state) return;
+		if (Date.now() - started > SETUP_MS) throw new Error(`the standby did not stream as ${state} in ${SETUP_MS} ms: it reports "${seen}"`);
+		await sleep$5(100);
+	}
+};
+/**
+* This function opens the pair of an environment. The table comes first and the standby next: the
+* base backup has it. A synchronous standby is named last, thus no setup step waits for it.
+*/
+const openStandby = async (lab, cell) => {
+	const pair = cell.pair;
+	await lab.exec([
+		"sh",
+		"-c",
+		"echo \"host replication all all trust\" >> \"$PGDATA/pg_hba.conf\""
+	]);
+	await lab.exec(psql("SELECT pg_reload_conf()"));
+	await lab.exec(psql(`CREATE TABLE ${TABLE} (n integer PRIMARY KEY)`));
+	const standby = await lab.sibling(await targetImageOf(cell.target), standbyCommand(lab.serverIp), STANDBY_READY);
+	await untilStreaming(lab, "async");
+	if (pair.synchronous) {
+		await lab.exec(psql("ALTER SYSTEM SET synchronous_standby_names = '*'"));
+		await lab.exec(psql("SELECT pg_reload_conf()"));
+		await untilStreaming(lab, "sync");
+	}
+	return standby;
+};
+/** This function waits for a count of acknowledged writes. */
+const untilSteady = async (probe) => {
+	const started = Date.now();
+	while (probe.events.filter((event) => event.ev === "ack").length < STEADY$4) {
+		if (Date.now() - started > SETUP_MS) throw new Error(`the client did not write ${STEADY$4} times in ${SETUP_MS} ms`);
+		await sleep$5(10);
+	}
+};
+/** This function gives the count of lost records in the latest clock line of the prober. */
+const droppedSoFar$5 = (records) => {
+	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
+	return 0;
+};
+/**
+* This function does one run of a pair. A dry run writes and stops, with no fault: the warm-up of the
+* environment. It removes its rows, thus the numbers of the run start on an empty table.
+*/
+const pairOnce = async (lab, standby, cell, environment, index, dry = false) => {
+	const pair = cell.pair;
+	await lab.forget();
+	const from = lab.kernel.length;
+	const lostBefore = droppedSoFar$5(lab.kernel);
+	const probe = await lab.probe(pairArgs(cell, lab));
+	const holdNs = cell.hold === void 0 ? 0 : valueIn(cell.hold, "s") * 1e9;
+	let fault = null;
+	let heal = null;
+	try {
+		await probe.until(["ready"], 15e3);
+		await untilSteady(probe);
+		if (!dry) {
+			fault = await lab.fault("blackhole", "target", { peers: peersOf$1(pair, standby.ip, lab) });
+			const started = Date.now();
+			for (const at of cancelsOf(pair, holdNs)) {
+				await sleep$5(Math.max(0, at / 1e6 - (Date.now() - started)));
+				await lab.exec(psql("SELECT count(pg_cancel_backend(pid)) FROM pg_stat_activity WHERE wait_event = 'SyncRep'"));
+			}
+			heal = await lab.signal("KILL", fault.t0 + holdNs);
+			await sleep$5(LAST_WRITE_MS);
+			if (cell.target.kill !== void 0) await lab.exec(cell.target.kill);
+		}
+		await probe.go().catch(() => void 0);
+		await probe.until(["done"], STOP_MS);
+	} finally {
+		await probe.stop();
+	}
+	if (dry) await lab.exec(psql(`TRUNCATE ${TABLE}`));
+	const settled = await lab.settle();
+	const lost = settled === null ? null : settled - lostBefore;
+	const read = `SELECT n FROM ${TABLE} ORDER BY n`;
+	let holds = null;
+	let oldHolds = [];
+	if (!dry) {
+		await standby.exec(psql("SELECT pg_promote(true, 60)"));
+		holds = numbersOf(await standby.exec(psql(read)));
+		await lab.exec([...RESTART]);
+		oldHolds = numbersOf(await lab.exec(psql(read)));
+	}
+	return {
+		environment,
+		index,
+		events: [...probe.events],
+		fault,
+		...heal === null ? {} : { heal },
+		kernel: lab.kernel.slice(from),
+		netns: lab.netns.client,
+		targetNetns: lab.netns.target,
+		lost,
+		...lab.topology === null ? {} : {
+			topology: lab.topology,
+			middlebox: []
+		},
+		group: {
+			members: {
+				primary: {
+					ip: lab.serverIp,
+					netns: lab.netns.target
+				},
+				standby: {
+					ip: standby.ip,
+					netns: standby.netns
+				}
+			},
+			sentinels: {},
+			primary: "primary",
+			at: "primary",
+			timeline: [],
+			after: dry ? "primary" : "standby",
+			holds,
+			oldHolds
+		}
+	};
+};
+//#endregion
+//#region src/pins.ts
+/**
+* This module tells if a manifest pins each program that a run operated. A record that does not pin
+* them is not a record. A later run cannot operate the same programs, thus nobody can reproduce
+* its numbers (SYS-101). Each image is pulled by its digest, or the harness builds it. A build
+* records its bases by digest, the sha256 of each lockfile, and the versions of the packages that
+* its manifests name. The commit of the code that operated is a field of the record of a cell, and
+* `commitProblems` examines it.
+*/
+/** This pattern matches the id of an image: `sha256:` and 64 hex digits. */
+const IMAGE_ID = /^sha256:[0-9a-f]{64}$/;
+/** This pattern matches the tag of a build of the harness: its name, then 12 hex digits of the sha256 of its context. */
+const BUILD_TAG = /^[a-z0-9][a-z0-9._\-/]*:[0-9a-f]{12}$/;
+/** This pattern matches a commit of git: 40 hex digits. */
+const COMMIT = /^[0-9a-f]{40}$/;
+/** This function gives each problem of the record of a build. A problem is a base that no digest pins, or a tag of another form. */
+const buildProblems = (what, build) => [...BUILD_TAG.test(build.tag) ? [] : [`${what}: the build has the tag ${build.tag}, which is not a tag of a build`], ...build.bases.filter((base) => !PINNED.test(base)).map((base) => `${what}: the base ${base} is not pinned by digest`)];
+/**
+* This function gives each problem of an image of a manifest. A pulled image needs a digest. An
+* image that the harness builds needs its build record when the manifest can hold one (`build`).
+* Otherwise, it needs the tag of a build and the id of its image.
+*/
+const pinProblems = (what, image, id, build, needsBuild) => {
+	if (PINNED.test(image)) return [];
+	if (build !== void 0) return buildProblems(what, build);
+	if (BUILD_TAG.test(image) && !needsBuild && id !== void 0 && IMAGE_ID.test(id)) return [];
+	return BUILD_TAG.test(image) && needsBuild ? [`${what}: the image ${image} is a build with no build record`] : [`${what}: the image ${image} has no digest`];
+};
+/**
+* This function gives each problem of a manifest that stops it from pinning the programs of its
+* run. An empty list means that the manifest pins each program.
+* - the target: an image by its digest, or the build of a subject with its build record and its base
+*   by digest.
+* - the harness: the version of its package, and the build record of its image.
+* - the client: an image by its digest, or a build with its build record.
+* - each hop: an image by its digest, or the tag of a build and the id of its image.
+*/
+const manifestProblems = (manifest) => [
+	...pinProblems("the target", manifest.target.image, manifest.target.id, manifest.target.build, manifest.target.subject !== void 0),
+	...manifest.target.subject !== void 0 && (manifest.target.base === void 0 || !PINNED.test(manifest.target.base)) ? [`the target: the base ${manifest.target.base ?? "(none)"} of its subject is not pinned by digest`] : [],
+	...manifest.harness.version === void 0 ? ["the harness: the manifest has no version of the harness"] : [],
+	...manifest.harness.build === void 0 ? ["the harness: the image has no build record"] : buildProblems("the harness", manifest.harness.build),
+	...pinProblems("the client", manifest.client.image, void 0, manifest.client.build, true),
+	...(manifest.hops ?? []).flatMap((hop) => pinProblems(`the hop ${hop.name}`, hop.image, hop.id, void 0, false))
+];
+/** This function gives the problem of the commit of a record: no commit, or a text that is not a commit of git. */
+const commitProblems = (commit) => commit === void 0 ? ["the record has no commit of the code that operated"] : COMMIT.test(commit) ? [] : [`the record has the commit ${commit}, which is not 40 hex digits`];
+/** This problem is the problem of a checkout with changes: its commit does not hold the code that operated. */
+const DIRTY = "the checkout has changes to tracked files that no commit holds";
+/** This problem is the problem of a record of a local debug run. */
+const DEBUG = "the record is of a local debug run, and no page reads it";
+/**
+* This function gives each problem that stops a record from pinning the programs of its runs. A
+* record with a problem is not a record (SYS-101). The writer of the cells refuses it, and the read
+* of a schedule refuses it. A commit with changes to tracked files is not a pin.
+*/
+const recordProblems = (record) => [.../* @__PURE__ */ new Set([
+	...commitProblems(record.code?.commit),
+	...record.code?.dirty === true ? [DIRTY] : [],
+	...record.manifests.flatMap((manifest) => manifestProblems(manifest)),
+	...record.debug === true ? [DEBUG] : []
+])];
+//#endregion
+//#region src/group.ts
+/** This function gives the configuration file of a member: its port, its settings, and the primary that a replica follows. */
+const memberConfig = (group, primary) => [
+	`port ${group.port}`,
+	...Object.entries(group.memberSettings).map(([key, value]) => `${key} ${value}`),
+	...primary === null ? [] : [`replicaof ${primary} ${group.port}`],
+	""
+].join("\n");
+/** This function gives the configuration file of a Sentinel: its port, the group that it monitors, and the settings of the group. */
+const sentinelConfig = (group, primary) => [
+	`port ${group.sentinelPort}`,
+	`sentinel monitor ${group.name} ${primary} ${group.port} ${group.quorum}`,
+	...Object.entries(group.sentinelSettings).map(([key, value]) => `sentinel ${key} ${group.name} ${value}`),
+	""
+].join("\n");
+/**
+* This function gives the peers that a fault on a part cuts it from, by their addresses. `all` is
+* each other part and the client, `group` is the other parts, and `clients` and `to-clients` are
+* the client. `one` and `one-way` are the first other member. The observer is not a peer, thus the timeline of the
+* group stays whole.
+*/
+const peersOf = (parts, client, part, cuts) => {
+	const others = [...Object.entries(parts.members), ...Object.entries(parts.sentinels)].filter(([name]) => name !== part).map(([, at]) => at.ip);
+	if (cuts === "clients" || cuts === "to-clients") return [client];
+	if (cuts === "group") return others;
+	if (cuts === "one" || cuts === "one-way") return Object.entries(parts.members).filter(([name]) => name !== part).slice(0, 1).map(([, at]) => at.ip);
+	return [...others, client];
+};
+/** This function gives the direction of the rules of a cut: `one-way` and `to-clients` drop only the segments from the part. */
+const directionOf = (cuts) => cuts === "one-way" || cuts === "to-clients" ? ["--direction", "out"] : [];
+/** This function reads the flat reply of a command line client into its fields: name, value, name, value. */
+const fieldsOf = (text) => {
+	const out = [];
+	const words = text.split("\n").map((line) => line.replace(/\r$/, ""));
+	let current = null;
+	for (let at = 0; at + 1 < words.length; at += 2) {
+		const [key, value] = [words[at], words[at + 1]];
+		if (key === "name" || current === null) {
+			current = {};
+			out.push(current);
 		}
 		current[key] = value;
 	}
@@ -3308,7 +4603,7 @@ const viewOf = (group, ips, sentinels, members) => {
 		why: "healthy"
 	};
 };
-const sleep$3 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const sleep$4 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** This time is the longest that a group takes to converge before a run, or after it. */
 const CONVERGE_MS = 24e4;
 /**
@@ -3503,7 +4798,7 @@ const openGroupLab = async (group, options = {}) => {
 					}));
 					calm = last.healthy ? calm + 1 : 0;
 					if (calm >= 2 && last.primary !== null) return last.primary;
-					await sleep$3(1e3);
+					await sleep$4(1e3);
 				}
 				throw new Error(`the group did not converge in ${within} ms: ${last.why}`);
 			},
@@ -3524,11 +4819,16 @@ const openGroupLab = async (group, options = {}) => {
 				]);
 			},
 			async holds(primary) {
-				return (await cli(primary, [
+				return holdsOf(await cli(primary, [
 					"--scan",
 					"--pattern",
 					"w:*"
-				])).split("\n").map((key) => Number(key.trim().slice(2))).filter((n) => Number.isInteger(n) && n > 0).sort((a, b) => a - b);
+				]), await cli(primary, [
+					"LRANGE",
+					"w",
+					"0",
+					"-1"
+				]));
 			},
 			partOf: roles,
 			async client(args) {
@@ -3583,7 +4883,7 @@ const openGroupLab = async (group, options = {}) => {
 				const started = Date.now();
 				while (await cli(member, ["PING"]).catch(() => "") !== "PONG") {
 					if (Date.now() - started > 3e4) throw new Error(`${member} did not start again`);
-					await sleep$3(200);
+					await sleep$4(200);
 				}
 			},
 			failover(coordinated) {
@@ -3675,12 +4975,10 @@ const openGroupLab = async (group, options = {}) => {
 							"apk",
 							"info",
 							"-v"
-						])).split("\n").sort()
-					},
-					client: {
-						kind: "ioredis",
-						image: node
+						])).split("\n").sort(),
+						...harnessRecord()
 					},
+					client: shippedClientRecord("ioredis", node),
 					netns: {
 						client: clientNetns,
 						target: parts.members[first].netns
@@ -3701,7 +4999,7 @@ const openGroupLab = async (group, options = {}) => {
 	}
 };
 /** This count is the number of accepted writes that a run waits for before its fault: the stream is steady then. */
-const STEADY$2 = 20;
+const STEADY$3 = 20;
 /** This time is how long a run continues after its outcome, so that the record shows what came next. */
 const TAIL_MS$3 = 2e3;
 /** This function gives the address of a member, with the port of the group. */
@@ -3711,22 +5009,32 @@ const deadlineOf = (cell) => {
 	const { prediction } = cell;
 	return (prediction.detects === true ? valueIn(prediction.high, "s") : prediction.detects === false && prediction.recovers !== void 0 ? valueIn(prediction.recovers.high, "s") : valueIn(prediction.window, "s")) * 1e3 + 1e4;
 };
+/**
+* This function gives the sequence numbers that a group holds, in order, from the output of its
+* command line. They are the keys `w:<n>` of a stream of SET, and the entries of the list `w` of a
+* stream of RPUSH. A list keeps each copy, thus a number can repeat.
+*/
+const holdsOf = (keys, list) => [...keys.split("\n").map((key) => Number(key.trim().slice(2))), ...list.split("\n").map((entry) => Number(entry.trim()))].filter((n) => Number.isInteger(n) && n > 0).sort((a, b) => a - b);
 /** This function gives the arguments of the client of a group cell. */
 const clientArgs = (cell, lab) => {
+	const sentinels = cell.group.sentinels.map((name) => `${lab.sentinels[name].ip}:${cell.group.sentinelPort}`);
+	const driver = cell.probe.driver?.name === "ioredis" ? cell.probe.driver : void 0;
 	return [
 		"--sentinels",
-		cell.group.sentinels.map((name) => `${lab.sentinels[name].ip}:${cell.group.sentinelPort}`).join(","),
+		sentinels.join(","),
 		"--name",
 		cell.group.name,
 		"--interval-ms",
 		String(cell.probe.interval === void 0 ? 50 : valueIn(cell.probe.interval, "ms")),
 		"--max-s",
 		String(Math.ceil(deadlineOf(cell) / 1e3 + 120)),
-		...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
+		...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)],
+		...driver?.op === void 0 ? [] : ["--op", driver.op],
+		...driver?.window === void 0 ? [] : ["--window", String(driver.window)]
 	];
 };
 /** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
-const waitFor$2 = (client, match, within) => new Promise((resolve) => {
+const waitFor$3 = (client, match, within) => new Promise((resolve) => {
 	const started = Date.now();
 	const look = () => {
 		const found = client.events.find(match);
@@ -3742,7 +5050,7 @@ const subscriptionsOf = (timeline) => {
 	return [...latest.values()];
 };
 /** This function gives the count of lost records in the latest clock line of the prober. */
-const droppedSoFar$3 = (records) => {
+const droppedSoFar$4 = (records) => {
 	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
 	return 0;
 };
@@ -3764,7 +5072,7 @@ const groupOnce = async (lab, cell, environment, index) => {
 	await lab.forget();
 	const from = lab.kernel.length;
 	const timelineFrom = lab.timeline.length;
-	const lostBefore = droppedSoFar$3(lab.kernel);
+	const lostBefore = droppedSoFar$4(lab.kernel);
 	const client = await lab.client(clientArgs(cell, lab));
 	const lands = cell.prediction.group?.lands ?? "new";
 	const place = cell.faultAt ?? { role: "primary" };
@@ -3774,7 +5082,7 @@ const groupOnce = async (lab, cell, environment, index) => {
 	const undo = [];
 	try {
 		await client.until(["ready"], 6e4);
-		if (await waitFor$2(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$2, 3e4) === void 0) throw new Error(`the client did not write ${STEADY$2} times in 30 s`);
+		if (await waitFor$3(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$3, 3e4) === void 0) throw new Error(`the client did not write ${STEADY$3} times in 30 s`);
 		let healing = null;
 		switch (cell.fault) {
 			case "blackhole":
@@ -3788,7 +5096,7 @@ const groupOnce = async (lab, cell, environment, index) => {
 				undo.push(() => lab.lift(kind, part, cuts));
 				if (cell.stall !== void 0) {
 					const t1 = fault.t1;
-					const sent = await waitFor$2(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
+					const sent = await waitFor$3(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
 					if (sent !== void 0) healing = lab.lift(kind, part, cuts, sent.t + valueIn(cell.stall, "s") * 1e9);
 				} else if (cell.hold !== void 0) healing = lab.lift(kind, part, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
 				break;
@@ -3822,11 +5130,11 @@ const groupOnce = async (lab, cell, environment, index) => {
 		const t0 = fault.t0;
 		const old = addressOf(lab, primary);
 		const deadline = deadlineOf(cell);
-		if ((lands === "new" ? await waitFor$2(client, (event) => event.ev === "ack" && event.t > t0 && typeof event["at"] === "string" && event["at"] !== old, deadline) : await (async () => {
+		if ((lands === "new" ? await waitFor$3(client, (event) => event.ev === "ack" && event.t > t0 && typeof event["at"] === "string" && event["at"] !== old, deadline) : await (async () => {
 			await healed;
 			const after = heal?.t0 ?? t0;
-			return waitFor$2(client, (event) => event.ev === "ack" && event.t > after, deadline);
-		})()) !== void 0) await sleep$3(TAIL_MS$3);
+			return waitFor$3(client, (event) => event.ev === "ack" && event.t > after, deadline);
+		})()) !== void 0) await sleep$4(TAIL_MS$3);
 		await healed;
 	} finally {
 		for (const step of undo.reverse()) await step().catch(() => void 0);
@@ -3919,6 +5227,10 @@ const cutOf = (set, members, client, part, cuts) => {
 		peers: others.slice(0, 1),
 		direction: "out"
 	};
+	if (cuts === "to-clients") return {
+		peers: [client],
+		direction: "out"
+	};
 	return { peers: [...others, client] };
 };
 /** This function gives the command of the server of each member. The server is the child of a shell, thus a fin can stop it and the container stays up. */
@@ -3989,7 +5301,8 @@ const statusViewOf = (set, ips, status) => {
 		why: "healthy"
 	};
 };
-const sleep$2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/** This function waits for a time, in ms. */
+const sleep$3 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** These are the arguments of replSetStepDown: the time that the old primary cannot stand again, and the wait for a secondary to catch up. */
 const STEP_DOWN = {
 	seconds: 60,
@@ -4004,8 +5317,9 @@ const STEP_DOWN = {
 * - an injector sidecar for each member, in its network and process namespaces.
 * - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
 * - the prober, which watches the namespace of the client.
+* - a backup host for a recovery drill: the stock image, for its tools, with no server.
 */
-const openReplicaSetLab = async (set) => {
+const openReplicaSetLab = async (set, backups = false) => {
 	const image = await harnessImage();
 	const mongo = await mongoClientImage();
 	const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();
@@ -4064,6 +5378,12 @@ const openReplicaSetLab = async (set) => {
 			"while :; do sleep 3600; done"
 		]).start();
 		opened.push(commander);
+		const backupHost = backups ? await new GenericContainer(set.image).withNetwork(network).withCommand([
+			"sh",
+			"-c",
+			"while :; do sleep 3600; done"
+		]).start() : null;
+		if (backupHost !== null) opened.push(backupHost);
 		const clientNetns = await netnsOf(admin);
 		const records = lines();
 		const prober = await new GenericContainer(image).withPrivilegedMode().withNetworkMode("none").withCommand(["/opt/harness/prober", String(clientNetns)]).withLogConsumer(records.consume).start();
@@ -4081,11 +5401,11 @@ const openReplicaSetLab = async (set) => {
 			return found;
 		};
 		/** This function starts a script of the shell on a member, against its own server only. */
-		const shell = (name, script) => output(box(name), [
+		const shell = (name, script, port = set.port) => output(box(name), [
 			set.programs.shell,
 			"--quiet",
 			"--norc",
-			`mongodb://127.0.0.1:${set.port}/?directConnection=true&serverSelectionTimeoutMS=3000`,
+			`mongodb://127.0.0.1:${port}/?directConnection=true&serverSelectionTimeoutMS=3000`,
 			"--eval",
 			script
 		]);
@@ -4130,7 +5450,7 @@ const openReplicaSetLab = async (set) => {
 				calm = last.healthy ? calm + 1 : 0;
 				if (last.primary !== null) known = last.primary;
 				if (calm >= 2 && last.primary !== null) return last.primary;
-				await sleep$2(1e3);
+				await sleep$3(1e3);
 			}
 			throw new Error(`the set did not converge in ${within} ms: ${last.why}`);
 		};
@@ -4236,9 +5556,17 @@ const openReplicaSetLab = async (set) => {
 				const started = Date.now();
 				while (!(await shell(member, "print(db.hello().ok)").catch(() => "")).includes("1")) {
 					if (Date.now() - started > 6e4) throw new Error(`${member} did not start again`);
-					await sleep$2(500);
+					await sleep$3(500);
 				}
 			},
+			exec(member, argv) {
+				return output(box(member), argv);
+			},
+			script: shell,
+			vault(argv) {
+				if (backupHost === null) throw new Error("this lab has no backup host");
+				return output(backupHost, argv);
+			},
 			stepDown(primary) {
 				return stamped(commander, [
 					"node",
@@ -4291,7 +5619,7 @@ const openReplicaSetLab = async (set) => {
 					architecture: await output(holder, ["uname", "-m"]),
 					hz: /^CONFIG_HZ=(\d+)$/m.exec(config)?.[1] ?? null,
 					boot_id: await output(holder, ["cat", "/proc/sys/kernel/random/boot_id"]),
-					substrate: substrateOn("E1"),
+					substrate: substrateHere("E1"),
 					docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
 					target: {
 						image: set.image,
@@ -4306,12 +5634,10 @@ const openReplicaSetLab = async (set) => {
 							"apk",
 							"info",
 							"-v"
-						])).split("\n").sort()
-					},
-					client: {
-						kind: "mongodb",
-						image: mongo
+						])).split("\n").sort(),
+						...harnessRecord()
 					},
+					client: shippedClientRecord("mongodb", mongo),
 					netns: {
 						client: clientNetns,
 						target: members[set.members[0]].netns
@@ -4331,10 +5657,10 @@ const openReplicaSetLab = async (set) => {
 		throw error;
 	}
 };
-/** This count is the number of accepted writes that a run waits for before its fault: the stream is steady then. */
-const STEADY$1 = 20;
 /** This time is how long a run continues after its outcome, so that the record shows what came next. */
 const TAIL_MS$2 = 2e3;
+/** This function gives how long a run of a cell continues after its lift, in ms: the tail, and the watch of the cell. */
+const afterLiftOf = (cell) => TAIL_MS$2 + (cell.watch === void 0 ? 0 : valueIn(cell.watch, "s") * 1e3);
 /** This function gives the arguments of the client of a replica set cell: each member, and the stream of writes. */
 const replicaSetArgs = (cell, members) => {
 	const set = cell.replicaSet;
@@ -4351,14 +5677,14 @@ const replicaSetArgs = (cell, members) => {
 		"--interval-ms",
 		String(cell.probe.interval === void 0 ? 50 : valueIn(cell.probe.interval, "ms")),
 		"--max-s",
-		String(Math.ceil(deadlineOf(cell) / 1e3 + (cell.hold === void 0 ? 0 : valueIn(cell.hold, "s")) + 120)),
+		String(Math.ceil(deadlineOf(cell) / 1e3 + (cell.hold === void 0 ? 0 : valueIn(cell.hold, "s")) + (cell.drill === void 0 ? 0 : drillSpan(cell.drill)) + 120)),
 		"--w",
 		String(cell.writeConcern ?? "majority"),
 		...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
 	];
 };
 /** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
-const waitFor$1 = (client, match, within) => new Promise((resolve) => {
+const waitFor$2 = (client, match, within) => new Promise((resolve) => {
 	const started = Date.now();
 	const look = () => {
 		const found = client.events.find(match);
@@ -4368,7 +5694,7 @@ const waitFor$1 = (client, match, within) => new Promise((resolve) => {
 	look();
 });
 /** This function gives the count of lost records in the latest clock line of the prober. */
-const droppedSoFar$2 = (records) => {
+const droppedSoFar$3 = (records) => {
 	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
 	return 0;
 };
@@ -4393,7 +5719,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 	await lab.forget();
 	const from = lab.kernel.length;
 	const logFrom = lab.log.length;
-	const lostBefore = droppedSoFar$2(lab.kernel);
+	const lostBefore = droppedSoFar$3(lab.kernel);
 	const client = await lab.client(replicaSetArgs(cell, lab.members));
 	const lands = cell.prediction.group?.lands ?? "new";
 	const place = cell.faultAt ?? { role: "primary" };
@@ -4403,7 +5729,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 	const undo = [];
 	try {
 		await client.until(["ready"], 6e4);
-		if (await waitFor$1(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$1, 3e4) === void 0) throw new Error(`the client did not write ${STEADY$1} times in 30 s`);
+		if (await waitFor$2(client, (event) => event.ev === "ack" && Number(event["n"]) >= 20, 3e4) === void 0) throw new Error(`the client did not write 20 times in 30 s`);
 		let healing = null;
 		switch (cell.fault) {
 			case "blackhole":
@@ -4417,7 +5743,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 				undo.push(() => lab.lift(kind, part, cuts));
 				if (cell.stall !== void 0) {
 					const t1 = fault.t1;
-					const sent = await waitFor$1(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
+					const sent = await waitFor$2(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
 					healing = lab.lift(kind, part, cuts, (sent?.t ?? t1) + valueIn(cell.stall, "s") * 1e9);
 				} else if (cell.hold !== void 0) healing = lab.lift(kind, part, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
 				break;
@@ -4452,10 +5778,10 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 		const t1 = fault.t1;
 		const old = `${lab.members[primary].ip}:${set.port}`;
 		const sentAt = (n) => client.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
-		if (await waitFor$1(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1 && (lands === "same" || typeof event["at"] === "string" && event["at"] !== old), deadlineOf(cell)) !== void 0) await sleep$2(TAIL_MS$2);
+		if (await waitFor$2(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1 && (lands === "same" || typeof event["at"] === "string" && event["at"] !== old), deadlineOf(cell)) !== void 0) await sleep$3(TAIL_MS$2);
 		if (healing !== null) {
 			await healed.catch(() => void 0);
-			await sleep$2(TAIL_MS$2);
+			await sleep$3(afterLiftOf(cell));
 		}
 	} finally {
 		for (const step of undo.reverse()) await step().catch(() => void 0);
@@ -4487,6 +5813,296 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 	};
 };
 //#endregion
+//#region src/drill.ts
+/**
+* This module does one run of a recovery drill on the lab of a replica set. A drill stops the store
+* during writes, removes its disk, and recovers from real backups, or by an election.
+*
+* A backup is a real `mongodump`, on a host of its own. The dump has the oplog of its own time, and
+* each slice has the oplog after the backup before it. The fault comes at the age of the newest
+* backup, on the clock of the VM. Thus a run measures the worst case of its interval by construction.
+*
+* A restore cannot have a client of the set: the client writes to any primary. So the restore writes
+* to a server with no set, on its own port. That server then stops, each member starts with an empty
+* set, and the old primary starts the set. Only that member has the data, thus it wins.
+*
+* The driver refuses a primary with an election id under the highest one that it saw, and a new
+* set starts its ids again. So a restore has two clients. The first one writes up to the fault, and
+* the second one starts after the restore, with the sequence numbers that follow the first.
+*/
+/** This folder is the store of the backups, on the backup host. */
+const VAULT = "/backup";
+/** This time is the longest wait for a server to answer, in ms. */
+const STARTS_WITHIN = 6e4;
+/** This function waits until a check holds, or it stops the run after `within` ms. */
+const until = async (what, check, within) => {
+	const started = Date.now();
+	while (!await check().catch(() => false)) {
+		if (Date.now() - started > within) throw new Error(`${what} did not come in ${within} ms`);
+		await sleep$3(500);
+	}
+};
+/** This function gives the time that the client sent a write. Zero is no write. */
+const writeOf = (events, n) => {
+	const sent = n === 0 ? void 0 : events.find((event) => event.ev === "sent" && Number(event["n"]) === n);
+	return sent === void 0 ? null : {
+		n,
+		t: sent.t
+	};
+};
+/** This function gives the size of a folder of the backup host. */
+const sizeOf = async (lab, path) => {
+	const [bytes] = (await lab.vault([
+		"du",
+		"-sb",
+		path
+	])).split(/\s+/);
+	return {
+		value: Number(bytes),
+		unit: "B"
+	};
+};
+/** This function takes the dump with its oplog, and each slice of the oplog at its interval. It gives each backup in sequence. */
+const backupsOf = async (lab, drill, primary, writer) => {
+	const host = `${lab.members[primary].ip}:${lab.set.port}`;
+	const started = Date.now();
+	await lab.vault([
+		"mongodump",
+		"--host",
+		host,
+		"--oplog",
+		"--out",
+		`${VAULT}/dump-0`
+	]);
+	const ended = Date.now();
+	const oplog = readOplog(await lab.vault([
+		"bsondump",
+		"--quiet",
+		`${VAULT}/dump-0/oplog.bson`
+	]));
+	const held = highestHeld(await lab.vault([
+		"bsondump",
+		"--quiet",
+		`${VAULT}/dump-0/cells/w.bson`
+	]));
+	if (oplog.last === null) throw new Error("the oplog of the dump has no entry");
+	let highest = Math.max(oplog.highest ?? 0, held ?? 0);
+	let from = oplog.last;
+	const backups = [{
+		kind: "dump",
+		started,
+		ended,
+		last: writeOf(writer.events, highest),
+		size: await sizeOf(lab, `${VAULT}/dump-0`)
+	}];
+	const every = valueIn(drill.sliceEvery, "s") * 1e3;
+	for (let slice = 1; slice <= drill.slices; slice += 1) {
+		await sleep$3(Math.max(0, ended + slice * every - Date.now()));
+		const out = `${VAULT}/slice-${slice}`;
+		const begun = Date.now();
+		await lab.vault([
+			"mongodump",
+			"--host",
+			host,
+			"--db",
+			"local",
+			"--collection",
+			"oplog.rs",
+			"--query",
+			sliceQuery(from),
+			"--out",
+			out
+		]);
+		const finished = Date.now();
+		const read = readOplog(await lab.vault([
+			"bsondump",
+			"--quiet",
+			`${out}/local/oplog.rs.bson`
+		]));
+		highest = Math.max(highest, read.highest ?? 0);
+		from = read.last ?? from;
+		backups.push({
+			kind: "slice",
+			started: begun,
+			ended: finished,
+			last: writeOf(writer.events, highest),
+			size: await sizeOf(lab, out)
+		});
+	}
+	return backups;
+};
+/**
+* This function stops the server of each member that the drill loses, and it removes each disk. A
+* restore loses each member, and an election loses the primary. The fault comes at the age of the
+* drill after the last write of the newest backup. A control has no later write, and an election
+* has no backup: their age counts from this point. The stamp is the one of the primary.
+*/
+const lose = async (lab, drill, primary, last) => {
+	const parts = drill.recovery === "restore" ? [primary, ...lab.set.members.filter((name) => name !== primary)] : [primary];
+	const age = valueIn(drill.age, "s");
+	const at = last === null || drill.quiet === true ? void 0 : last.t + age * 1e9;
+	if (at === void 0) await sleep$3(age * 1e3);
+	const stamps = await Promise.all(parts.map((part) => lab.signal(part, "KILL", at)));
+	for (const part of parts) {
+		await lab.exec(part, [
+			"sh",
+			"-c",
+			stopScript(false)
+		]);
+		await lab.exec(part, [
+			"find",
+			"/data/db",
+			"-mindepth",
+			"1",
+			"-delete"
+		]);
+	}
+	return stamps[0];
+};
+/**
+* This function restores the old primary from the backups and starts the set again:
+* - The slices go after the oplog of the dump, in their sequence, thus one replay does the whole restore.
+* - The restore writes to a server of the old primary with no set, on its own port.
+* - Each member that lost its disk starts before the set, because `rs.initiate` asks each member.
+* - The old primary starts the set. It has the data, and the other members make their first sync from it.
+*/
+const restore = async (lab, drill, primary) => {
+	const { set } = lab;
+	const started = Date.now();
+	const port = set.port + 1;
+	const slices = Array.from({ length: drill.slices }, (_, at) => `${VAULT}/slice-${at + 1}/local/oplog.rs.bson`);
+	if (slices.length > 0) await lab.vault([
+		"sh",
+		"-c",
+		`cat ${slices.join(" ")} >> ${VAULT}/dump-0/oplog.bson`
+	]);
+	await lab.exec(primary, [
+		"sh",
+		"-c",
+		`${standaloneCommand(set, port)} > /proc/1/fd/1 2>&1 & echo $! > /tmp/server.pid`
+	]);
+	await until("the server of the restore", async () => (await lab.script(primary, "print(db.hello().ok)", port)).includes("1"), STARTS_WITHIN);
+	await lab.vault([
+		"mongorestore",
+		"--host",
+		`${lab.members[primary].ip}:${port}`,
+		"--oplogReplay",
+		`${VAULT}/dump-0`
+	]);
+	await lab.exec(primary, [
+		"sh",
+		"-c",
+		stopScript(true)
+	]);
+	for (const name of set.members) if (name !== primary) await lab.restart(name);
+	await lab.restart(primary);
+	const ips = Object.fromEntries(Object.entries(lab.members).map(([name, at]) => [name, at.ip]));
+	const initiated = await lab.script(primary, initiateScript(set, ips));
+	if (!initiated.includes("\"ok\":1")) throw new Error(`rs.initiate failed after the restore: ${initiated}`);
+	await until("the primary after the restore", async () => (await lab.script(primary, "print(db.hello().isWritablePrimary)")).includes("true"), STARTS_WITHIN);
+	return {
+		started,
+		ended: Date.now()
+	};
+};
+/**
+* This function does one run of a recovery drill:
+* - The set converges, and it loses its writes. The tenants get their data, and a hash reads it.
+* - The client writes until its stream is steady. A control stops it here.
+* - The backups go, each at its interval. The fault comes at the age of the newest backup.
+* - An election continues the same client. A restore stops the client, and it starts a second one.
+* - The set converges again. The writes that it then holds tell which writes it kept, and a second
+*   hash tells if the tenants have the same data.
+*/
+const drillOnce = async (lab, cell, environment, index) => {
+	const drill = cell.drill;
+	const { set } = lab;
+	const primary = await lab.converge();
+	await lab.flush(primary);
+	await lab.unfreeze(primary);
+	await lab.forget();
+	await lab.vault([
+		"sh",
+		"-c",
+		`rm -rf ${VAULT} && mkdir -p ${VAULT}`
+	]);
+	const seeded = await lab.script(primary, seedScript(drill, set.members.length));
+	if (!seeded.includes("\"ok\":true")) throw new Error(`the seed failed: ${seeded}`);
+	const before = hashOf(await lab.script(primary, HASH_SCRIPT));
+	const tool = (await lab.vault(["mongodump", "--version"])).split("\n")[0].trim();
+	const from = lab.kernel.length;
+	const logFrom = lab.log.length;
+	const lostBefore = droppedSoFar$3(lab.kernel);
+	const writer = await lab.client(replicaSetArgs(cell, lab.members));
+	const restores = drill.recovery === "restore";
+	let successor = null;
+	let second = null;
+	let fault = null;
+	let restored = null;
+	const backups = [];
+	try {
+		await writer.until(["ready"], 6e4);
+		if (await waitFor$2(writer, (event) => event.ev === "ack" && Number(event["n"]) >= 20, 3e4) === void 0) throw new Error(`the client did not write 20 times in 30 s`);
+		if (drill.quiet === true) await writer.stop();
+		if (restores) backups.push(...await backupsOf(lab, drill, primary, writer));
+		const stamp = await lose(lab, drill, primary, backups.at(-1)?.last ?? null);
+		fault = stamp;
+		if (restores) {
+			await writer.stop();
+			restored = await restore(lab, drill, primary);
+			second = Math.max(0, ...writer.events.filter((event) => event.ev === "sent").map((event) => Number(event["n"]))) + 1;
+			successor = await lab.client([
+				...replicaSetArgs(cell, lab.members),
+				"--from",
+				String(second)
+			]);
+			if (await waitFor$2(successor, (event) => event.ev === "ack", deadlineOf(cell)) !== void 0) await sleep$3(TAIL_MS$2);
+		} else {
+			const old = `${lab.members[primary].ip}:${set.port}`;
+			const sentAt = (n) => writer.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
+			if (await waitFor$2(writer, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > stamp.t1 && typeof event["at"] === "string" && event["at"] !== old, deadlineOf(cell)) !== void 0) await sleep$3(TAIL_MS$2);
+			await lab.restart(primary);
+		}
+	} finally {
+		await writer.stop();
+		await successor?.stop();
+	}
+	const settled = await lab.settle();
+	const lost = settled === null ? null : settled - lostBefore;
+	const after = await lab.converge().catch(() => null);
+	const holds = after === null ? null : await lab.holds(after);
+	const hash = after === null ? null : hashOf(await lab.script(after, HASH_SCRIPT));
+	const events = [...writer.events, ...successor?.events ?? []].sort((a, b) => a.t - b.t);
+	return {
+		environment,
+		index,
+		events,
+		fault,
+		kernel: lab.kernel.slice(from),
+		netns: lab.netns.client,
+		targetNetns: lab.members[primary].netns,
+		lost,
+		group: {
+			members: lab.members,
+			sentinels: {},
+			primary,
+			at: primary,
+			timeline: onMonotonic(lab.log.slice(logFrom), clockOffsetOf(events)),
+			after,
+			holds,
+			drill: {
+				tool,
+				seed: "generated",
+				backups,
+				before,
+				after: hash,
+				restored,
+				second
+			}
+		}
+	};
+};
+//#endregion
 //#region src/cluster.ts
 /** This form is a line of the console log of RabbitMQ 4.3: the time with its offset, the level, the process and the message. */
 const LINE = /^(\d{4}-\d\d-\d\d) (\d\d:\d\d:\d\d)\.(\d{3})(\d*)([+-]\d\d:\d\d) \[(\w+)\] <[^>]*> (.*)$/;
@@ -4606,6 +6222,7 @@ const clusterViewOf = (cluster, rows) => {
 * This function gives the peers that a fault on a node cuts it from. `all` is each other node and the
 * client, `group` is the other nodes, and `clients` is the client. `one` is the first other node in
 * the sequence of the cluster, and `one-way` is the same node for the segments from the part only.
+* `to-clients` is the client, for the segments from the part only.
 */
 const clusterCutOf = (cluster, members, client, part, cuts) => {
 	const others = cluster.members.filter((name) => name !== part).map((name) => members[name].ip);
@@ -4616,9 +6233,13 @@ const clusterCutOf = (cluster, members, client, part, cuts) => {
 		peers: others.slice(0, 1),
 		direction: "out"
 	};
+	if (cuts === "to-clients") return {
+		peers: [client],
+		direction: "out"
+	};
 	return { peers: [...others, client] };
 };
-const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const sleep$2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /**
 * This function opens one environment of a cluster cell:
 * - one new network, with each node, the client and the admin on it.
@@ -4780,7 +6401,7 @@ const openClusterLab = async (cluster) => {
 				calm = last.healthy ? calm + 1 : 0;
 				if (last.primary !== null) known = last.primary;
 				if (calm >= 2 && last.primary !== null) return last.primary;
-				await sleep$1(1e3);
+				await sleep$2(1e3);
 			}
 			throw new Error(`the queue did not converge in ${within} ms: ${last.why}`);
 		};
@@ -4796,7 +6417,7 @@ const openClusterLab = async (cluster) => {
 			const running = (text.trim() === "" ? {} : toolJsonOf(text)).running_nodes ?? [];
 			if (cluster.members.every((name) => running.includes(nodeNameOf(name)))) break;
 			if (Date.now() - started > 12e4) throw new Error(`the cluster did not form in 120 s: ${running.join(", ")} run`);
-			await sleep$1(1e3);
+			await sleep$2(1e3);
 		}
 		const declared = lastLine(await output(commander, [
 			"node",
@@ -4916,7 +6537,7 @@ const openClusterLab = async (cluster) => {
 					"check_running"
 				]).then(() => true).catch(() => false)) {
 					if (Date.now() - begun > 12e4) throw new Error(`${member} did not start again`);
-					await sleep$1(1e3);
+					await sleep$2(1e3);
 				}
 			},
 			stepDown(leader, to) {
@@ -4979,7 +6600,7 @@ const openClusterLab = async (cluster) => {
 					architecture: await output(holder, ["uname", "-m"]),
 					hz: /^CONFIG_HZ=(\d+)$/m.exec(config)?.[1] ?? null,
 					boot_id: await output(holder, ["cat", "/proc/sys/kernel/random/boot_id"]),
-					substrate: substrateOn("E1"),
+					substrate: substrateHere("E1"),
 					docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
 					target: {
 						image: cluster.image,
@@ -4994,12 +6615,10 @@ const openClusterLab = async (cluster) => {
 							"apk",
 							"info",
 							"-v"
-						])).split("\n").sort()
-					},
-					client: {
-						kind: "amqp",
-						image: amqp
+						])).split("\n").sort(),
+						...harnessRecord()
 					},
+					client: shippedClientRecord("amqp", amqp),
 					netns: {
 						client: clientNetns,
 						target: members[cluster.members[0]].netns
@@ -5020,7 +6639,7 @@ const openClusterLab = async (cluster) => {
 	}
 };
 /** This count is the number of confirmed publishes that a run waits for before its fault: the stream is steady then. */
-const STEADY = 20;
+const STEADY$1 = 20;
 /** This time is how long a run continues after its outcome, so that the record shows what came next. */
 const TAIL_MS$1 = 2e3;
 /**
@@ -5049,7 +6668,7 @@ const clusterArgs = (cell, members, leader) => [
 	...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
 ];
 /** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
-const waitFor = (client, match, within) => new Promise((resolve) => {
+const waitFor$1 = (client, match, within) => new Promise((resolve) => {
 	const started = Date.now();
 	const look = () => {
 		const found = client.events.find(match);
@@ -5059,7 +6678,7 @@ const waitFor = (client, match, within) => new Promise((resolve) => {
 	look();
 });
 /** This function gives the count of lost records in the latest clock line of the prober. */
-const droppedSoFar$1 = (records) => {
+const droppedSoFar$2 = (records) => {
 	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
 	return 0;
 };
@@ -5089,7 +6708,7 @@ const stampOf = async (lab, client, logFrom, marks, wall) => {
 			},
 			line
 		};
-		await sleep$1(100);
+		await sleep$2(100);
 	}
 	return {
 		stamp: monotonicOf(wall, client.events),
@@ -5126,7 +6745,7 @@ const clusterOnce = async (lab, cell, environment, index) => {
 	await lab.forget();
 	const from = lab.kernel.length;
 	const logFrom = lab.log.length;
-	const lostBefore = droppedSoFar$1(lab.kernel);
+	const lostBefore = droppedSoFar$2(lab.kernel);
 	const client = await lab.client(clusterArgs(cell, lab.members, leader));
 	const place = cell.faultAt ?? { role: "primary" };
 	let fault = null;
@@ -5137,7 +6756,7 @@ const clusterOnce = async (lab, cell, environment, index) => {
 	const undo = [];
 	try {
 		await client.until(["ready"], 6e4);
-		if (await waitFor(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY, 3e4) === void 0) throw new Error(`the client did not publish ${STEADY} times in 30 s`);
+		if (await waitFor$1(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$1, 3e4) === void 0) throw new Error(`the client did not publish ${STEADY$1} times in 30 s`);
 		let healing = null;
 		switch (cell.fault) {
 			case "blackhole":
@@ -5191,10 +6810,10 @@ const clusterOnce = async (lab, cell, environment, index) => {
 		healed.catch(() => void 0);
 		const t1 = fault.t1;
 		const sentAt = (n) => client.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
-		if (await waitFor(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1, deadlineOf(cell)) !== void 0) await sleep$1(TAIL_MS$1);
+		if (await waitFor$1(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1, deadlineOf(cell)) !== void 0) await sleep$2(TAIL_MS$1);
 		if (healing !== null) {
 			await healed.catch(() => void 0);
-			await sleep$1(TAIL_MS$1);
+			await sleep$2(TAIL_MS$1);
 		}
 	} finally {
 		for (const step of undo.reverse()) await step().catch(() => void 0);
@@ -5226,15 +6845,513 @@ const clusterOnce = async (lab, cell, environment, index) => {
 	};
 };
 //#endregion
-//#region src/run.ts
+//#region src/kafka.ts
 /**
-* This function measures and judges the raw records of a cell, with the code of the harness at
-* this time. `runCell` uses it during a cell. A holder of the raw records of a cell can use it
-* again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.
+* This module is the lab of a group of Kafka brokers in KRaft mode with one controller. It opens one
+* environment for a Kafka cell, for the runs of the cell. Each node is a container from the stock
+* image, in its own network namespace, and an injector sidecar shares its network and process
+* namespaces. The image starts by its own command, and each setting comes from the environment.
+*
+* Each run makes one new topic, thus each run starts with an empty log and its client counts from 1.
+* The client is the producer of kafka-clients. The commander holds the admin, on the network but
+* not in the namespace that the prober watches. The record of a run is the lines of the client and
+* the whole partition, read to its high watermark after the group converged.
 */
-const judge = (cell, runs) => {
-	const measured = runs.filter((run) => run.warmup !== true).map((run) => measure(cell, run));
-	return {
+/** This path is the class path of the Kafka image: the JVM expands the star itself. */
+const CLASS_PATH = "/opt/harness/kafka/classes:/opt/harness/kafka/lib/*";
+/** This option is the heap of the client and of the admin. It has no source: the two hold a producer or a read of one partition. */
+const CLIENT_HEAP = "-Xmx256m";
+/** This function gives the node id of a name. A broker has its place in the list from 1, and the controller has the id after them. */
+const nodeIdOf = (kafka, name) => {
+	const at = kafka.brokers.indexOf(name);
+	if (at !== -1) return at + 1;
+	if (name === kafka.controller) return kafka.brokers.length + 1;
+	throw new Error(`the Kafka group has no node ${name}`);
+};
+/**
+* This function gives the variable of a server property for the image. The wrapper of the image
+* reverses this map (KafkaDockerWrapper.scala:188-203). After the prefix `KAFKA_`, `___` is a
+* hyphen, `__` is an underscore and `_` is a dot.
+*/
+const envOf = (property) => `KAFKA_${property.replace(/_/g, "__").replace(/-/g, "___").replace(/\./g, "_").toUpperCase()}`;
+/** This function gives the variables of each node: the listener names, the voters, the log and the settings of the group. */
+const commonEnv = (kafka) => ({
+	CLUSTER_ID: kafka.clusterId,
+	KAFKA_HEAP_OPTS: kafka.heap,
+	KAFKA_CONTROLLER_QUORUM_VOTERS: `${nodeIdOf(kafka, kafka.controller)}@${kafka.controller}:${kafka.controllerPort}`,
+	KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER",
+	KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT",
+	KAFKA_LOG_DIRS: "/var/lib/kafka/data",
+	...Object.fromEntries(Object.entries(kafka.settings).map(([property, value]) => [envOf(property), value]))
+});
+/** This function gives the environment of a broker. */
+const brokerEnv = (kafka, name) => ({
+	...commonEnv(kafka),
+	KAFKA_PROCESS_ROLES: "broker",
+	KAFKA_NODE_ID: String(nodeIdOf(kafka, name)),
+	KAFKA_LISTENERS: `PLAINTEXT://0.0.0.0:${kafka.port}`,
+	KAFKA_ADVERTISED_LISTENERS: `PLAINTEXT://${name}:${kafka.port}`,
+	KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
+});
+/** This function gives the environment of the controller. The start script of the image stops on an advertised listener here (configure:43-48). */
+const controllerEnv = (kafka) => ({
+	...commonEnv(kafka),
+	KAFKA_PROCESS_ROLES: "controller",
+	KAFKA_NODE_ID: String(nodeIdOf(kafka, kafka.controller)),
+	KAFKA_LISTENERS: `CONTROLLER://0.0.0.0:${kafka.controllerPort}`
+});
+/**
+* This function judges a view of a partition: its leader, and if the group is healthy. A healthy
+* group has each broker not fenced, one leader, and each broker in the in-sync replicas.
+*/
+const kafkaViewOf = (kafka, view) => {
+	const primary = view.leader === void 0 || view.leader === null ? null : kafka.brokers[view.leader - 1] ?? null;
+	const all = kafka.brokers.length;
+	if (view.brokers.length !== all) return {
+		primary,
+		healthy: false,
+		why: `${view.brokers.length} brokers are not fenced`
+	};
+	if (primary === null) return {
+		primary,
+		healthy: false,
+		why: "the partition has no leader"
+	};
+	if (view.replicas?.length !== all) return {
+		primary,
+		healthy: false,
+		why: `the partition has ${view.replicas?.length ?? 0} replicas`
+	};
+	if (view.isr?.length !== all) return {
+		primary,
+		healthy: false,
+		why: `the partition has ${view.isr?.length ?? 0} in-sync replicas`
+	};
+	return {
+		primary,
+		healthy: true,
+		why: "healthy"
+	};
+};
+/**
+* This function gives the peers that a fault on a broker cuts it from. Only `group` has a law: the
+* other brokers and the controller, and the client keeps its route to the broker.
+*/
+const kafkaCutOf = (kafka, members, controller, part, cuts) => {
+	if (cuts !== "group") throw new Error(`a Kafka cell has no cut ${cuts}`);
+	return { peers: [...kafka.brokers.filter((name) => name !== part).map((name) => members[name].ip), controller.ip] };
+};
+/** This function gives the addresses that the client and the admin bootstrap from. */
+const bootstrapOf = (kafka) => kafka.brokers.map((name) => `${name}:${kafka.port}`).join(",");
+/** This function gives the name of the topic of a run. A warm-up has the index -1. */
+const topicOf = (environment, index) => `cells-e${environment}-r${index + 1}`;
+const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/**
+* This function opens one environment of a Kafka cell:
+* - one new network, with each node, the client and the commander on it.
+* - the controller first, then each broker: the stock image with its own command. The brokers
+*   register with the controller, thus they need it first.
+* - an injector sidecar for each node, in its network and process namespaces.
+* - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
+* - the prober, which watches the namespace of the client.
+*/
+const openKafkaLab = async (kafka) => {
+	const image = await harnessImage();
+	const clientImage = await kafkaClientImage();
+	const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();
+	const runtime = await getContainerRuntimeClient();
+	const opened = [];
+	let network;
+	const close = async () => {
+		for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
+		await network?.stop().catch(() => void 0);
+	};
+	const ipOf = async (container) => {
+		const ip = (await runtime.container.inspect(runtime.container.getById(container.getId()))).NetworkSettings.Networks[network.getName()]?.IPAddress;
+		if (ip === void 0 || ip === "") throw new Error(`${container.getName()} has no address on ${network.getName()}`);
+		return ip;
+	};
+	try {
+		network = await new Network().start();
+		const started = network;
+		const boxes = /* @__PURE__ */ new Map();
+		const injectors = /* @__PURE__ */ new Map();
+		const parts = {};
+		const raw = {};
+		const openNode = async (name, environment) => {
+			let partial = "";
+			const box = await new GenericContainer(kafka.image).withNetwork(started).withNetworkAliases(name).withHostname(name).withEnvironment(environment).withWaitStrategy(Wait.forLogMessage(kafka.ready)).withStartupTimeout(18e4).withLogConsumer((stream) => stream.on("data", (chunk) => {
+				partial += chunk.toString();
+				const split = partial.split("\n");
+				partial = split.pop() ?? "";
+				for (const text of split) {
+					const tail = raw[name] ??= [];
+					tail.push(text);
+					if (tail.length > 400) tail.shift();
+				}
+			})).start();
+			opened.push(box);
+			boxes.set(name, box);
+			const injector = await withPid(new GenericContainer(image).withNetworkMode(`container:${box.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW"), `container:${box.getId()}`).start();
+			opened.push(injector);
+			injectors.set(name, injector);
+			parts[name] = {
+				ip: await ipOf(box),
+				netns: await netnsOf(injector)
+			};
+		};
+		await openNode(kafka.controller, controllerEnv(kafka));
+		for (const name of kafka.brokers) await openNode(name, brokerEnv(kafka, name));
+		const members = Object.fromEntries(kafka.brokers.map((name) => [name, parts[name]]));
+		const controller = parts[kafka.controller];
+		const holder = await new GenericContainer(image).withNetwork(network).start();
+		opened.push(holder);
+		const admin = await new GenericContainer(image).withNetworkMode(`container:${holder.getId()}`).withPrivilegedMode().start();
+		opened.push(admin);
+		const clientIp = await ipOf(holder);
+		const commander = await new GenericContainer(clientImage).withNetwork(network).withCommand([
+			"sh",
+			"-c",
+			"while :; do sleep 3600; done"
+		]).start();
+		opened.push(commander);
+		const clientNetns = await netnsOf(admin);
+		const records = lines();
+		const prober = await new GenericContainer(image).withPrivilegedMode().withNetworkMode("none").withCommand(["/opt/harness/prober", String(clientNetns)]).withLogConsumer(records.consume).start();
+		opened.push(prober);
+		const begun = await records.until(["begin", "fail"], 3e4, "prober");
+		if (begun.ev === "fail") throw new Error(`the prober failed: ${String(begun["why"])}`);
+		const box = (name) => {
+			const found = boxes.get(name);
+			if (found === void 0) throw new Error(`the group has no node ${name}`);
+			return found;
+		};
+		const injectorOf = (name) => {
+			const found = injectors.get(name);
+			if (found === void 0) throw new Error(`the group has no node ${name}`);
+			return found;
+		};
+		const lastLine = (text) => text.split("\n").filter((line) => line.trim() !== "").at(-1) ?? "";
+		const stamped = async (where, argv) => {
+			const stamp = JSON.parse(lastLine(await output(where, argv)));
+			return {
+				t0: stamp.t0,
+				t1: stamp.t1
+			};
+		};
+		const after = (at) => at === void 0 ? [] : ["--at", String(Math.round(at))];
+		const bootstrap = bootstrapOf(kafka);
+		const tool = async (...argv) => lastLine(await output(commander, [
+			"java",
+			CLIENT_HEAP,
+			"-cp",
+			CLASS_PATH,
+			"AdminKafka",
+			bootstrap,
+			...argv
+		]));
+		const view = async (topic) => JSON.parse(await tool("view", ...topic === void 0 ? [] : [topic]));
+		return {
+			kafka,
+			clientIp,
+			members,
+			controller,
+			netns: { client: clientNetns },
+			kernel: records.seen,
+			raw,
+			async sysctl(values) {
+				const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
+				if (settings.length > 0) await output(admin, [
+					"sysctl",
+					"-q",
+					"-w",
+					...settings
+				]);
+				const read = {};
+				for (const key of Object.keys(values)) read[key] = await output(admin, [
+					"sysctl",
+					"-n",
+					key
+				]);
+				return read;
+			},
+			async brokersUp(within = CONVERGE_MS) {
+				const begin = Date.now();
+				let last = "no view yet";
+				while (Date.now() - begin < within) {
+					const read = await view().catch((error) => error.message.slice(0, 200));
+					if (typeof read !== "string" && read.brokers.length === kafka.brokers.length) return;
+					last = typeof read === "string" ? read : `${read.brokers.length} brokers are not fenced`;
+					await sleep$1(1e3);
+				}
+				throw new Error(`the brokers did not come up in ${within} ms: ${last}`);
+			},
+			async create(topic) {
+				const ids = kafka.brokers.map((name) => nodeIdOf(kafka, name)).join(",");
+				const reply = await tool("create", topic, String(kafka.minInsync), ids);
+				if (!reply.includes("\"ok\":true")) throw new Error(`the create of the topic failed: ${reply}`);
+			},
+			async converge(topic, within = CONVERGE_MS) {
+				const begin = Date.now();
+				let calm = 0;
+				let last = {
+					primary: null,
+					healthy: false,
+					why: "no view yet"
+				};
+				while (Date.now() - begin < within) {
+					last = await view(topic).then((read) => kafkaViewOf(kafka, read), (error) => ({
+						primary: null,
+						healthy: false,
+						why: error.message.slice(0, 200)
+					}));
+					calm = last.healthy ? calm + 1 : 0;
+					if (calm >= 2 && last.primary !== null) return last.primary;
+					await sleep$1(1e3);
+				}
+				throw new Error(`the partition did not converge in ${within} ms: ${last.why}`);
+			},
+			async holds(topic) {
+				const reply = await tool("holds", topic);
+				const { held } = JSON.parse(reply);
+				if (held === void 0) throw new Error(`the read of the partition failed: ${reply}`);
+				return held.map(Number).filter((n) => Number.isInteger(n) && n > 0);
+			},
+			async client(args) {
+				const events = lines();
+				const container = await new GenericContainer(clientImage).withNetworkMode(`container:${holder.getId()}`).withCommand([
+					"java",
+					CLIENT_HEAP,
+					"-cp",
+					CLASS_PATH,
+					"ClientKafka",
+					bootstrap,
+					...args
+				]).withLogConsumer(events.consume).start();
+				return {
+					events: events.seen,
+					until: (kinds, within) => events.until(kinds, within, "client"),
+					stop: () => container.stop({ timeout: 2 }).then(() => void 0).catch(() => void 0)
+				};
+			},
+			fault(kind, part, cuts, at) {
+				const cut = kafkaCutOf(kafka, members, controller, part, cuts);
+				return stamped(injectorOf(part), [
+					"/opt/harness/inject",
+					kind,
+					cut.peers.join(","),
+					...after(at)
+				]);
+			},
+			lift(kind, part, cuts, at) {
+				const cut = kafkaCutOf(kafka, members, controller, part, cuts);
+				return stamped(injectorOf(part), [
+					"/opt/harness/inject",
+					kind === "rst" ? "clear-rst" : "clear",
+					cut.peers.join(","),
+					...after(at)
+				]);
+			},
+			async forget() {
+				await output(admin, [
+					"ip",
+					"tcp_metrics",
+					"flush",
+					"all"
+				]);
+			},
+			async settle() {
+				const from = records.seen.length;
+				const clock = await new Promise((resolve) => {
+					const begin = Date.now();
+					const look = () => {
+						const line = records.seen.slice(from).find((record) => record.ev === "clock");
+						if (line !== void 0) resolve(line);
+						else if (Date.now() - begin > 5e3) resolve(null);
+						else setTimeout(look, 50);
+					};
+					look();
+				});
+				return clock === null ? null : Number(clock["lost"] ?? 0);
+			},
+			async manifest(sysctls) {
+				const config = await output(admin, ["zcat", "/proc/config.gz"]).catch(() => "");
+				const inspect = async (container) => runtime.container.inspect(runtime.container.getById(container.getId()));
+				const settings = {
+					settings: Object.entries(kafka.settings).map(([property, value]) => `${property}=${value}`).join("; "),
+					heap: kafka.heap,
+					"min.insync.replicas": String(kafka.minInsync)
+				};
+				const list = async (names) => {
+					const out = [];
+					for (const name of names) {
+						const version = lastLine(await output(box(name), ["/opt/kafka/bin/kafka-topics.sh", "--version"]).catch(() => ""));
+						out.push({
+							name,
+							ip: parts[name].ip,
+							id: (await inspect(box(name))).Image,
+							settings: {
+								version,
+								...settings
+							}
+						});
+					}
+					return out;
+				};
+				const brokers = await list(kafka.brokers);
+				return {
+					kernel: await output(holder, ["uname", "-r"]),
+					architecture: await output(holder, ["uname", "-m"]),
+					hz: /^CONFIG_HZ=(\d+)$/m.exec(config)?.[1] ?? null,
+					boot_id: await output(holder, ["cat", "/proc/sys/kernel/random/boot_id"]),
+					docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
+					target: {
+						image: kafka.image,
+						id: brokers[0].id,
+						settings: brokers[0].settings,
+						sysctls: {}
+					},
+					harness: {
+						image,
+						id: (await inspect(holder)).Image,
+						packages: (await output(holder, [
+							"apk",
+							"info",
+							"-v"
+						])).split("\n").sort()
+					},
+					client: {
+						kind: "image",
+						image: clientImage
+					},
+					netns: {
+						client: clientNetns,
+						target: members[kafka.brokers[0]].netns
+					},
+					sysctls,
+					group: {
+						members: brokers,
+						sentinels: await list([kafka.controller]),
+						observer: { image: clientImage }
+					}
+				};
+			},
+			close
+		};
+	} catch (error) {
+		await close();
+		throw error;
+	}
+};
+/** This count is the number of acknowledged writes that a run waits for before its fault: the stream is steady then. */
+const STEADY = 20;
+/**
+* This time is how long a run continues after its lift. The record then shows the step-down of the
+* leader and the writes after it. It has no source. It is above the largest step-down of the
+* law, which is approximately 2 s after the lift.
+*/
+const KAFKA_TAIL_MS = 1e4;
+/** This function gives the arguments of the client of a Kafka cell: the topic, the writers, the stream of writes and the settings. */
+const kafkaArgs = (cell, topic) => [
+	"--topic",
+	topic,
+	"--writers",
+	String(cell.kafka.writers),
+	"--interval-ms",
+	String(cell.probe.interval === void 0 ? 50 : valueIn(cell.probe.interval, "ms")),
+	"--max-s",
+	String(Math.ceil(deadlineOf(cell) / 1e3 + (cell.hold === void 0 ? 0 : valueIn(cell.hold, "s")) + 120)),
+	...Object.entries(cell.driverOptions ?? {}).flatMap(([name, value]) => ["--set", `${name}=${String(value)}`])
+];
+/** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
+const waitFor = (client, match, within) => new Promise((resolve) => {
+	const started = Date.now();
+	const look = () => {
+		const found = client.events.find(match);
+		if (found !== void 0 || Date.now() - started > within) resolve(found);
+		else setTimeout(look, 20);
+	};
+	look();
+});
+/** This function gives the count of lost records in the latest clock line of the prober. */
+const droppedSoFar$1 = (records) => {
+	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
+	return 0;
+};
+/**
+* This function does one run of a Kafka cell:
+* - Each broker is up, the topic of the run exists, and its partition is whole. The client starts,
+*   and its stream of writes gets steady.
+* - The fault cuts the leader from the other brokers and the controller, and it lifts at its instant
+*   in the VM. The client keeps its route to the leader, thus only the group knows of the cut.
+* - The run waits for the lift and for the tail after it.
+* - Each fault goes, the client stops, and the partition converges again. The admin then reads
+*   the whole partition, thus the record tells which writes the group kept.
+*/
+const kafkaOnce = async (lab, cell, environment, index) => {
+	if (cell.fault !== "partition") throw new Error(`a Kafka cell has no fault ${cell.fault}`);
+	if (cell.hold === void 0) throw new Error("a Kafka cell needs a hold: its law reads it");
+	const place = cell.faultAt ?? { role: "primary" };
+	if (place.role !== "primary") throw new Error(`a Kafka cell has no fault on the ${place.role}`);
+	const cuts = place.cuts ?? "group";
+	const topic = topicOf(environment, index);
+	await lab.brokersUp();
+	await lab.create(topic);
+	const leader = await lab.converge(topic);
+	await lab.forget();
+	const from = lab.kernel.length;
+	const lostBefore = droppedSoFar$1(lab.kernel);
+	const client = await lab.client(kafkaArgs(cell, topic));
+	let fault = null;
+	let heal = null;
+	const undo = [];
+	try {
+		await client.until(["ready"], 6e4);
+		if (await waitFor(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY, 3e4) === void 0) throw new Error(`the client did not write ${STEADY} times in 30 s`);
+		fault = await lab.fault("blackhole", leader, cuts);
+		undo.push(() => lab.lift("blackhole", leader, cuts));
+		heal = await lab.lift("blackhole", leader, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
+		undo.length = 0;
+		await sleep$1(KAFKA_TAIL_MS);
+	} finally {
+		for (const step of undo.reverse()) await step().catch(() => void 0);
+		await client.stop();
+	}
+	const settled = await lab.settle();
+	const lost = settled === null ? null : settled - lostBefore;
+	const after = await lab.converge(topic).catch(() => null);
+	const holds = after === null ? null : await lab.holds(topic);
+	return {
+		environment,
+		index,
+		events: [...client.events],
+		fault,
+		...heal === null ? {} : { heal },
+		kernel: lab.kernel.slice(from),
+		netns: lab.netns.client,
+		targetNetns: lab.members[leader].netns,
+		lost,
+		group: {
+			members: lab.members,
+			sentinels: { [lab.kafka.controller]: lab.controller },
+			primary: leader,
+			at: leader,
+			timeline: [],
+			after,
+			holds
+		}
+	};
+};
+//#endregion
+//#region src/run.ts
+/**
+* This function measures and judges the raw records of a cell, with the code of the harness at
+* this time. `runCell` uses it during a cell. A holder of the raw records of a cell can use it
+* again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.
+*/
+const judge = (cell, runs) => {
+	const measured = runs.filter((run) => run.warmup !== true).map((run) => measure(cell, run));
+	return {
 		measured,
 		verdict: verdictOf(cell.prediction, measured, noiseOf(cell))
 	};
@@ -5295,6 +7412,7 @@ const probeArgs = (cell, lab) => {
 	if (probe.driver?.name === "pg" || probe.driver?.name === "hikari" || probe.driver?.name === "amqp") {
 		if (probe.driver.name !== "amqp" && probe.driver.query !== void 0) args.push("--query", probe.driver.query);
 		if (probe.driver.name !== "amqp" && probe.driver.inflight !== void 0) args.push("--inflight-query", probe.driver.inflight);
+		if (probe.driver.name === "hikari" && probe.driver.burst !== void 0) args.push("--burst", String(probe.driver.burst.count), "--burst-gap-ms", String(Math.round(valueIn(probe.driver.burst.gap, "ms"))));
 		if (cell.before === true) args.push("--before");
 		if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
 		args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
@@ -5367,6 +7485,8 @@ const requestSent = (probe, within) => new Promise((resolve, reject) => {
 });
 /** This time is how long a run of a driver cell continues after its outcome, so that the record shows what came next. */
 const TAIL_MS = 1e3;
+/** This time lets the last lines of a route monitor reach the harness after a stall ends. */
+const MONITOR_MS = 200;
 /** This function waits for the outcome of a run of a driver cell, or for `within` ms. The outcome is the end of its band and of each later band. */
 const driverOutcome = async (cell, current, within) => {
 	const started = Date.now();
@@ -5454,6 +7574,19 @@ const dials = (probe, count, within) => new Promise((resolve) => {
 	};
 	look();
 });
+/** This time is the most that a run waits for the lines of the target after the end of its client. */
+const LINES_MS = 5e3;
+/**
+* This function waits for the lines of a run's counts. The lines of the target come on their own
+* stream, thus they can come after the last line of the client. A line above a count comes in the
+* pause after the wait, thus it is in the run that wrote it.
+*/
+const linesCounted = async (lab, from, counts) => {
+	const started = Date.now();
+	const short = () => counts.some((each) => countOf(lab.server.slice(from), each.match) < each.count);
+	while (short() && Date.now() - started < LINES_MS) await sleep(20);
+	await sleep(250);
+};
 /** This function gives the netem specification of a cell: its latency, then the netem of its fault when it has one. */
 const netemSpec = (cell, fault) => {
 	const parts = [...cell.latency === void 0 ? [] : [`delay ${valueIn(cell.latency, "ms")}ms`], ...fault === void 0 ? [] : [fault]];
@@ -5474,6 +7607,9 @@ const netemSpec = (cell, fault) => {
 */
 const once = async (lab, cell, environment, index, dry = false) => {
 	if (cell.before === true) return beforeOnce(lab, cell, environment, index);
+	await lab.fabric?.ready();
+	const sessions = await lab.fabric?.sessions();
+	const routesFrom = lab.fabric?.routes.length ?? 0;
 	await lab.forget();
 	const from = lab.kernel.length;
 	const middleboxFrom = lab.middlebox.length;
@@ -5493,6 +7629,10 @@ const once = async (lab, cell, environment, index, dry = false) => {
 		throw new Error(`${error.message}; the client's last lines: ${last.join(" ")}`);
 	});
 	const placed = lab.topology === null ? {} : { topology: lab.topology };
+	const fabricOf = () => {
+		if (lab.fabric === void 0) throw new Error(`${cell.name} silences a router of a lab with no fabric`);
+		return lab.fabric;
+	};
 	/** This function gives the run as the records show it so far: the waits read it. */
 	const partial = () => ({
 		environment,
@@ -5566,6 +7706,7 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					case "server-close":
 						await lab.serverClose();
 						return null;
+					case "silence": return fabricOf().silence();
 					case "hop-event": {
 						if (cell.event === void 0) throw new Error(`${cell.name} names no event`);
 						const stamp = await lab.event(cell.event);
@@ -5623,6 +7764,9 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					case "toxic":
 						await lab.toxic(null);
 						return;
+					case "silence":
+						await fabricOf().clear();
+						return;
 					default: return heal === null ? lab.lift("blackhole", faultAt) : void 0;
 				}
 			};
@@ -5643,6 +7787,7 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					const gap = cell.probe.http?.gap === void 0 ? 0 : valueIn(cell.probe.http.gap, "s");
 					const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
 					await probe.until(HTTP_DONE, (gap + window) * 2e3 + 1e4).catch(() => void 0);
+					if (prediction.detects === null && prediction.counts !== void 0) await linesCounted(lab, serverFrom, prediction.counts);
 				} else if (cell.fault === "stall") {
 					if (cell.stall === void 0) throw new Error(`${cell.name} says no stall's length`);
 					const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
@@ -5654,6 +7799,21 @@ const once = async (lab, cell, environment, index, dry = false) => {
 						"eof",
 						"drop"
 					], window * 1e3).catch(() => void 0);
+				} else if (cell.fault === "silence") {
+					const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
+					await requestSent(probe, 5e3);
+					await probe.until([
+						"data",
+						"error",
+						"eof",
+						"drop"
+					], window * 1e3).catch(() => void 0);
+					await sleep(MONITOR_MS);
+					const moved = laterMove(fabricOf().routes.slice(routesFrom), fault?.t0 ?? 0);
+					heal = moved === null ? null : {
+						t0: moved,
+						t1: moved
+					};
 				} else if (prediction.detects === null) {
 					const window = valueIn(prediction.window, "s") * 1e3;
 					await probe.until([
@@ -5695,14 +7855,21 @@ const once = async (lab, cell, environment, index, dry = false) => {
 		index,
 		events: [...probe.events],
 		fault,
-		...cell.fault === "stall" || thawing !== null ? { heal } : {},
+		...cell.fault === "stall" || cell.fault === "silence" || thawing !== null ? { heal } : {},
 		kernel: lab.kernel.slice(from),
 		netns: lab.netns.client,
 		targetNetns: lab.netns.target,
 		lost,
 		...placed,
 		...lab.topology === null ? {} : { middlebox: lab.middlebox.slice(middleboxFrom) },
-		...cell.target.lines === true ? { server: lab.server.slice(serverFrom) } : {}
+		...cell.target.lines === true ? { server: lab.server.slice(serverFrom) } : {},
+		...lab.fabric === void 0 || sessions === void 0 ? {} : { fabric: {
+			routes: lab.fabric.routes.slice(routesFrom),
+			established: {
+				before: sessions,
+				after: await lab.fabric.sessions()
+			}
+		} }
 	};
 };
 /** This function divides a count of runs across environments, as equally as possible. */
@@ -5714,6 +7881,17 @@ const spread = (runs, environments) => Array.from({ length: environments }, (_,
 const endsEnvironment = (cell) => cell.fresh === true || cell.fault === "fin" && (cell.observer === void 0 || cell.observer === "client");
 /** This function says one line for a run. */
 const line = (cell, run, read) => `${cell.name}: env ${run.environment} ${run.warmup === true ? "warm-up" : `run ${run.index}`} ${read.outcome.kind}${read.elapsed === null ? "" : ` ${read.elapsed.value} s`}${read.void === null ? "" : ` (void: ${read.void})`}`;
+/** This function adds to a run the share of the CPU that the host took while the run operated. */
+const withSteal = async (work) => {
+	const before = readCpu();
+	const run = await work();
+	const after = readCpu();
+	const steal = before === null || after === null ? null : stealShare(before, after);
+	return steal === null ? run : {
+		...run,
+		steal
+	};
+};
 /**
 * This function does the part of the runs of a cell in one environment, and records its manifest.
 * A warm-up goes first, and the verdict does not count it. On a loaded machine, a new environment is
@@ -5724,6 +7902,8 @@ const inEnvironment = async (cell, environment, count, say) => {
 	if (cell.group !== void 0) return inGroupEnvironment(cell, environment, count, say);
 	if (cell.replicaSet !== void 0) return inReplicaSetEnvironment(cell, environment, count, say);
 	if (cell.cluster !== void 0) return inClusterEnvironment(cell, environment, count, say);
+	if (cell.kafka !== void 0) return inKafkaEnvironment(cell, environment, count, say);
+	if (cell.pair !== void 0) return inPairEnvironment(cell, environment, count, say);
 	const lab = await openLab(cell.target, cell.client ?? cell.probe.driver?.name ?? "tcp", cell.topology);
 	try {
 		const settings = await lab.configure(cell.targetSettings ?? {});
@@ -5743,13 +7923,13 @@ const inEnvironment = async (cell, environment, count, say) => {
 			jitter: jitterOf(cell, run)
 		});
 		const warmup = {
-			...timed(await once(lab, cell, environment, -1, endsEnvironment(cell))),
+			...timed(await withSteal(() => once(lab, cell, environment, -1, endsEnvironment(cell)))),
 			warmup: true
 		};
 		say(line(cell, warmup, measure(cell, warmup)));
 		const runs = [];
 		for (let index = 0; index < count; index += 1) {
-			const run = timed(await once(lab, cell, environment, index));
+			const run = timed(await withSteal(() => once(lab, cell, environment, index)));
 			runs.push(run);
 			say(line(cell, run, measure(cell, run)));
 		}
@@ -5796,17 +7976,48 @@ const inGroupEnvironment = async (cell, environment, count, say) => {
 * is a whole run with its fault, as for a group under Sentinels. The set converges before each run.
 */
 const inReplicaSetEnvironment = async (cell, environment, count, say) => {
-	const lab = await openReplicaSetLab(cell.replicaSet);
+	const lab = await openReplicaSetLab(cell.replicaSet, cell.drill !== void 0);
+	const once = cell.drill === void 0 ? replicaSetOnce : drillOnce;
 	try {
 		const manifest = await lab.manifest(await lab.sysctl(cell.sysctls));
 		const warmup = {
-			...await replicaSetOnce(lab, cell, environment, -1),
+			...await once(lab, cell, environment, -1),
 			warmup: true
 		};
 		say(line(cell, warmup, measure(cell, warmup)));
 		const runs = [];
 		for (let index = 0; index < count; index += 1) {
-			const run = await replicaSetOnce(lab, cell, environment, index);
+			const run = await once(lab, cell, environment, index);
+			runs.push(run);
+			say(line(cell, run, measure(cell, run)));
+		}
+		return {
+			manifest,
+			runs,
+			warmup
+		};
+	} finally {
+		await lab.close();
+	}
+};
+/**
+* This function does the part of the runs of a pair cell in one environment. The warm-up is a dry
+* run. The run promotes the standby, thus a pair cell has an environment for each run.
+*/
+const inPairEnvironment = async (cell, environment, count, say) => {
+	const lab = await openLab(cell.target, cell.client ?? cell.probe.driver?.name ?? "tcp", cell.topology);
+	try {
+		await lab.warm();
+		const manifest = await lab.manifest(await lab.sysctl(cell.sysctls));
+		const standby = await openStandby(lab, cell);
+		const warmup = {
+			...await pairOnce(lab, standby, cell, environment, -1, true),
+			warmup: true
+		};
+		say(line(cell, warmup, measure(cell, warmup)));
+		const runs = [];
+		for (let index = 0; index < count; index += 1) {
+			const run = await pairOnce(lab, standby, cell, environment, index);
 			runs.push(run);
 			say(line(cell, run, measure(cell, run)));
 		}
@@ -5848,6 +8059,34 @@ const inClusterEnvironment = async (cell, environment, count, say) => {
 	}
 };
 /**
+* This function does the part of the runs of a Kafka cell in one environment. The warm-up is a whole
+* run with its fault, as for a cluster. Each run makes its own topic.
+*/
+const inKafkaEnvironment = async (cell, environment, count, say) => {
+	const lab = await openKafkaLab(cell.kafka);
+	try {
+		const manifest = await lab.manifest(await lab.sysctl(cell.sysctls));
+		const warmup = {
+			...await kafkaOnce(lab, cell, environment, -1),
+			warmup: true
+		};
+		say(line(cell, warmup, measure(cell, warmup)));
+		const runs = [];
+		for (let index = 0; index < count; index += 1) {
+			const run = await kafkaOnce(lab, cell, environment, index);
+			runs.push(run);
+			say(line(cell, run, measure(cell, run)));
+		}
+		return {
+			manifest,
+			runs,
+			warmup
+		};
+	} finally {
+		await lab.close();
+	}
+};
+/**
 * This function does the runs of a cell. They go across new environments, and each environment records
 * its manifest. Then the verdict judges the runs. A median across the edge of the band extends
 * the cell from 11 runs to 22, and then to 44, each extension in new environments. With a
@@ -5901,6 +8140,6 @@ const runCell = async (cell, say = () => {}) => {
 	}
 };
 //#endregion
-export { AWS_FROM, CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, DROP_HANDLING, GIVES_AWAY, INJECT, KEPT, LINGER_MS, LISTING, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, callOf, clockOffsetOf, closeOrExit, cloudOf, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, comparable, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, dropFinding, dropOf, endpointAfter, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, goHttpImage, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isHopPlace, jitterOf, jitterOfLines, judge, lagsOf, latestEndOf, leftAfter, logNameOf, matches, measure, measureDriver, measureGroup, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicOf, nodeClientImage, nodeCommand, nodeHttpImage, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, onMonotonic, openClusterLab, openGroupLab, openLab, openReplicaSetLab, pairingOf, pairingsOf, peersOf, pgClientImage, pingsOf, plainOf, planOf, programLines, programOf, quantileOf, readHost, reconnectFinding, reconnectOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routerImage, routesOf, runCell, sentinelConfig, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, statusViewOf, subjectClient, subjectProblems, subscriptionsOf, substrateKey, substrateOf, substrateOn, targetImageOf, targetSourceOf, templated, toolJsonOf, toxicBody, unlockedManifests, unpinnedBases, unpoolable, verdictOf, viewOf, writesOf };
+export { AWS_FROM, CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, DAEMONS, DEBUG, DIRTY, DROP_HANDLING, FABRIC_CLIENT, FABRIC_DEVICE, FABRIC_TARGET, FRR_IMAGE, GIVES_AWAY, HASH_SCRIPT, INJECT, INSERT, KAFKA_TAIL_MS, KEPT, LINGER_MS, LISTING, LOCK_NAMES, LOSS_MARGIN, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, RESTART, ROUTERS, SEED_DOCUMENT, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STANDBY_READY, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, basesOf, bootstrapOf, brokerEnv, buildOf, buildRecordOf, callOf, cancelsOf, cargoVersions, clockOffsetOf, closeOrExit, cloudOf, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, commitProblems, comparable, controllerEnv, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, drillOnce, drillSpan, dropFinding, dropOf, endpointAfter, envOf, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, frrConf, goHttpImage, goVersions, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, harnessRecord, harnessVersion, hashOf, highestHeld, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isHopPlace, jarVersions, jitterOf, jitterOfLines, judge, kafkaArgs, kafkaClientImage, kafkaCutOf, kafkaOnce, kafkaViewOf, lagsOf, laterMove, latestEndOf, leafLacks, leftAfter, locksOf, logNameOf, manifestProblems, matches, measure, measureDriver, measureGroup, measurePorts, measureQueue, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicAt, monotonicOf, nodeClientImage, nodeCommand, nodeHttpImage, nodeIdOf, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, npmVersions, numbersOf, onMonotonic, openClusterLab, openFabric, openGroupLab, openKafkaLab, openLab, openReplicaSetLab, openStandby, pairArgs, pairOnce, pairingOf, pairingsOf, peersOf, pgClientImage, pingsOf, plainOf, planOf, programLines, programOf, quantileOf, readCpu, readHost, readOplog, reconnectFinding, reconnectOf, recordProblems, recoveryFindings, recoveryOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routeLine, routerImage, routesOf, runCell, seedScript, sentinelConfig, sessionsOf, shippedClientRecord, shippedSource, sliceQuery, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, standaloneCommand, standbyCommand, statusViewOf, stealShare, stopScript, subjectClient, subjectProblems, subscriptionsOf, substrateHere, substrateKey, substrateOf, substrateOn, targetImageOf, targetSourceOf, templated, toolJsonOf, topicOf, topologyOf, toxicBody, unlockedManifests, unpinnedBases, unpoolable, verdictOf, versionsOf, viewOf, writesOf };
 
 //# sourceMappingURL=index.js.map