@mark1russell7/harness 0.11.16 → 0.11.17

package/build/index.js

@@ -1,4 +1,4 @@
-import { C as targetImageOf, E as unpinnedBases, S as subjectProblems, T as unlockedManifests, _ as planOf, a as SHIPPED_NAMES, b as routerImage, c as harnessImage, d as imageProblems, f as inContext, g as pgClientImage, h as nodeHttpImage, i as SHIPPED_CLIENTS, l as hikariClientImage, m as nodeClientImage, n as PINNED, o as SUBJECTS, p as mongoClientImage, r as REPOSITORY, s as goHttpImage, t as CLIENT_NAMES, u as imageOf, v as proceduresImage, w as targetSourceOf, x as subjectClient, y as programOf } from "./image-RM60Paqe.js";
+import { C as subjectProblems, D as unpinnedBases, E as unlockedManifests, S as subjectClient, T as targetSourceOf, _ as pgClientImage, a as SHIPPED_NAMES, b as programOf, c as goHttpImage, d as imageOf, f as imageProblems, g as nodeHttpImage, h as nodeClientImage, i as SHIPPED_CLIENTS, l as harnessImage, m as mongoClientImage, n as PINNED, o as SUBJECTS, p as inContext, r as REPOSITORY, s as amqpClientImage, t as CLIENT_NAMES, u as hikariClientImage, v as planOf, w as targetImageOf, x as routerImage, y as proceduresImage } from "./image-67OW9r9u.js";
 import { quantity, valueIn } from "@mark1russell7/quantity";
 import { createHash } from "node:crypto";
 //#region src/cell.ts
@@ -1031,8 +1031,8 @@ const measureDriver = (cell, run) => {
 /**
 * This module measures a run of a group cell. It gives the first write that a primary accepted
 * after the fault, and the timeline of the group. The Sentinels of a group announce its timeline.
-* The logs of the members of a replica set and its client tell it. The module also gives what
-* became of each write.
+* The logs of the members of a replica set or of the nodes of a cluster, and its client, tell it. The
+* module also gives what became of each write.
 */
 /** These are the stages of a timeline under Sentinels, in their sequence after a fault. */
 const STAGES = [
@@ -1152,6 +1152,58 @@ const replicaSetStagesOf = (run, from, sentAfter, lands) => {
 	if (accepted !== void 0) out["first-write"] = between$1(from, accepted.t);
 	return out;
 };
+/** These are the stages of the timeline of a cluster of RabbitMQ, in their usual sequence after a fault. */
+const CLUSTER_STAGES = [
+	"monitor-down",
+	"elected",
+	"followed",
+	"old-leader-steps-down",
+	"first-write"
+];
+/**
+* These are the log lines of RabbitMQ that mark each stage of a cluster, by their message (ra 3.1.10,
+* ra_server_proc.erl:871, :1336-1345 and :1974). `at` tells where the line comes from: the node of the
+* old leader, or another node.
+*/
+const NODE_MARKS = {
+	"monitor-down": {
+		message: /Leader monitor down/,
+		at: "other"
+	},
+	elected: {
+		message: /: candidate -> leader in term/,
+		at: "other"
+	},
+	followed: {
+		message: /detected a new leader/,
+		at: "other"
+	},
+	"old-leader-steps-down": {
+		message: /: leader -> \w+ in term/,
+		at: "old"
+	}
+};
+/**
+* This function gives the time of each stage of a cluster from an instant, in a run. The stages of
+* the nodes come from their log lines (`ev` "rabbit"), whose times the lab puts on the monotonic
+* clock. `first-write` is the first confirm of a publish that the client sent after the commit of
+* the fault (`sentAfter`), through any node.
+*/
+const clusterStagesOf = (run, from, sentAfter) => {
+	const out = {};
+	const old = run.group?.primary ?? "";
+	for (const line of run.group?.timeline ?? []) {
+		if (line.ev !== "rabbit" || line.t < from) continue;
+		const member = String(line["member"] ?? "");
+		for (const stage of Object.keys(NODE_MARKS)) {
+			const mark = NODE_MARKS[stage];
+			if (out[stage] === void 0 && mark.at === "old" === (member === old) && mark.message.test(String(line["msg"] ?? ""))) out[stage] = between$1(from, line.t);
+		}
+	}
+	const accepted = acceptedAfter(run, sentAfter, "same");
+	if (accepted !== void 0) out["first-write"] = between$1(from, accepted.t);
+	return out;
+};
 /** This function gives the first write that a primary accepted, of a write sent after an instant. It is on a new primary for `new`. */
 const acceptedAfter = (run, sentAfter, lands) => {
 	const sent = /* @__PURE__ */ new Map();
@@ -1296,9 +1348,10 @@ const measureGroup = (cell, run) => {
 	const timeline = run.group?.timeline ?? [];
 	const heal = run.heal ?? null;
 	const lands = expected?.lands ?? "new";
-	const replica = cell.replicaSet !== void 0;
+	const cluster = cell.cluster !== void 0;
+	const replica = cell.replicaSet !== void 0 || cluster;
 	const stall = prediction.detects === false && prediction.recovers !== void 0;
-	const stages = replica ? replicaSetStagesOf(run, t0, run.fault.t1, lands) : stagesOf(timeline, t0);
+	const stages = cluster ? clusterStagesOf(run, t0, run.fault.t1) : replica ? replicaSetStagesOf(run, t0, run.fault.t1, lands) : stagesOf(timeline, t0);
 	const after = lands === "same" && heal !== null && (!replica || stall) ? heal.t0 : t0;
 	const sentAt = /* @__PURE__ */ new Map();
 	for (const event of events) if (event.ev === "sent") sentAt.set(Number(event["n"]), event.t);
@@ -1306,7 +1359,7 @@ const measureGroup = (cell, run) => {
 		if (event.ev !== "ack" || event.t <= after) return false;
 		if (replica && (sentAt.get(Number(event["n"])) ?? -Infinity) <= run.fault.t1) return false;
 		const member = memberAt(run, event["at"]);
-		return member !== null && (lands === "new" ? member !== primary : true);
+		return member !== null && (lands === "new" && !cluster ? member !== primary : true);
 	});
 	const to = accepted === void 0 ? null : memberAt(run, accepted["at"]);
 	if (replica) {
@@ -1317,7 +1370,7 @@ const measureGroup = (cell, run) => {
 		from: primary,
 		to,
 		stages,
-		writes: writesOf(run, t0, primary),
+		writes: writesOf(run, t0, cluster ? "" : primary),
 		errors: errorsOf(events, t0),
 		...cell.calibrates === "sentinels" ? { lags: lagsOf(run, kernel) } : {}
 	};
@@ -1336,8 +1389,9 @@ const measureGroup = (cell, run) => {
 			if (late > valueIn(INJECT, "s")) voided = `the lift came ${Math.round(late * 1e4) / 10} ms after its hold: the injector was late, not the group`;
 		}
 	}
+	const firstAddress = cluster ? String(events.find((event) => event.ev === "connected")?.["at"] ?? "") : `${run.group?.members[primary]?.ip}:${(cell.group ?? cell.replicaSet)?.port}`;
 	const assumed = prediction.detects === null ? void 0 : prediction.rtt;
-	const first = kernel.find((record) => record.netns === run.netns && record.dst === `${run.group?.members[primary]?.ip}:${(cell.group ?? cell.replicaSet)?.port}` && record.tcp?.srtt !== void 0);
+	const first = kernel.find((record) => record.netns === run.netns && record.dst === firstAddress && record.tcp?.srtt !== void 0);
 	const rtt = first?.tcp?.srtt === void 0 ? null : valueIn(first.tcp.srtt, "us") / 1e3;
 	if (voided === null && assumed !== void 0 && rtt !== null && rtt > valueIn(assumed.high, "ms")) voided = `rtt-over-assumption: its first round trip was ${rtt} ms, above the ${valueIn(assumed.high, "ms")} ms that the prediction assumes`;
 	if (voided === null && prediction.detects === true && prediction.from === "fault" && (run.fault.t1 - run.fault.t0) / 1e6 > valueIn(INJECT, "ms")) voided = `the commit of the fault took ${Math.round((run.fault.t1 - run.fault.t0) / 1e5) / 10} ms, longer than the noise of the pipeline: its start is not known`;
@@ -1359,7 +1413,7 @@ const measureGroup = (cell, run) => {
 		const at = stages[stage];
 		if (at !== void 0) findings.push(`${stage} came ${at.value} s after the fault, where it should not`);
 	}
-	if (accepted !== void 0 && lands === "same" && to !== primary) findings.push(`the client wrote to ${to ?? String(accepted["at"])}, where it should stay on ${primary}`);
+	if (accepted !== void 0 && lands === "same" && !cluster && to !== primary) findings.push(`the client wrote to ${to ?? String(accepted["at"])}, where it should stay on ${primary}`);
 	for (const error of expected?.errors ?? []) {
 		const met = group.errors.find((each) => each.name === error.name);
 		const noise = valueIn(noiseOf(cell), "s");
@@ -1614,7 +1668,7 @@ const requestOf = (events) => {
 };
 /** This function reads the socket of one run against its cell, as `measure` does without the second request of an HTTP client. */
 const measureSocket = (cell, run) => {
-	if (cell.group !== void 0 || cell.replicaSet !== void 0) {
+	if (cell.group !== void 0 || cell.replicaSet !== void 0 || cell.cluster !== void 0) {
 		const read = measureGroup(cell, run);
 		return {
 			environment: run.environment,
@@ -2983,7 +3037,7 @@ const viewOf = (group, ips, sentinels, members) => {
 		why: "healthy"
 	};
 };
-const sleep$2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const sleep$3 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** This time is the longest that a group takes to converge before a run, or after it. */
 const CONVERGE_MS = 24e4;
 /**
@@ -3178,7 +3232,7 @@ const openGroupLab = async (group, options = {}) => {
 					}));
 					calm = last.healthy ? calm + 1 : 0;
 					if (calm >= 2 && last.primary !== null) return last.primary;
-					await sleep$2(1e3);
+					await sleep$3(1e3);
 				}
 				throw new Error(`the group did not converge in ${within} ms: ${last.why}`);
 			},
@@ -3256,7 +3310,7 @@ const openGroupLab = async (group, options = {}) => {
 				const started = Date.now();
 				while (await cli(member, ["PING"]).catch(() => "") !== "PONG") {
 					if (Date.now() - started > 3e4) throw new Error(`${member} did not start again`);
-					await sleep$2(200);
+					await sleep$3(200);
 				}
 			},
 			failover(coordinated) {
@@ -3374,9 +3428,9 @@ const openGroupLab = async (group, options = {}) => {
 	}
 };
 /** This count is the number of accepted writes that a run waits for before its fault: the stream is steady then. */
-const STEADY$1 = 20;
+const STEADY$2 = 20;
 /** This time is how long a run continues after its outcome, so that the record shows what came next. */
-const TAIL_MS$2 = 2e3;
+const TAIL_MS$3 = 2e3;
 /** This function gives the address of a member, with the port of the group. */
 const addressOf = (lab, member) => `${lab.members[member].ip}:${lab.group.port}`;
 /** This function gives the latest time that a group cell waits for its outcome, in ms from the fault. */
@@ -3399,7 +3453,7 @@ const clientArgs = (cell, lab) => {
 	];
 };
 /** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
-const waitFor$1 = (client, match, within) => new Promise((resolve) => {
+const waitFor$2 = (client, match, within) => new Promise((resolve) => {
 	const started = Date.now();
 	const look = () => {
 		const found = client.events.find(match);
@@ -3415,7 +3469,7 @@ const subscriptionsOf = (timeline) => {
 	return [...latest.values()];
 };
 /** This function gives the count of lost records in the latest clock line of the prober. */
-const droppedSoFar$2 = (records) => {
+const droppedSoFar$3 = (records) => {
 	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
 	return 0;
 };
@@ -3437,7 +3491,7 @@ const groupOnce = async (lab, cell, environment, index) => {
 	await lab.forget();
 	const from = lab.kernel.length;
 	const timelineFrom = lab.timeline.length;
-	const lostBefore = droppedSoFar$2(lab.kernel);
+	const lostBefore = droppedSoFar$3(lab.kernel);
 	const client = await lab.client(clientArgs(cell, lab));
 	const lands = cell.prediction.group?.lands ?? "new";
 	const place = cell.faultAt ?? { role: "primary" };
@@ -3447,7 +3501,7 @@ const groupOnce = async (lab, cell, environment, index) => {
 	const undo = [];
 	try {
 		await client.until(["ready"], 6e4);
-		if (await waitFor$1(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$1, 3e4) === void 0) throw new Error(`the client did not write ${STEADY$1} times in 30 s`);
+		if (await waitFor$2(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$2, 3e4) === void 0) throw new Error(`the client did not write ${STEADY$2} times in 30 s`);
 		let healing = null;
 		switch (cell.fault) {
 			case "blackhole":
@@ -3461,7 +3515,7 @@ const groupOnce = async (lab, cell, environment, index) => {
 				undo.push(() => lab.lift(kind, part, cuts));
 				if (cell.stall !== void 0) {
 					const t1 = fault.t1;
-					const sent = await waitFor$1(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
+					const sent = await waitFor$2(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
 					if (sent !== void 0) healing = lab.lift(kind, part, cuts, sent.t + valueIn(cell.stall, "s") * 1e9);
 				} else if (cell.hold !== void 0) healing = lab.lift(kind, part, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
 				break;
@@ -3495,11 +3549,11 @@ const groupOnce = async (lab, cell, environment, index) => {
 		const t0 = fault.t0;
 		const old = addressOf(lab, primary);
 		const deadline = deadlineOf(cell);
-		if ((lands === "new" ? await waitFor$1(client, (event) => event.ev === "ack" && event.t > t0 && typeof event["at"] === "string" && event["at"] !== old, deadline) : await (async () => {
+		if ((lands === "new" ? await waitFor$2(client, (event) => event.ev === "ack" && event.t > t0 && typeof event["at"] === "string" && event["at"] !== old, deadline) : await (async () => {
 			await healed;
 			const after = heal?.t0 ?? t0;
-			return waitFor$1(client, (event) => event.ev === "ack" && event.t > after, deadline);
-		})()) !== void 0) await sleep$2(TAIL_MS$2);
+			return waitFor$2(client, (event) => event.ev === "ack" && event.t > after, deadline);
+		})()) !== void 0) await sleep$3(TAIL_MS$3);
 		await healed;
 	} finally {
 		for (const step of undo.reverse()) await step().catch(() => void 0);
@@ -3662,7 +3716,7 @@ const statusViewOf = (set, ips, status) => {
 		why: "healthy"
 	};
 };
-const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const sleep$2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** These are the arguments of replSetStepDown: the time that the old primary cannot stand again, and the wait for a secondary to catch up. */
 const STEP_DOWN = {
 	seconds: 60,
@@ -3803,7 +3857,7 @@ const openReplicaSetLab = async (set) => {
 				calm = last.healthy ? calm + 1 : 0;
 				if (last.primary !== null) known = last.primary;
 				if (calm >= 2 && last.primary !== null) return last.primary;
-				await sleep$1(1e3);
+				await sleep$2(1e3);
 			}
 			throw new Error(`the set did not converge in ${within} ms: ${last.why}`);
 		};
@@ -3909,7 +3963,7 @@ const openReplicaSetLab = async (set) => {
 				const started = Date.now();
 				while (!(await shell(member, "print(db.hello().ok)").catch(() => "")).includes("1")) {
 					if (Date.now() - started > 6e4) throw new Error(`${member} did not start again`);
-					await sleep$1(500);
+					await sleep$2(500);
 				}
 			},
 			stepDown(primary) {
@@ -4004,9 +4058,9 @@ const openReplicaSetLab = async (set) => {
 	}
 };
 /** This count is the number of accepted writes that a run waits for before its fault: the stream is steady then. */
-const STEADY = 20;
+const STEADY$1 = 20;
 /** This time is how long a run continues after its outcome, so that the record shows what came next. */
-const TAIL_MS$1 = 2e3;
+const TAIL_MS$2 = 2e3;
 /** This function gives the arguments of the client of a replica set cell: each member, and the stream of writes. */
 const replicaSetArgs = (cell, members) => {
 	const set = cell.replicaSet;
@@ -4030,7 +4084,7 @@ const replicaSetArgs = (cell, members) => {
 	];
 };
 /** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
-const waitFor = (client, match, within) => new Promise((resolve) => {
+const waitFor$1 = (client, match, within) => new Promise((resolve) => {
 	const started = Date.now();
 	const look = () => {
 		const found = client.events.find(match);
@@ -4040,7 +4094,7 @@ const waitFor = (client, match, within) => new Promise((resolve) => {
 	look();
 });
 /** This function gives the count of lost records in the latest clock line of the prober. */
-const droppedSoFar$1 = (records) => {
+const droppedSoFar$2 = (records) => {
 	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
 	return 0;
 };
@@ -4065,7 +4119,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 	await lab.forget();
 	const from = lab.kernel.length;
 	const logFrom = lab.log.length;
-	const lostBefore = droppedSoFar$1(lab.kernel);
+	const lostBefore = droppedSoFar$2(lab.kernel);
 	const client = await lab.client(replicaSetArgs(cell, lab.members));
 	const lands = cell.prediction.group?.lands ?? "new";
 	const place = cell.faultAt ?? { role: "primary" };
@@ -4075,7 +4129,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 	const undo = [];
 	try {
 		await client.until(["ready"], 6e4);
-		if (await waitFor(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY, 3e4) === void 0) throw new Error(`the client did not write ${STEADY} times in 30 s`);
+		if (await waitFor$1(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$1, 3e4) === void 0) throw new Error(`the client did not write ${STEADY$1} times in 30 s`);
 		let healing = null;
 		switch (cell.fault) {
 			case "blackhole":
@@ -4089,7 +4143,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 				undo.push(() => lab.lift(kind, part, cuts));
 				if (cell.stall !== void 0) {
 					const t1 = fault.t1;
-					const sent = await waitFor(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
+					const sent = await waitFor$1(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
 					healing = lab.lift(kind, part, cuts, (sent?.t ?? t1) + valueIn(cell.stall, "s") * 1e9);
 				} else if (cell.hold !== void 0) healing = lab.lift(kind, part, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
 				break;
@@ -4124,10 +4178,10 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 		const t1 = fault.t1;
 		const old = `${lab.members[primary].ip}:${set.port}`;
 		const sentAt = (n) => client.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
-		if (await waitFor(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1 && (lands === "same" || typeof event["at"] === "string" && event["at"] !== old), deadlineOf(cell)) !== void 0) await sleep$1(TAIL_MS$1);
+		if (await waitFor$1(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1 && (lands === "same" || typeof event["at"] === "string" && event["at"] !== old), deadlineOf(cell)) !== void 0) await sleep$2(TAIL_MS$2);
 		if (healing !== null) {
 			await healed.catch(() => void 0);
-			await sleep$1(TAIL_MS$1);
+			await sleep$2(TAIL_MS$2);
 		}
 	} finally {
 		for (const step of undo.reverse()) await step().catch(() => void 0);
@@ -4159,171 +4213,909 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 	};
 };
 //#endregion
-//#region src/run.ts
+//#region src/cluster.ts
+/** This form is a line of the console log of RabbitMQ 4.3: the time with its offset, the level, the process and the message. */
+const LINE = /^(\d{4}-\d\d-\d\d) (\d\d:\d\d:\d\d)\.(\d{3})(\d*)([+-]\d\d:\d\d) \[(\w+)\] <[^>]*> (.*)$/;
+/** This form is an escape of ANSI: RabbitMQ colors each line of its console by its level. */
+const ANSI = new RegExp(String.fromCharCode(27) + "\\[[0-9;]*m", "g");
+/** This function gives a line of the console without its colors. */
+const plainOf = (text) => text.replace(ANSI, "").trim();
+/** This text is the name of a quorum queue in the log of Ra: its friendly name. */
+const logNameOf = (queue) => `queue '${queue}' in vhost '/'`;
 /**
-* This function measures and judges the raw records of a cell, with the code of the harness at
-* this time. `runCell` uses it during a cell. A holder of the raw records of a cell can use it
-* again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.
+* This function reads one log line of a node, or gives null for a line that a run does not keep. A
+* run keeps each line of the queue, each warning and each error, and each line of the maintenance
+* mode of a drain.
 */
-const judge = (cell, runs) => {
-	const measured = runs.filter((run) => run.warmup !== true).map((run) => measure(cell, run));
+const nodeLineOf = (member, text, queue) => {
+	const found = LINE.exec(plainOf(text));
+	if (found === null) return null;
+	const [, day, time, ms, us, offset, level, msg] = found;
+	if (!(msg.includes(logNameOf(queue)) || level === "warning" || level === "error" || msg.includes("maintenance"))) return null;
+	const real = Date.parse(`${day}T${time}.${ms}${offset}`) + (us === "" ? 0 : Number(`0.${us}`));
+	if (!Number.isFinite(real)) return null;
 	return {
-		measured,
-		verdict: verdictOf(cell.prediction, measured, noiseOf(cell))
+		ev: "rabbit",
+		t: 0,
+		member,
+		real,
+		level,
+		msg
 	};
 };
-/** This function gives the count of lost records in the latest clock line of the prober, since it began. */
-const droppedSoFar = (records) => {
-	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
-	return 0;
-};
-/** This function gives the argv of the client for a cell, with a deadline far after all that it waits for. */
-const probeArgs = (cell, lab) => {
-	const { prediction, probe } = cell;
-	const longest = (prediction.detects === true && prediction.driver !== void 0 ? longestOf(cell) : prediction.detects ? latestEndOf(prediction) : valueIn(prediction.window, "s")) + (cell.idleBefore === void 0 ? 0 : valueIn(cell.idleBefore, "s"));
-	const args = [
-		lab.targetIp,
-		String(lab.targetPort),
-		"--mode",
-		probe.mode
-	];
-	if (probe.ws !== void 0) {
-		const { ws } = probe;
-		const waits = (prediction.detects !== false ? prediction.reconnects?.waits ?? [] : []).reduce((sum, wait) => sum + valueIn(wait.high, "s"), 0);
-		args.push("--websocket", ws.socket);
-		args.push("--heartbeat", ws.heartbeat === null ? "off" : "on");
-		if (ws.heartbeat !== null) args.push("--heartbeat-interval-ms", String(valueIn(ws.heartbeat.interval, "ms")), "--heartbeat-timeout-ms", String(valueIn(ws.heartbeat.timeout, "ms")), "--heartbeat-misses", String(ws.heartbeat.misses));
-		args.push("--reconnect", ws.reconnect === null ? "off" : "on");
-		if (ws.reconnect !== null) args.push("--reconnect-initial-ms", String(valueIn(ws.reconnect.initialDelay, "ms")), "--reconnect-multiplier", String(ws.reconnect.multiplier), "--reconnect-max-ms", String(valueIn(ws.reconnect.maxDelay, "ms")), "--reconnect-attempts", String(ws.reconnect.attempts), "--reconnect-jitter", String(ws.reconnect.jitter));
-		args.push("--request-timeout-ms", String(valueIn(ws.requestTimeout, "ms")));
-		args.push("--connection-timeout-ms", String(valueIn(ws.connectionTimeout, "ms")));
-		if (ws.hold !== void 0) args.push("--hold-ms", String(valueIn(ws.hold, "ms")));
-		args.push("--max-s", String(Math.ceil((longest + waits) * 2 + 10)));
-		return args;
-	}
-	if (probe.http !== void 0) {
-		const { http } = probe;
-		const gap = http.gap === void 0 ? 0 : valueIn(http.gap, "s");
-		args.push("--method", http.method);
-		if (http.gap !== void 0) args.push("--gap-ms", String(valueIn(http.gap, "ms")));
-		if (http.hold !== void 0) args.push("--hold-ms", String(valueIn(http.hold, "ms")));
-		if (http.bytes !== void 0) args.push("--bytes", String(http.bytes));
-		for (const [name, value] of Object.entries(http.options ?? {})) args.push(`--${name}`, String(value));
-		args.push("--linger-ms", String(LINGER_MS));
-		args.push("--max-s", String(Math.ceil(longest * 2 + gap + 10)));
-		return args;
-	}
-	if (probe.request !== void 0) args.push("--request", probe.request);
-	if (probe.count !== void 0) args.push("--count", String(probe.count));
-	if (probe.interval !== void 0) args.push("--interval-ms", String(valueIn(probe.interval, "ms")));
-	if (probe.driver?.name === "mongodb") {
-		args.push("--set", probe.driver.set);
-		if (probe.driver.op !== void 0) args.push("--op", probe.driver.op);
-		if (cell.writeConcern !== void 0) args.push("--w", String(cell.writeConcern));
-		if (cell.before === true) args.push("--before");
-		if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
-		args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
-		return args;
-	}
-	if (probe.driver?.name === "pg" || probe.driver?.name === "hikari" || probe.driver?.name === "amqp") {
-		if (probe.driver.name !== "amqp" && probe.driver.query !== void 0) args.push("--query", probe.driver.query);
-		if (probe.driver.name !== "amqp" && probe.driver.inflight !== void 0) args.push("--inflight-query", probe.driver.inflight);
-		if (cell.before === true) args.push("--before");
-		if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
-		args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
-		return args;
-	}
-	if (probe.driver !== void 0) {
-		args.push("--keepalive-ms", String(valueIn(probe.driver.keepAlive, "ms")));
-		args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
-		return args;
-	}
-	if (probe.keepalive) args.push("--keepalive");
-	if (probe.userTimeout !== void 0) args.push("--user-timeout-ms", String(valueIn(probe.userTimeout, "ms")));
-	args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
-	return args;
-};
+/** This function puts the log lines of the nodes on the monotonic clock, with the offset of a client. */
+const nodesOnMonotonic = (log, offset) => log.map((line) => ({
+	...line,
+	t: offset === null ? 0 : Math.round(line.real * 1e6 - offset)
+}));
 /**
-* This function waits for an error on the socket of the observer, or for its end of file
-* (CLOSE_WAIT, state 8). The prober records both. A cell whose observer is the server, or a proxy
-* of a topology, waits for that. A target with its own detector can also drop the connection
-* itself (`dropped`). The promise resolves after `within` ms at the latest.
+* This function reads the JSON output of a tool of RabbitMQ (`--formatter json`). The tools print a
+* value over one line or over more than one, thus it reads from the first bracket to the end.
 */
-const observerError = (cell, partial, within, dropped = () => false) => new Promise((resolve) => {
-	const started = Date.now();
-	const look = () => {
-		const run = partial();
-		const at = socketOf(cell, run);
-		if (at !== void 0 && run.kernel.some((record) => record.netns === at.netns && record.cookie === at.cookie && (record.ev === "error" && (record.err ?? 0) !== 0 || record.ev === "state" && record.new === 8)) || dropped() || Date.now() - started > within) resolve();
-		else setTimeout(look, 50);
-	};
-	look();
-});
-const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const toolJsonOf = (text) => {
+	const at = text.search(/[[{]/);
+	if (at === -1) throw new Error(`no JSON in the output: ${text.slice(0, 200)}`);
+	return JSON.parse(text.slice(at));
+};
+/** This function gives the name of a node in the Erlang distribution: `rabbit@<name>`. */
+const nodeNameOf = (member) => `rabbit@${member}`;
 /**
-* This time is the pause between the client's first reply and the fault. It is more than Linux's
-* TCP_DELACK_MAX of 200 ms. Thus the delayed ACK of the reply is out before the fault.
+* This function gives the file of rabbitmq.conf of each node. The nodes form the cluster from the list
+* of their names, and the user guest connects from each address. The lines of the cluster come last.
 */
-const QUIET_MS = 250;
+const clusterConf = (cluster) => [
+	"cluster_formation.peer_discovery_backend = classic_config",
+	...cluster.members.map((name, at) => `cluster_formation.classic_config.nodes.${at + 1} = ${nodeNameOf(name)}`),
+	"loopback_users.guest = false",
+	...cluster.conf,
+	""
+].join("\n");
 /**
-* This time is how long an HTTP client waits after its second request, before it exits. The exit
-* closes its sockets, thus that close comes after the window of the cell.
+* This function gives the command of each node. It writes the cookie of the server and of the tools.
+* Then it starts the server as the child of a shell. A fin can then stop the server, and the container
+* stays up.
 */
-const LINGER_MS = 1e3;
-/** These are the lines that end the workload of an HTTP client: the end of its second request, its deadline, or its failure. */
-const HTTP_DONE = [
-	"done",
-	"deadline",
-	"fail"
-];
+const nodeCommand = (cluster) => [
+	"for f in /var/lib/rabbitmq/.erlang.cookie /root/.erlang.cookie; do cp /tmp/cookie $f; chmod 400 $f; done",
+	"chown rabbitmq:rabbitmq /var/lib/rabbitmq/.erlang.cookie",
+	`${cluster.programs.start} & while :; do sleep 3600; done`
+].join("; ");
 /**
-* This function waits for the request that an in-flight client sends on the signal: its second
-* send, after the PING that made it ready. Its stamp is when the request first met the fault.
+* This function judges the status of the queue: its leader, and if the queue is healthy. A healthy
+* queue has one leader and a follower on each other node. Each follower is in the term of the leader,
+* at the commit index of the leader.
 */
-const requestSent = (probe, within) => new Promise((resolve, reject) => {
-	const started = Date.now();
-	const look = () => {
-		const sent = probe.events.filter((event) => event.ev === "sent")[1];
-		if (sent !== void 0) resolve(sent);
-		else if (Date.now() - started > within) reject(/* @__PURE__ */ new Error(`the client sent no request within ${within} ms of being told to`));
-		else setTimeout(look, 1);
+const clusterViewOf = (cluster, rows) => {
+	const byNode = new Map(cluster.members.map((name) => [nodeNameOf(name), name]));
+	const leaders = rows.filter((row) => row["Raft State"] === "leader");
+	if (leaders.length !== 1) return {
+		primary: null,
+		healthy: false,
+		why: `the queue has ${leaders.length} leaders`
 	};
-	look();
-});
-/** This time is how long a run of a driver cell continues after its outcome, so that the record shows what came next. */
-const TAIL_MS = 1e3;
-/** This function waits for the outcome of a run of a driver cell, or for `within` ms. The outcome is the end of its band and of each later band. */
-const driverOutcome = async (cell, current, within) => {
-	const started = Date.now();
-	while (Date.now() - started < within) {
-		if (settled(cell, current())) return;
-		await sleep(250);
+	const leader = leaders[0];
+	const primary = byNode.get(leader["Node Name"]) ?? null;
+	if (primary === null) return {
+		primary: null,
+		healthy: false,
+		why: `no node has the name ${leader["Node Name"]}`
+	};
+	if (rows.length !== cluster.members.length) return {
+		primary,
+		healthy: false,
+		why: `the queue has ${rows.length} members`
+	};
+	for (const row of rows) {
+		if (row !== leader && row["Raft State"] !== "follower") return {
+			primary,
+			healthy: false,
+			why: `${row["Node Name"]} is ${row["Raft State"]}`
+		};
+		if (row.Term !== leader.Term) return {
+			primary,
+			healthy: false,
+			why: `${row["Node Name"]} is in term ${row.Term}`
+		};
+		if (row["Commit Index"] !== leader["Commit Index"]) return {
+			primary,
+			healthy: false,
+			why: `${row["Node Name"]} is behind the leader`
+		};
 	}
+	return {
+		primary,
+		healthy: true,
+		why: "healthy"
+	};
 };
-/** This function gives the longest wait of a run of a driver cell for its outcome, in ms. */
-const driverWithin = (cell) => (longestOf(cell) * 1.2 + 10) * 1e3;
 /**
-* This function does one run of a cell whose fault holds before the client starts (`Cell.before`).
-* The fault is a network fault at its place. The client then starts into it, and the run waits for
-* each band of the driver. Then the fault lifts.
+* This function gives the peers that a fault on a node cuts it from. `all` is each other node and the
+* client, `group` is the other nodes, and `clients` is the client. `one` is the first other node in
+* the sequence of the cluster, and `one-way` is the same node for the segments from the part only.
 */
-const beforeOnce = async (lab, cell, environment, index) => {
-	if (cell.fault !== "rst" && cell.fault !== "blackhole" && cell.fault !== "unreachable") throw new Error(`${cell.name}: a fault before the client starts is a network fault, not ${cell.fault}`);
-	await lab.forget();
-	const from = lab.kernel.length;
-	const middleboxFrom = lab.middlebox.length;
-	const lostBefore = droppedSoFar(lab.kernel);
-	const faultAt = cell.faultAt ?? "target";
-	const kind = cell.fault === "unreachable" ? "reject" : cell.fault;
-	const options = cell.fault === "unreachable" ? { code: cell.icmp ?? "host" } : {};
-	const fault = await lab.fault(kind, faultAt, options);
-	let probe;
+const clusterCutOf = (cluster, members, client, part, cuts) => {
+	const others = cluster.members.filter((name) => name !== part).map((name) => members[name].ip);
+	if (cuts === "clients") return { peers: [client] };
+	if (cuts === "group") return { peers: others };
+	if (cuts === "one") return { peers: others.slice(0, 1) };
+	if (cuts === "one-way") return {
+		peers: others.slice(0, 1),
+		direction: "out"
+	};
+	return { peers: [...others, client] };
+};
+const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/**
+* This function opens one environment of a cluster cell:
+* - one new network, with each node, the client and the admin on it.
+* - each node: the stock image, its server as the child of a shell, with its hostname as the name of
+*   the node. A log consumer reads its console. The nodes form the cluster from the list of their
+*   names, and the lab waits until each node runs in it.
+* - the admin declares the quorum queue on the first node, thus its member there is the first leader.
+* - an injector sidecar for each node, in its network and process namespaces.
+* - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
+* - the prober, which watches the namespace of the client.
+*/
+const openClusterLab = async (cluster) => {
+	const image = await harnessImage();
+	const amqp = await amqpClientImage();
+	const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();
+	const runtime = await getContainerRuntimeClient();
+	const opened = [];
+	let network;
+	const close = async () => {
+		for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
+		await network?.stop().catch(() => void 0);
+	};
+	const ipOf = async (container) => {
+		const ip = (await runtime.container.inspect(runtime.container.getById(container.getId()))).NetworkSettings.Networks[network.getName()]?.IPAddress;
+		if (ip === void 0 || ip === "") throw new Error(`${container.getName()} has no address on ${network.getName()}`);
+		return ip;
+	};
 	try {
-		probe = await lab.probe(probeArgs(cell, lab));
-		const started = probe;
-		const current = () => ({
-			environment,
-			index,
+		network = await new Network().start();
+		const boxes = /* @__PURE__ */ new Map();
+		const injectors = /* @__PURE__ */ new Map();
+		const members = {};
+		const log = [];
+		const raw = {};
+		for (const name of cluster.members) {
+			let partial = "";
+			const box = await new GenericContainer(cluster.image).withNetwork(network).withNetworkAliases(name).withHostname(name).withCopyContentToContainer([{
+				content: clusterConf(cluster),
+				target: "/etc/rabbitmq/conf.d/90-harness.conf"
+			}, {
+				content: cluster.cookie,
+				target: "/tmp/cookie"
+			}]).withCommand([
+				"sh",
+				"-c",
+				nodeCommand(cluster)
+			]).withWaitStrategy(Wait.forLogMessage(cluster.ready)).withStartupTimeout(18e4).withLogConsumer((stream) => stream.on("data", (chunk) => {
+				partial += chunk.toString();
+				const split = partial.split("\n");
+				partial = split.pop() ?? "";
+				for (const text of split) {
+					const tail = raw[name] ??= [];
+					tail.push(text);
+					if (tail.length > 400) tail.shift();
+					const line = nodeLineOf(name, text.trim(), cluster.queue);
+					if (line !== null) log.push(line);
+				}
+			})).start();
+			opened.push(box);
+			boxes.set(name, box);
+			const injector = await withPid(new GenericContainer(image).withNetworkMode(`container:${box.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW"), `container:${box.getId()}`).start();
+			opened.push(injector);
+			injectors.set(name, injector);
+			members[name] = {
+				ip: await ipOf(box),
+				netns: await netnsOf(injector)
+			};
+		}
+		const holder = await new GenericContainer(image).withNetwork(network).start();
+		opened.push(holder);
+		const admin = await new GenericContainer(image).withNetworkMode(`container:${holder.getId()}`).withPrivilegedMode().start();
+		opened.push(admin);
+		const clientIp = await ipOf(holder);
+		const commander = await new GenericContainer(amqp).withNetwork(network).withCommand([
+			"sh",
+			"-c",
+			"while :; do sleep 3600; done"
+		]).start();
+		opened.push(commander);
+		const clientNetns = await netnsOf(admin);
+		const records = lines();
+		const prober = await new GenericContainer(image).withPrivilegedMode().withNetworkMode("none").withCommand(["/opt/harness/prober", String(clientNetns)]).withLogConsumer(records.consume).start();
+		opened.push(prober);
+		const begun = await records.until(["begin", "fail"], 3e4, "prober");
+		if (begun.ev === "fail") throw new Error(`the prober failed: ${String(begun["why"])}`);
+		const box = (name) => {
+			const found = boxes.get(name);
+			if (found === void 0) throw new Error(`the cluster has no node ${name}`);
+			return found;
+		};
+		const injectorOf = (name) => {
+			const found = injectors.get(name);
+			if (found === void 0) throw new Error(`the cluster has no node ${name}`);
+			return found;
+		};
+		const lastLine = (text) => text.split("\n").filter((line) => line.trim() !== "").at(-1) ?? "";
+		const stamped = async (where, argv) => {
+			const stamp = JSON.parse(lastLine(await output(where, argv)));
+			return {
+				t0: stamp.t0,
+				t1: stamp.t1
+			};
+		};
+		const after = (at) => at === void 0 ? [] : ["--at", String(Math.round(at))];
+		/** This function runs a command of the tools of RabbitMQ on a node, between two stamps of its wall clock. */
+		const walled = async (name, command) => {
+			const text = await output(box(name), [
+				"sh",
+				"-c",
+				`date +%s%N; ${command} >&2; date +%s%N`
+			]);
+			const stamps = text.split("\n").map((line) => line.trim()).filter((line) => /^\d{18,}$/.test(line)).map(Number);
+			if (stamps.length < 2) throw new Error(`${name}: no stamps around ${command}: ${text.slice(0, 300)}`);
+			return {
+				wall0: stamps[0],
+				wall1: stamps.at(-1)
+			};
+		};
+		const status = async (name) => {
+			const text = await output(box(name), [
+				"rabbitmq-queues",
+				"-q",
+				"quorum_status",
+				cluster.queue,
+				"--formatter",
+				"json"
+			]);
+			return clusterViewOf(cluster, toolJsonOf(text));
+		};
+		let known = cluster.members[0];
+		/** This function gives the view of the queue: from the node that had the leader last, then from each other node. */
+		const view = async () => {
+			let last = {
+				primary: null,
+				healthy: false,
+				why: "no node answered"
+			};
+			for (const name of [known, ...cluster.members.filter((each) => each !== known)]) {
+				const read = await status(name).catch((error) => ({
+					primary: null,
+					healthy: false,
+					why: `${name}: ${error.message.slice(0, 200)}`
+				}));
+				if (read.primary !== null) return read;
+				last = read;
+			}
+			return last;
+		};
+		const converge = async (within = CONVERGE_MS) => {
+			const started = Date.now();
+			let calm = 0;
+			let last = {
+				primary: null,
+				healthy: false,
+				why: "no view yet"
+			};
+			while (Date.now() - started < within) {
+				last = await view();
+				calm = last.healthy ? calm + 1 : 0;
+				if (last.primary !== null) known = last.primary;
+				if (calm >= 2 && last.primary !== null) return last.primary;
+				await sleep$1(1e3);
+			}
+			throw new Error(`the queue did not converge in ${within} ms: ${last.why}`);
+		};
+		const started = Date.now();
+		for (;;) {
+			const text = await output(box(cluster.members[0]), [
+				"rabbitmq-diagnostics",
+				"-q",
+				"cluster_status",
+				"--formatter",
+				"json"
+			]).catch(() => "{}");
+			const running = (text.trim() === "" ? {} : toolJsonOf(text)).running_nodes ?? [];
+			if (cluster.members.every((name) => running.includes(nodeNameOf(name)))) break;
+			if (Date.now() - started > 12e4) throw new Error(`the cluster did not form in 120 s: ${running.join(", ")} run`);
+			await sleep$1(1e3);
+		}
+		const declared = lastLine(await output(commander, [
+			"node",
+			"/opt/harness/amqp/admin-cluster.mjs",
+			`${members[cluster.members[0]].ip}:${cluster.port}`,
+			"declare",
+			cluster.queue
+		]));
+		if (!declared.includes("\"ok\":true")) throw new Error(`the declare of the queue failed: ${declared}`);
+		await converge();
+		return {
+			cluster,
+			clientIp,
+			members,
+			netns: { client: clientNetns },
+			kernel: records.seen,
+			log,
+			raw,
+			async sysctl(values) {
+				const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
+				if (settings.length > 0) await output(admin, [
+					"sysctl",
+					"-q",
+					"-w",
+					...settings
+				]);
+				const read = {};
+				for (const key of Object.keys(values)) read[key] = await output(admin, [
+					"sysctl",
+					"-n",
+					key
+				]);
+				return read;
+			},
+			converge,
+			async flush(leader) {
+				await output(box(leader), [
+					"rabbitmqctl",
+					"-q",
+					"purge_queue",
+					cluster.queue
+				]);
+			},
+			async holds(leader) {
+				const reply = lastLine(await output(commander, [
+					"node",
+					"/opt/harness/amqp/admin-cluster.mjs",
+					`${members[leader].ip}:${cluster.port}`,
+					"holds",
+					cluster.queue
+				]));
+				const held = JSON.parse(reply).held;
+				if (held === void 0) throw new Error(`the read of the queue failed: ${reply}`);
+				return held.map(Number).filter((n) => Number.isInteger(n) && n > 0);
+			},
+			partOf(role, leader) {
+				if (role === "primary") return leader;
+				if (role === "sentinel") throw new Error("a cluster has no Sentinel");
+				const follower = cluster.members.find((name) => name !== leader);
+				if (follower === void 0) throw new Error("the cluster has no follower");
+				return follower;
+			},
+			async client(args) {
+				const events = lines();
+				const container = await new GenericContainer(amqp).withNetworkMode(`container:${holder.getId()}`).withCommand([
+					"node",
+					"/opt/harness/amqp/client-cluster.mjs",
+					...args
+				]).withLogConsumer(events.consume).start();
+				return {
+					events: events.seen,
+					until: (kinds, within) => events.until(kinds, within, "client"),
+					stop: () => container.stop({ timeout: 2 }).then(() => void 0).catch(() => void 0)
+				};
+			},
+			fault(kind, part, cuts, at) {
+				const cut = clusterCutOf(cluster, members, clientIp, part, cuts);
+				const direction = cut.direction === void 0 ? [] : ["--direction", cut.direction];
+				return stamped(injectorOf(part), [
+					"/opt/harness/inject",
+					kind,
+					cut.peers.join(","),
+					...direction,
+					...after(at)
+				]);
+			},
+			lift(kind, part, cuts, at) {
+				const cut = clusterCutOf(cluster, members, clientIp, part, cuts);
+				const direction = cut.direction === void 0 ? [] : ["--direction", cut.direction];
+				return stamped(injectorOf(part), [
+					"/opt/harness/inject",
+					kind === "rst" ? "clear-rst" : "clear",
+					cut.peers.join(","),
+					...direction,
+					...after(at)
+				]);
+			},
+			signal(part, signal, at) {
+				return stamped(injectorOf(part), [
+					"/opt/harness/inject",
+					"signal",
+					cluster.programs.server,
+					signal,
+					...after(at)
+				]);
+			},
+			async restart(member) {
+				await output(box(member), [
+					"sh",
+					"-c",
+					`${cluster.programs.start} > /proc/1/fd/1 2>&1 &`
+				]);
+				const begun = Date.now();
+				while (!await output(box(member), [
+					"rabbitmq-diagnostics",
+					"-q",
+					"check_running"
+				]).then(() => true).catch(() => false)) {
+					if (Date.now() - begun > 12e4) throw new Error(`${member} did not start again`);
+					await sleep$1(1e3);
+				}
+			},
+			stepDown(leader, to) {
+				return walled(leader, `rabbitmq-queues -q transfer_leadership ${cluster.queue} ${nodeNameOf(to)}`);
+			},
+			drain(member) {
+				return walled(member, "rabbitmq-upgrade -q drain");
+			},
+			async revive(member) {
+				await output(box(member), [
+					"rabbitmq-upgrade",
+					"-q",
+					"revive"
+				]);
+			},
+			async forget() {
+				await output(admin, [
+					"ip",
+					"tcp_metrics",
+					"flush",
+					"all"
+				]);
+			},
+			async settle() {
+				const from = records.seen.length;
+				const clock = await new Promise((resolve) => {
+					const begun = Date.now();
+					const look = () => {
+						const line = records.seen.slice(from).find((record) => record.ev === "clock");
+						if (line !== void 0) resolve(line);
+						else if (Date.now() - begun > 5e3) resolve(null);
+						else setTimeout(look, 50);
+					};
+					look();
+				});
+				return clock === null ? null : Number(clock["lost"] ?? 0);
+			},
+			async manifest(sysctls) {
+				const config = await output(admin, ["zcat", "/proc/config.gz"]).catch(() => "");
+				const inspect = async (container) => runtime.container.inspect(runtime.container.getById(container.getId()));
+				const list = [];
+				for (const name of cluster.members) {
+					const version = lastLine(await output(box(name), [
+						"rabbitmqctl",
+						"-q",
+						"version"
+					]).catch(() => ""));
+					list.push({
+						name,
+						ip: members[name].ip,
+						id: (await inspect(box(name))).Image,
+						settings: {
+							version,
+							conf: cluster.conf.join("; ")
+						}
+					});
+				}
+				return {
+					kernel: await output(holder, ["uname", "-r"]),
+					architecture: await output(holder, ["uname", "-m"]),
+					hz: /^CONFIG_HZ=(\d+)$/m.exec(config)?.[1] ?? null,
+					boot_id: await output(holder, ["cat", "/proc/sys/kernel/random/boot_id"]),
+					docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
+					target: {
+						image: cluster.image,
+						id: list[0].id,
+						settings: list[0].settings,
+						sysctls: {}
+					},
+					harness: {
+						image,
+						id: (await inspect(holder)).Image,
+						packages: (await output(holder, [
+							"apk",
+							"info",
+							"-v"
+						])).split("\n").sort()
+					},
+					client: {
+						kind: "amqp",
+						image: amqp
+					},
+					netns: {
+						client: clientNetns,
+						target: members[cluster.members[0]].netns
+					},
+					sysctls,
+					group: {
+						members: list,
+						sentinels: [],
+						observer: { image: amqp }
+					}
+				};
+			},
+			close
+		};
+	} catch (error) {
+		await close();
+		throw error;
+	}
+};
+/** This count is the number of confirmed publishes that a run waits for before its fault: the stream is steady then. */
+const STEADY = 20;
+/** This time is how long a run continues after its outcome, so that the record shows what came next. */
+const TAIL_MS$1 = 2e3;
+/**
+* This function gives the hosts of the client, in the sequence of its connects: its node first, then
+* each other node. The node of a follower is the last follower in the sequence of the cluster. A fault
+* on one member (`one`, `one-way`) cuts the first follower, thus not the node of the client.
+*/
+const clusterHostsOf = (cell, members, leader) => {
+	const cluster = cell.cluster;
+	const first = cell.clientAt === "leader" ? leader : cluster.members.filter((name) => name !== leader).at(-1);
+	return [first, ...cluster.members.filter((name) => name !== first)].map((name) => `${members[name].ip}:${cluster.port}`);
+};
+/** This form is the NOTICE of a leader of the queue that starts to give its leadership away: to await_condition, for a transfer. */
+const GIVES_AWAY = /: leader -> await_condition in term/;
+/** This form is the INFO of a follower whose monitor of the leader goes down (ra_server_proc.erl:871). A drain stops the leader so. */
+const MONITOR_DOWN = /Leader monitor down/;
+/** This function gives the arguments of the client of a cluster cell: its hosts, the queue, and the stream of publishes. */
+const clusterArgs = (cell, members, leader) => [
+	clusterHostsOf(cell, members, leader).join(","),
+	"--queue",
+	cell.cluster.queue,
+	"--interval-ms",
+	String(cell.probe.interval === void 0 ? 50 : valueIn(cell.probe.interval, "ms")),
+	"--max-s",
+	String(Math.ceil(deadlineOf(cell) / 1e3 + (cell.hold === void 0 ? 0 : valueIn(cell.hold, "s")) + 120)),
+	...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
+];
+/** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
+const waitFor = (client, match, within) => new Promise((resolve) => {
+	const started = Date.now();
+	const look = () => {
+		const found = client.events.find(match);
+		if (found !== void 0 || Date.now() - started > within) resolve(found);
+		else setTimeout(look, 20);
+	};
+	look();
+});
+/** This function gives the count of lost records in the latest clock line of the prober. */
+const droppedSoFar$1 = (records) => {
+	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
+	return 0;
+};
+/** This function puts a wall stamp on the monotonic clock, with the clock lines of the client. Without one it gives null. */
+const monotonicOf = (stamp, events) => {
+	const offset = clockOffsetOf(events);
+	return offset === null ? null : {
+		t0: Math.round(stamp.wall0 - offset),
+		t1: Math.round(stamp.wall1 - offset)
+	};
+};
+/**
+* This function gives the instant of a change of the leader that a command of the tools starts, as the
+* stamps of a fault. The command starts an Erlang VM first, thus its stamps hold more than the change.
+* A line of the log marks the start of the change. For a step-down it is the NOTICE of the old leader
+* (ra_server_proc.erl:1336-1345), and for a drain the INFO of a follower. Its time on the monotonic
+* clock is the stamp. Without that line, the stamps of the command are the stamps.
+*/
+const stampOf = async (lab, client, logFrom, marks, wall) => {
+	const started = Date.now();
+	while (Date.now() - started < 5e3) {
+		const line = nodesOnMonotonic(lab.log.slice(logFrom), clockOffsetOf(client.events)).find(marks);
+		if (line !== void 0 && line.t > 0) return {
+			stamp: {
+				t0: line.t,
+				t1: line.t
+			},
+			line
+		};
+		await sleep$1(100);
+	}
+	return {
+		stamp: monotonicOf(wall, client.events),
+		line: null
+	};
+};
+/**
+* This function puts the stamp of a fault from a line of the log on the monotonic clock again. It uses
+* the offset of all the clock lines of the client, as the timeline of the run does. Thus the line of the
+* stamp is at the stamp. The offset of the clock lines at the fault can be up to 1 ms smaller.
+*/
+const restamped = (line, events) => {
+	const [on] = nodesOnMonotonic([line], clockOffsetOf(events));
+	return on === void 0 || on.t <= 0 ? null : {
+		t0: on.t,
+		t1: on.t
+	};
+};
+/**
+* This function does one run of a cluster cell:
+* - The queue converges, and it is empty. The client connects through its node, and its stream of
+*   publishes gets steady.
+* - The fault occurs on its node, as the leader of this run makes the roles. A fault with a hold
+*   lifts at its instant in the VM. A step-down moves the leadership to a follower, and a drain puts
+*   the node in maintenance mode. The stamp of each of them is a line of the log (`stampOf`).
+* - The run waits for its outcome: the first confirm of a publish sent after the commit of the fault.
+*   Then it waits for the end of the hold, thus the record shows what the cluster did in the whole hold.
+* - Each fault goes, the client stops, and the queue converges again. The sequence numbers that it
+*   then holds tell which publishes it kept. The log lines of the nodes go onto the monotonic clock.
+*/
+const clusterOnce = async (lab, cell, environment, index) => {
+	const leader = await lab.converge();
+	await lab.flush(leader);
+	await lab.forget();
+	const from = lab.kernel.length;
+	const logFrom = lab.log.length;
+	const lostBefore = droppedSoFar$1(lab.kernel);
+	const client = await lab.client(clusterArgs(cell, lab.members, leader));
+	const place = cell.faultAt ?? { role: "primary" };
+	let fault = null;
+	let heal = null;
+	let at = null;
+	let wall = null;
+	let stampLine = null;
+	const undo = [];
+	try {
+		await client.until(["ready"], 6e4);
+		if (await waitFor(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY, 3e4) === void 0) throw new Error(`the client did not publish ${STEADY} times in 30 s`);
+		let healing = null;
+		switch (cell.fault) {
+			case "blackhole":
+			case "partition":
+			case "rst": {
+				const kind = cell.fault === "rst" ? "rst" : "blackhole";
+				const cuts = place.cuts ?? (cell.fault === "partition" ? "group" : "all");
+				const part = lab.partOf(place.role, leader);
+				at = part;
+				fault = await lab.fault(kind, part, cuts);
+				undo.push(() => lab.lift(kind, part, cuts));
+				if (cell.hold !== void 0) healing = lab.lift(kind, part, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
+				break;
+			}
+			case "freeze-process": {
+				const part = lab.partOf(place.role, leader);
+				at = part;
+				fault = await lab.signal(part, "STOP");
+				undo.push(() => lab.signal(part, "CONT"));
+				if (cell.hold !== void 0) healing = lab.signal(part, "CONT", fault.t0 + valueIn(cell.hold, "s") * 1e9);
+				break;
+			}
+			case "fin": {
+				const part = lab.partOf(place.role, leader);
+				at = part;
+				fault = await lab.signal(part, "KILL");
+				undo.push(() => lab.restart(part));
+				break;
+			}
+			case "step-down":
+				at = leader;
+				wall = await lab.stepDown(leader, lab.partOf("replica", leader));
+				({stamp: fault, line: stampLine} = await stampOf(lab, client, logFrom, (line) => line.member === leader && GIVES_AWAY.test(line.msg), wall));
+				break;
+			case "drain": {
+				const part = lab.partOf(place.role, leader);
+				at = part;
+				wall = await lab.drain(part);
+				undo.push(() => lab.revive(part));
+				({stamp: fault, line: stampLine} = await stampOf(lab, client, logFrom, (line) => line.member !== part && MONITOR_DOWN.test(line.msg), wall));
+				break;
+			}
+			default: throw new Error(`a cluster cell has no fault ${cell.fault}`);
+		}
+		if (fault === null) throw new Error("the client wrote no clock line: the stamps of the command have no place on the monotonic clock");
+		const healed = healing === null ? Promise.resolve(null) : healing.then((stamp) => {
+			heal = stamp;
+			undo.length = 0;
+			return stamp;
+		});
+		healed.catch(() => void 0);
+		const t1 = fault.t1;
+		const sentAt = (n) => client.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
+		if (await waitFor(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1, deadlineOf(cell)) !== void 0) await sleep$1(TAIL_MS$1);
+		if (healing !== null) {
+			await healed.catch(() => void 0);
+			await sleep$1(TAIL_MS$1);
+		}
+	} finally {
+		for (const step of undo.reverse()) await step().catch(() => void 0);
+		await client.stop();
+	}
+	const settled = await lab.settle();
+	const lost = settled === null ? null : settled - lostBefore;
+	const after = await lab.converge().catch(() => null);
+	const holds = after === null ? null : await lab.holds(after);
+	return {
+		environment,
+		index,
+		events: [...client.events],
+		fault: stampLine === null ? fault : restamped(stampLine, client.events) ?? fault,
+		...heal === null ? {} : { heal },
+		kernel: lab.kernel.slice(from),
+		netns: lab.netns.client,
+		targetNetns: lab.members[at ?? leader].netns,
+		lost,
+		group: {
+			members: lab.members,
+			sentinels: {},
+			primary: leader,
+			at,
+			timeline: nodesOnMonotonic(lab.log.slice(logFrom), clockOffsetOf(client.events)),
+			after,
+			holds
+		}
+	};
+};
+//#endregion
+//#region src/run.ts
+/**
+* This function measures and judges the raw records of a cell, with the code of the harness at
+* this time. `runCell` uses it during a cell. A holder of the raw records of a cell can use it
+* again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.
+*/
+const judge = (cell, runs) => {
+	const measured = runs.filter((run) => run.warmup !== true).map((run) => measure(cell, run));
+	return {
+		measured,
+		verdict: verdictOf(cell.prediction, measured, noiseOf(cell))
+	};
+};
+/** This function gives the count of lost records in the latest clock line of the prober, since it began. */
+const droppedSoFar = (records) => {
+	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
+	return 0;
+};
+/** This function gives the argv of the client for a cell, with a deadline far after all that it waits for. */
+const probeArgs = (cell, lab) => {
+	const { prediction, probe } = cell;
+	const longest = (prediction.detects === true && prediction.driver !== void 0 ? longestOf(cell) : prediction.detects ? latestEndOf(prediction) : valueIn(prediction.window, "s")) + (cell.idleBefore === void 0 ? 0 : valueIn(cell.idleBefore, "s"));
+	const args = [
+		lab.targetIp,
+		String(lab.targetPort),
+		"--mode",
+		probe.mode
+	];
+	if (probe.ws !== void 0) {
+		const { ws } = probe;
+		const waits = (prediction.detects !== false ? prediction.reconnects?.waits ?? [] : []).reduce((sum, wait) => sum + valueIn(wait.high, "s"), 0);
+		args.push("--websocket", ws.socket);
+		args.push("--heartbeat", ws.heartbeat === null ? "off" : "on");
+		if (ws.heartbeat !== null) args.push("--heartbeat-interval-ms", String(valueIn(ws.heartbeat.interval, "ms")), "--heartbeat-timeout-ms", String(valueIn(ws.heartbeat.timeout, "ms")), "--heartbeat-misses", String(ws.heartbeat.misses));
+		args.push("--reconnect", ws.reconnect === null ? "off" : "on");
+		if (ws.reconnect !== null) args.push("--reconnect-initial-ms", String(valueIn(ws.reconnect.initialDelay, "ms")), "--reconnect-multiplier", String(ws.reconnect.multiplier), "--reconnect-max-ms", String(valueIn(ws.reconnect.maxDelay, "ms")), "--reconnect-attempts", String(ws.reconnect.attempts), "--reconnect-jitter", String(ws.reconnect.jitter));
+		args.push("--request-timeout-ms", String(valueIn(ws.requestTimeout, "ms")));
+		args.push("--connection-timeout-ms", String(valueIn(ws.connectionTimeout, "ms")));
+		if (ws.hold !== void 0) args.push("--hold-ms", String(valueIn(ws.hold, "ms")));
+		args.push("--max-s", String(Math.ceil((longest + waits) * 2 + 10)));
+		return args;
+	}
+	if (probe.http !== void 0) {
+		const { http } = probe;
+		const gap = http.gap === void 0 ? 0 : valueIn(http.gap, "s");
+		args.push("--method", http.method);
+		if (http.gap !== void 0) args.push("--gap-ms", String(valueIn(http.gap, "ms")));
+		if (http.hold !== void 0) args.push("--hold-ms", String(valueIn(http.hold, "ms")));
+		if (http.bytes !== void 0) args.push("--bytes", String(http.bytes));
+		for (const [name, value] of Object.entries(http.options ?? {})) args.push(`--${name}`, String(value));
+		args.push("--linger-ms", String(LINGER_MS));
+		args.push("--max-s", String(Math.ceil(longest * 2 + gap + 10)));
+		return args;
+	}
+	if (probe.request !== void 0) args.push("--request", probe.request);
+	if (probe.count !== void 0) args.push("--count", String(probe.count));
+	if (probe.interval !== void 0) args.push("--interval-ms", String(valueIn(probe.interval, "ms")));
+	if (probe.driver?.name === "mongodb") {
+		args.push("--set", probe.driver.set);
+		if (probe.driver.op !== void 0) args.push("--op", probe.driver.op);
+		if (cell.writeConcern !== void 0) args.push("--w", String(cell.writeConcern));
+		if (cell.before === true) args.push("--before");
+		if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
+		args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
+		return args;
+	}
+	if (probe.driver?.name === "pg" || probe.driver?.name === "hikari" || probe.driver?.name === "amqp") {
+		if (probe.driver.name !== "amqp" && probe.driver.query !== void 0) args.push("--query", probe.driver.query);
+		if (probe.driver.name !== "amqp" && probe.driver.inflight !== void 0) args.push("--inflight-query", probe.driver.inflight);
+		if (cell.before === true) args.push("--before");
+		if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
+		args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
+		return args;
+	}
+	if (probe.driver !== void 0) {
+		args.push("--keepalive-ms", String(valueIn(probe.driver.keepAlive, "ms")));
+		args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
+		return args;
+	}
+	if (probe.keepalive) args.push("--keepalive");
+	if (probe.userTimeout !== void 0) args.push("--user-timeout-ms", String(valueIn(probe.userTimeout, "ms")));
+	args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
+	return args;
+};
+/**
+* This function waits for an error on the socket of the observer, or for its end of file
+* (CLOSE_WAIT, state 8). The prober records both. A cell whose observer is the server, or a proxy
+* of a topology, waits for that. A target with its own detector can also drop the connection
+* itself (`dropped`). The promise resolves after `within` ms at the latest.
+*/
+const observerError = (cell, partial, within, dropped = () => false) => new Promise((resolve) => {
+	const started = Date.now();
+	const look = () => {
+		const run = partial();
+		const at = socketOf(cell, run);
+		if (at !== void 0 && run.kernel.some((record) => record.netns === at.netns && record.cookie === at.cookie && (record.ev === "error" && (record.err ?? 0) !== 0 || record.ev === "state" && record.new === 8)) || dropped() || Date.now() - started > within) resolve();
+		else setTimeout(look, 50);
+	};
+	look();
+});
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/**
+* This time is the pause between the client's first reply and the fault. It is more than Linux's
+* TCP_DELACK_MAX of 200 ms. Thus the delayed ACK of the reply is out before the fault.
+*/
+const QUIET_MS = 250;
+/**
+* This time is how long an HTTP client waits after its second request, before it exits. The exit
+* closes its sockets, thus that close comes after the window of the cell.
+*/
+const LINGER_MS = 1e3;
+/** These are the lines that end the workload of an HTTP client: the end of its second request, its deadline, or its failure. */
+const HTTP_DONE = [
+	"done",
+	"deadline",
+	"fail"
+];
+/**
+* This function waits for the request that an in-flight client sends on the signal: its second
+* send, after the PING that made it ready. Its stamp is when the request first met the fault.
+*/
+const requestSent = (probe, within) => new Promise((resolve, reject) => {
+	const started = Date.now();
+	const look = () => {
+		const sent = probe.events.filter((event) => event.ev === "sent")[1];
+		if (sent !== void 0) resolve(sent);
+		else if (Date.now() - started > within) reject(/* @__PURE__ */ new Error(`the client sent no request within ${within} ms of being told to`));
+		else setTimeout(look, 1);
+	};
+	look();
+});
+/** This time is how long a run of a driver cell continues after its outcome, so that the record shows what came next. */
+const TAIL_MS = 1e3;
+/** This function waits for the outcome of a run of a driver cell, or for `within` ms. The outcome is the end of its band and of each later band. */
+const driverOutcome = async (cell, current, within) => {
+	const started = Date.now();
+	while (Date.now() - started < within) {
+		if (settled(cell, current())) return;
+		await sleep(250);
+	}
+};
+/** This function gives the longest wait of a run of a driver cell for its outcome, in ms. */
+const driverWithin = (cell) => (longestOf(cell) * 1.2 + 10) * 1e3;
+/**
+* This function does one run of a cell whose fault holds before the client starts (`Cell.before`).
+* The fault is a network fault at its place. The client then starts into it, and the run waits for
+* each band of the driver. Then the fault lifts.
+*/
+const beforeOnce = async (lab, cell, environment, index) => {
+	if (cell.fault !== "rst" && cell.fault !== "blackhole" && cell.fault !== "unreachable") throw new Error(`${cell.name}: a fault before the client starts is a network fault, not ${cell.fault}`);
+	await lab.forget();
+	const from = lab.kernel.length;
+	const middleboxFrom = lab.middlebox.length;
+	const lostBefore = droppedSoFar(lab.kernel);
+	const faultAt = cell.faultAt ?? "target";
+	const kind = cell.fault === "unreachable" ? "reject" : cell.fault;
+	const options = cell.fault === "unreachable" ? { code: cell.icmp ?? "host" } : {};
+	const fault = await lab.fault(kind, faultAt, options);
+	let probe;
+	try {
+		probe = await lab.probe(probeArgs(cell, lab));
+		const started = probe;
+		const current = () => ({
+			environment,
+			index,
 			events: started.events,
 			fault,
 			kernel: lab.kernel.slice(from),
@@ -4625,6 +5417,7 @@ const line = (cell, run, read) => `${cell.name}: env ${run.environment} ${run.wa
 const inEnvironment = async (cell, environment, count, say) => {
 	if (cell.group !== void 0) return inGroupEnvironment(cell, environment, count, say);
 	if (cell.replicaSet !== void 0) return inReplicaSetEnvironment(cell, environment, count, say);
+	if (cell.cluster !== void 0) return inClusterEnvironment(cell, environment, count, say);
 	const lab = await openLab(cell.target, cell.client ?? cell.probe.driver?.name ?? "tcp", cell.topology);
 	try {
 		const settings = await lab.configure(cell.targetSettings ?? {});
@@ -4717,6 +5510,34 @@ const inReplicaSetEnvironment = async (cell, environment, count, say) => {
 	}
 };
 /**
+* This function does the part of the runs of a cluster cell in one environment. The warm-up is a
+* whole run with its fault, as for a replica set. The queue converges before each run.
+*/
+const inClusterEnvironment = async (cell, environment, count, say) => {
+	const lab = await openClusterLab(cell.cluster);
+	try {
+		const manifest = await lab.manifest(await lab.sysctl(cell.sysctls));
+		const warmup = {
+			...await clusterOnce(lab, cell, environment, -1),
+			warmup: true
+		};
+		say(line(cell, warmup, measure(cell, warmup)));
+		const runs = [];
+		for (let index = 0; index < count; index += 1) {
+			const run = await clusterOnce(lab, cell, environment, index);
+			runs.push(run);
+			say(line(cell, run, measure(cell, run)));
+		}
+		return {
+			manifest,
+			runs,
+			warmup
+		};
+	} finally {
+		await lab.close();
+	}
+};
+/**
 * This function does the runs of a cell. They go across new environments, and each environment records
 * its manifest. Then the verdict judges the runs. A median across the edge of the band extends
 * the cell from 11 runs to 22, and then to 44, each extension in new environments. With a
@@ -4770,6 +5591,6 @@ const runCell = async (cell, say = () => {}) => {
 	}
 };
 //#endregion
-export { CLIENT_NAMES, CONVERGE_MS, INJECT, KEPT, LINGER_MS, LISTING, MEMBER_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STATUS_SCRIPT, STEPS, STEP_DOWN, SUBJECTS, callOf, clockOffsetOf, closeOrExit, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, dropFinding, dropOf, endpointAfter, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, goHttpImage, groupOnce, groupVerdictOf, harnessImage, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isHopPlace, judge, lagsOf, latestEndOf, leftAfter, matches, measure, measureDriver, measureGroup, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, nodeClientImage, nodeHttpImage, noiseOf, onMonotonic, openGroupLab, openLab, openReplicaSetLab, peersOf, pgClientImage, pingsOf, planOf, proceduresImage, programLines, programOf, quantileOf, reconnectFinding, reconnectOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, routerImage, routesOf, runCell, sentinelConfig, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, statusViewOf, subjectClient, subjectProblems, subscriptionsOf, targetImageOf, targetSourceOf, templated, toxicBody, unlockedManifests, unpinnedBases, verdictOf, viewOf, writesOf };
+export { CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, GIVES_AWAY, INJECT, KEPT, LINGER_MS, LISTING, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STATUS_SCRIPT, STEPS, STEP_DOWN, SUBJECTS, amqpClientImage, callOf, clockOffsetOf, closeOrExit, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, dropFinding, dropOf, endpointAfter, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, goHttpImage, groupOnce, groupVerdictOf, harnessImage, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isHopPlace, judge, lagsOf, latestEndOf, leftAfter, logNameOf, matches, measure, measureDriver, measureGroup, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicOf, nodeClientImage, nodeCommand, nodeHttpImage, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, onMonotonic, openClusterLab, openGroupLab, openLab, openReplicaSetLab, peersOf, pgClientImage, pingsOf, plainOf, planOf, proceduresImage, programLines, programOf, quantileOf, reconnectFinding, reconnectOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routerImage, routesOf, runCell, sentinelConfig, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, statusViewOf, subjectClient, subjectProblems, subscriptionsOf, targetImageOf, targetSourceOf, templated, toolJsonOf, toxicBody, unlockedManifests, unpinnedBases, verdictOf, viewOf, writesOf };
 
 //# sourceMappingURL=index.js.map