@mark1russell7/harness 0.4.0 → 0.5.0

package/build/index.js

@@ -5,6 +5,10 @@ import { createRequire } from "node:module";
 import { tmpdir } from "node:os";
 import { dirname, join, relative } from "node:path";
 import { fileURLToPath } from "node:url";
+//#region src/cell.ts
+/** This function tells if a fault place is a hop of a topology. */
+const isHopPlace = (place) => typeof place === "object" && "hop" in place;
+//#endregion
 //#region src/verdict.ts
 /**
 * This amount is the noise of the pipeline itself, δ_abs, as the calibration measured it. It is
@@ -111,6 +115,94 @@ const middle = (sorted) => {
 * which does not exist at this time.
 */
 const verdictOf = (prediction, runs) => {
+	const verdict = verdictOfBand(prediction, runs);
+	const counted = runs.filter((run) => run.void === null);
+	if (prediction.group === void 0 && !counted.some((run) => run.group !== void 0)) return verdict;
+	return {
+		...verdict,
+		group: groupVerdictOf(prediction, counted)
+	};
+};
+/** This function gives the median of some counts, or null for none. */
+const medianCount = (values) => values.length === 0 ? null : middle([...values].sort((a, b) => a - b));
+/** This function sums what the group of a group cell did in its counted runs, and tests each stage that has a band. */
+const groupVerdictOf = (prediction, counted) => {
+	const noise = valueIn(NOISE, "s");
+	const stamps = prediction.group?.stamps === void 0 ? 0 : valueIn(prediction.group.stamps, "s");
+	const stages = (prediction.group?.stages ?? []).map((band) => {
+		const interval = {
+			low: s(valueIn(band.low, "s") - noise),
+			high: s(valueIn(band.high, "s") + stamps + noise)
+		};
+		const outside = [];
+		const values = [];
+		for (const run of counted) {
+			const at = run.group?.stages[band.stage];
+			if (at === void 0) {
+				outside.push({
+					run: name(run),
+					why: `no Sentinel announced ${band.stage}`
+				});
+				continue;
+			}
+			const value = valueIn(at, "s");
+			values.push(value);
+			if (value < valueIn(interval.low, "s") || value > valueIn(interval.high, "s")) outside.push({
+				run: name(run),
+				why: `${band.stage} at ${value} s is outside [${interval.low.value}, ${interval.high.value}] s`
+			});
+		}
+		values.sort((a, b) => a - b);
+		return {
+			stage: band.stage,
+			band: interval,
+			median: values.length === 0 ? null : s(middle(values)),
+			outside,
+			decision: outside.length === 0 && values.length > 0 ? "holds" : "fails"
+		};
+	});
+	const medians = {};
+	const seen = new Set(counted.flatMap((run) => Object.keys(run.group?.stages ?? {})));
+	for (const stage of seen) {
+		const values = counted.flatMap((run) => {
+			const at = run.group?.stages[stage];
+			return at === void 0 ? [] : [valueIn(at, "s")];
+		});
+		medians[stage] = s(middle(values.sort((a, b) => a - b)));
+	}
+	const lostCounts = counted.flatMap((run) => run.group?.writes.lost === null || run.group === void 0 ? [] : [run.group.writes.lost.length]);
+	const oldCounts = counted.flatMap((run) => run.group === void 0 ? [] : [run.group.writes.old]);
+	const unknown = counted.flatMap((run) => run.group?.writes.unknown ?? []);
+	const errors = /* @__PURE__ */ new Map();
+	for (const run of counted) for (const error of run.group?.errors ?? []) errors.set(error.name, [...errors.get(error.name) ?? [], valueIn(error.first, "s")]);
+	const lags = counted.flatMap((run) => run.group?.lags ?? []);
+	return {
+		stages,
+		medians,
+		lost: {
+			runs: lostCounts.length,
+			total: lostCounts.reduce((sum, n) => sum + n, 0),
+			median: medianCount(lostCounts),
+			max: lostCounts.length === 0 ? null : Math.max(...lostCounts)
+		},
+		old: {
+			median: medianCount(oldCounts),
+			max: oldCounts.length === 0 ? null : Math.max(...oldCounts)
+		},
+		unknown: {
+			total: unknown.length,
+			held: unknown.filter((each) => each.held === true).length
+		},
+		errors: [...errors].map(([error, firsts]) => ({
+			name: error,
+			runs: firsts.length,
+			first: s(middle(firsts.sort((a, b) => a - b)))
+		})),
+		...lags.length === 0 ? {} : { lags: spreadOf(lags) }
+	};
+};
+/** This function gives the verdict of the band of a cell: its containment and the equivalence of its median. */
+const verdictOfBand = (prediction, runs) => {
 	const counted = runs.filter((run) => run.void === null);
 	const voided = runs.length - counted.length;
 	const outside = counted.filter((run) => run.finding !== null).map((run) => ({
@@ -270,6 +362,252 @@ const verdictOf = (prediction, runs) => {
 	};
 };
 //#endregion
+//#region src/timeline.ts
+/**
+* This module measures a run of a group cell. It gives the first write that a primary accepted
+* after the fault, and the timeline of the group as its Sentinels announced it. It also gives what
+* became of each write.
+*/
+/** These are the stages of a timeline, in their sequence after a fault. */
+const STAGES = [
+	"+sdown",
+	"+odown",
+	"+try-failover",
+	"+elected-leader",
+	"+selected-slave",
+	"+promoted-slave",
+	"+switch-master",
+	"+convert-to-slave"
+];
+const between$1 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
+/** This function gives the member whose address and port is `at`, or null. */
+const memberAt = (run, at) => {
+	if (typeof at !== "string") return null;
+	const ip = at.split(":")[0];
+	return Object.entries(run.group?.members ?? {}).find(([, part]) => part.ip === ip)?.[0] ?? null;
+};
+/** This function gives the time of each stage of a timeline from an instant: the first announcement of each, after the instant. */
+const stagesOf = (timeline, from) => {
+	const out = {};
+	for (const line of timeline) {
+		if (line.ev !== "sentinel" || line.t < from) continue;
+		const stage = STAGES.find((each) => each === line.event);
+		if (stage === void 0 || out[stage] !== void 0) continue;
+		if ((stage === "+sdown" || stage === "+odown") && !String(line.detail ?? "").startsWith("master ")) continue;
+		out[stage] = between$1(from, line.t);
+	}
+	return out;
+};
+/** This function gives what became of each write of a run, from the lines of the client and the keys that the group kept. */
+const writesOf = (run, after, primary) => {
+	const { events } = run;
+	const sent = events.filter((event) => event.ev === "sent");
+	const done = events.find((event) => event.ev === "done")?.t ?? sent.at(-1)?.t ?? Infinity;
+	const acks = /* @__PURE__ */ new Map();
+	for (const event of events) if (event.ev === "ack") acks.set(Number(event["n"]), event);
+	const holds = run.group?.holds ?? null;
+	const kept = holds === null ? null : new Set(holds);
+	const old = [...acks.values()].filter((event) => event.t > after && memberAt(run, event["at"]) === primary).length;
+	const lost = kept === null ? null : [...acks.keys()].filter((n) => !kept.has(n)).sort((a, b) => a - b);
+	const unknown = sent.filter((event) => event.t < done && !acks.has(Number(event["n"]))).map((event) => {
+		const n = Number(event["n"]);
+		return {
+			n,
+			held: kept === null ? null : kept.has(n)
+		};
+	});
+	return {
+		sent: sent.length,
+		acked: acks.size,
+		old,
+		lost,
+		unknown
+	};
+};
+/** This function gives the errors that the client met after an instant: each kind, its first time, and its count. */
+const errorsOf = (events, after) => {
+	const done = events.find((event) => event.ev === "done")?.t ?? Infinity;
+	const kinds = /* @__PURE__ */ new Map();
+	for (const event of events) {
+		if (event.t <= after || event.t >= done) continue;
+		const name = event.ev === "error" ? String(event["errno"] ?? event["name"] ?? "error") : event.ev === "fail" ? String(event["name"] ?? "fail") : event.ev === "sentinel-error" ? "sentinel-error" : null;
+		if (name === null) continue;
+		const seen = kinds.get(name);
+		if (seen === void 0) kinds.set(name, {
+			first: event.t,
+			count: 1
+		});
+		else seen.count += 1;
+	}
+	return [...kinds].map(([name, { first, count }]) => ({
+		name,
+		first: between$1(after, first),
+		count
+	}));
+};
+/**
+* This function gives the delay of each stamp of the observer after the write of its Sentinel that
+* the observer read. That write is the latest write of the Sentinel's socket toward the observer,
+* at the stamp or before it. The prober records each write (`tx`) on the clock of the stamps.
+*/
+const lagsOf = (run, kernel) => {
+	const timeline = run.group?.timeline ?? [];
+	const locals = /* @__PURE__ */ new Map();
+	for (const line of timeline) if (line.ev === "subscribed") locals.set(String(line.sentinel), String(line["local"]));
+	const out = [];
+	for (const line of timeline) {
+		if (line.ev !== "sentinel") continue;
+		const sentinel = String(line.sentinel);
+		const local = locals.get(sentinel);
+		if (local === void 0) continue;
+		const write = kernel.filter((record) => record.ev === "tx" && record.src === sentinel && record.dst === local && record.mono <= line.t).at(-1);
+		if (write !== void 0) out.push(between$1(write.mono, line.t));
+	}
+	return out;
+};
+/**
+* This function reads one run of a group cell.
+* - The reference is the start of the commit of the fault, as the injector stamped it. For a
+*   stall, it is the first write into the fault, as for the stall of a direct pair.
+* - The outcome is the first write that a primary accepted after the fault. The prediction says
+*   which primary: a new one, or the same one after the lift.
+* - Its time is the last segment with data that the client's socket to that primary received
+*   before the client saw the acceptance. The stamp of the client is the fallback.
+* - A run is void when the prober lost records, or when the client did not become ready. It is
+*   also void when a Sentinel entered its TILT mode, or when the election split its votes. A band
+*   from the fault whose commit took longer than δ_abs is void too. So is a band whose hold lifted
+*   more than δ_abs late.
+*/
+const measureGroup = (cell, run) => {
+	const { events, kernel } = run;
+	const { prediction } = cell;
+	const expected = prediction.group;
+	const primary = run.group?.primary ?? "";
+	const ready = events.find((event) => event.ev === "ready");
+	const t0 = run.fault?.t0;
+	const empty = {
+		from: primary,
+		to: null,
+		stages: {},
+		writes: {
+			sent: 0,
+			acked: 0,
+			old: 0,
+			lost: null,
+			unknown: []
+		},
+		errors: []
+	};
+	const silent = {
+		outcome: { kind: "silent" },
+		elapsed: null,
+		socket: null,
+		noticed: null,
+		finding: null
+	};
+	if (ready === void 0) return {
+		...silent,
+		void: "the client never became ready",
+		group: empty
+	};
+	if (t0 === void 0 || run.fault === null) return {
+		...silent,
+		void: "the fault never committed",
+		group: empty
+	};
+	if (run.lost === null) return {
+		...silent,
+		void: "the prober wrote no clock line after the run, so nothing says what it dropped",
+		group: empty
+	};
+	if (run.lost > 0) return {
+		...silent,
+		void: `the prober dropped ${run.lost} records`,
+		group: empty
+	};
+	const timeline = run.group?.timeline ?? [];
+	const stages = stagesOf(timeline, t0);
+	const heal = run.heal ?? null;
+	const lands = expected?.lands ?? "new";
+	const after = lands === "same" && heal !== null ? heal.t0 : t0;
+	const accepted = events.find((event) => {
+		if (event.ev !== "ack" || event.t <= after) return false;
+		const member = memberAt(run, event["at"]);
+		return member !== null && (lands === "new" ? member !== primary : true);
+	});
+	const to = accepted === void 0 ? null : memberAt(run, accepted["at"]);
+	const group = {
+		from: primary,
+		to,
+		stages,
+		writes: writesOf(run, t0, primary),
+		errors: errorsOf(events, t0),
+		...cell.calibrates === "sentinels" ? { lags: lagsOf(run, kernel) } : {}
+	};
+	let voided = null;
+	const tilt = timeline.find((line) => line.ev === "sentinel" && line.event === "+tilt" && line.t >= t0);
+	if (tilt !== void 0) voided = `a Sentinel entered TILT ${between$1(t0, tilt.t).value} s after the fault: the host delayed it, not its timers`;
+	const split = timeline.find((line) => line.ev === "sentinel" && line.event === "-failover-abort-not-elected" && line.t >= t0);
+	const elected = timeline.find((line) => line.ev === "sentinel" && line.event === "+elected-leader" && line.t >= t0);
+	if (voided === null && split !== void 0 && (elected === void 0 || elected.t > split.t)) voided = "the election split its votes and runs again after 2 × failover-timeout, which the band does not bound";
+	const odown = stages["+odown"];
+	if (voided === null && odown !== void 0 && stages["+try-failover"] === void 0) voided = "no Sentinel started a failover after +odown: each was in the hold-off of 2 × failover-timeout after an earlier attempt";
+	if (voided === null && cell.hold !== void 0 && prediction.detects !== null) {
+		if (heal === null) voided = "the fault never lifted";
+		else {
+			const late = (heal.t0 - t0) / 1e9 - valueIn(cell.hold, "s");
+			if (late > valueIn(NOISE, "s")) voided = `the lift came ${Math.round(late * 1e4) / 10} ms after its hold: the injector was late, not the group`;
+		}
+	}
+	const assumed = prediction.detects === null ? void 0 : prediction.rtt;
+	const first = kernel.find((record) => record.netns === run.netns && record.dst === `${run.group?.members[primary]?.ip}:${cell.group?.port}` && record.tcp?.srtt !== void 0);
+	const rtt = first?.tcp?.srtt === void 0 ? null : valueIn(first.tcp.srtt, "us") / 1e3;
+	if (voided === null && assumed !== void 0 && rtt !== null && rtt > valueIn(assumed.high, "ms")) voided = `rtt-over-assumption: its first round trip was ${rtt} ms, above the ${valueIn(assumed.high, "ms")} ms that the prediction assumes`;
+	if (voided === null && prediction.detects === true && prediction.from === "fault" && (run.fault.t1 - run.fault.t0) / 1e6 > valueIn(NOISE, "ms")) voided = `the commit of the fault took ${Math.round((run.fault.t1 - run.fault.t0) / 1e5) / 10} ms, longer than the noise of the pipeline: its start is not known`;
+	let reference = t0;
+	if (prediction.detects === false && prediction.recovers !== void 0) {
+		const into = events.find((event) => event.ev === "sent" && event.t > run.fault.t1);
+		if (into === void 0 || heal !== null && into.t > heal.t0) voided ??= "the write in flight went out before the fault held";
+		else {
+			reference = into.t;
+			const { heal: bounds } = prediction.recovers;
+			if (heal === null) voided ??= "the stall never ended";
+			else if (valueIn(between$1(reference, heal.t0), "s") <= valueIn(bounds.low, "s") || valueIn(between$1(reference, heal.t1), "s") >= valueIn(bounds.high, "s")) voided ??= `the stall ended ${between$1(reference, heal.t0).value}–${between$1(reference, heal.t1).value} s after the write, outside the ${bounds.low.value}–${bounds.high.value} s its prediction assumed`;
+		}
+	}
+	const findings = [];
+	const switched = stages["+switch-master"];
+	if (expected?.failover === false && switched !== void 0) findings.push(`the group failed over ${switched.value} s after the fault, where it should not`);
+	if (accepted !== void 0 && lands === "same" && to !== primary) findings.push(`the client wrote to ${to ?? String(accepted["at"])}, where it should stay on ${primary}`);
+	for (const error of expected?.errors ?? []) {
+		const met = group.errors.find((each) => each.name === error.name);
+		const noise = valueIn(NOISE, "s");
+		if (met === void 0) findings.push(`the client never met ${error.name}`);
+		else if (met.first.value < valueIn(error.low, "s") - noise || met.first.value > valueIn(error.high, "s") + noise) findings.push(`the client met ${error.name} ${met.first.value} s after the fault, outside [${valueIn(error.low, "s")}, ${valueIn(error.high, "s")}] s`);
+	}
+	if (accepted === void 0) {
+		if (prediction.detects !== null) findings.push(lands === "new" ? "the client never wrote to a new primary in the window" : "the client never wrote again in the window");
+		return {
+			...silent,
+			void: voided,
+			finding: findings[0] ?? null,
+			group
+		};
+	}
+	const address = String(accepted["at"]);
+	const segment = kernel.filter((record) => record.netns === run.netns && record.ev === "rx" && record.dst === address && (record.seg?.len.value ?? 0) > 0 && record.mono <= accepted.t && record.mono > after).at(-1);
+	const at = segment?.mono ?? accepted.t;
+	return {
+		outcome: { kind: "answered" },
+		elapsed: between$1(reference, at),
+		socket: between$1(reference, accepted.t),
+		noticed: segment === void 0 ? null : between$1(segment.mono, accepted.t),
+		void: voided,
+		finding: findings[0] ?? null,
+		group
+	};
+};
+//#endregion
 //#region src/observe.ts
 /** These are the errno numbers that the kernel reports to a socket, by the names that predictions use. */
 const ERRNO = {
@@ -425,6 +763,24 @@ const crossingsOf = (run, hop, toClient, toTarget) => {
 *   prober lost is also void.
 */
 const measure = (cell, run) => {
+	if (cell.group !== void 0) {
+		const read = measureGroup(cell, run);
+		return {
+			environment: run.environment,
+			index: run.index,
+			outcome: read.outcome,
+			elapsed: read.elapsed,
+			socket: read.socket,
+			noticed: read.noticed,
+			steps: [],
+			probes: 0,
+			retransmissions: run.kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length,
+			acks: read.group.writes.acked,
+			void: read.void,
+			finding: read.finding,
+			group: read.group
+		};
+	}
 	const { events, kernel } = run;
 	const ready = events.find((event) => event.ev === "ready");
 	const local = events.find((event) => event.ev === "connected")?.["local"];
@@ -651,13 +1007,25 @@ const measure = (cell, run) => {
 	if (!prediction.detects) {
 		const window = valueIn(prediction.window, "s") * 1e9;
 		if (ended === void 0 || ended - request.t > window) {
-			const forgot = forgotten !== void 0 && forgotten.t - request.t <= window ? `a router forgot the flow ${between(lastRx, forgotten.t).value} s after the last segment, where it should keep it` : null;
+			const forgetIn = forgotten !== void 0 && forgotten.t - request.t <= window ? forgotten : void 0;
+			const expected = prediction.middlebox?.forgets;
+			const after = forgetIn === void 0 ? null : between(lastRx, forgetIn.t);
+			let finding = null;
+			if (expected === void 0) {
+				if (after !== null) finding = `a router forgot the flow ${after.value} s after the last segment, where it should keep it`;
+			} else if (after === null) finding = "no router forgot the flow in the window, where it should forget it";
+			else {
+				const noise = valueIn(NOISE, "s");
+				const low = valueIn(expected.low, "s") - noise;
+				const high = valueIn(expected.high, "s") + noise;
+				if (after.value < low || after.value > high) finding = `a router forgot the flow ${after.value} s after the last segment, outside [${Math.round(low * 1e6) / 1e6}, ${Math.round(high * 1e6) / 1e6}] s`;
+			}
 			return {
 				...base,
 				...nothing,
 				...chain,
 				void: voided,
-				finding: forgot
+				finding
 			};
 		}
 		return {
@@ -888,6 +1256,11 @@ const lines = (also) => {
 		}
 	};
 };
+/** This function makes a container share the process namespace of another container, for example `container:<id>`. */
+const withPid = (container, mode) => {
+	container.hostConfig.PidMode = mode;
+	return container;
+};
 /** This container class can share the process namespace of another container. */
 const shared = (Base) => class extends Base {
 	withPidMode(mode) {
@@ -1207,17 +1580,22 @@ const openLab = async (target, kind = "tcp", topology) => {
 		* on its sockets. A router forwards past INPUT and OUTPUT, thus its rules are FORWARD rules on
 		* its device toward the side (`--via`).
 		*/
-		const placeOf = (at) => {
-			if (at === "client") return [
+		const hopPlace = (at) => {
+			if (!isHopPlace(at)) throw new Error(`a lab of a topology has no part ${JSON.stringify(at)}: a group cell opens a group lab`);
+			return at;
+		};
+		const placeOf = (place) => {
+			if (place === "client") return [
 				admin,
 				targetIp,
 				[]
 			];
-			if (at === "target") return [
+			if (place === "target") return [
 				injector,
 				targetPeer,
 				[]
 			];
+			const at = hopPlace(place);
 			const hop = hopNamed(at.hop);
 			const peer = at.toward === "client" ? hop.peer.client : hop.peer.target.split(":")[0];
 			const via = hop.hop.kind === "router" ? ["--via", hop.devices[at.toward]] : [];
@@ -1228,9 +1606,10 @@ const openLab = async (target, kind = "tcp", topology) => {
 			];
 		};
 		/** This function gives the namespace and the device of netem at a place: an end, or the device of a hop toward a side. */
-		const deviceAt = (at) => {
-			if (at === "client") return [admin, "eth0"];
-			if (at === "target") return [injector, "eth0"];
+		const deviceAt = (place) => {
+			if (place === "client") return [admin, "eth0"];
+			if (place === "target") return [injector, "eth0"];
+			const at = hopPlace(place);
 			const hop = hopNamed(at.hop);
 			return [hop.injector, hop.devices[at.toward]];
 		};
@@ -1340,7 +1719,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 				return stamped(where, args);
 			},
 			async offload(on, at = "target") {
-				const devices = typeof at === "object" ? Object.values(hopNamed(at.hop).devices) : [deviceAt(at)[1]];
+				const devices = isHopPlace(at) ? Object.values(hopNamed(at.hop).devices) : [deviceAt(at)[1]];
 				const [where] = deviceAt(at);
 				for (const device of devices) await output(where, [
 					"/opt/harness/inject",
@@ -1547,6 +1926,673 @@ const openLab = async (target, kind = "tcp", topology) => {
 	}
 };
 //#endregion
+//#region src/group.ts
+/** This function gives the configuration file of a member: its port, its settings, and the primary that a replica follows. */
+const memberConfig = (group, primary) => [
+	`port ${group.port}`,
+	...Object.entries(group.memberSettings).map(([key, value]) => `${key} ${value}`),
+	...primary === null ? [] : [`replicaof ${primary} ${group.port}`],
+	""
+].join("\n");
+/** This function gives the configuration file of a Sentinel: its port, the group that it monitors, and the settings of the group. */
+const sentinelConfig = (group, primary) => [
+	`port ${group.sentinelPort}`,
+	`sentinel monitor ${group.name} ${primary} ${group.port} ${group.quorum}`,
+	...Object.entries(group.sentinelSettings).map(([key, value]) => `sentinel ${key} ${group.name} ${value}`),
+	""
+].join("\n");
+/**
+* This function gives the peers that a fault on a part cuts it from, by their addresses. `all` is
+* each other part and the client, `group` is the other parts, and `clients` is the client. The
+* observer is not a peer, thus the timeline of the group stays whole.
+*/
+const peersOf = (parts, client, part, cuts) => {
+	const others = [...Object.entries(parts.members), ...Object.entries(parts.sentinels)].filter(([name]) => name !== part).map(([, at]) => at.ip);
+	if (cuts === "clients") return [client];
+	if (cuts === "group") return others;
+	return [...others, client];
+};
+/** This function reads the flat reply of a command line client into its fields: name, value, name, value. */
+const fieldsOf = (text) => {
+	const out = [];
+	const words = text.split("\n").map((line) => line.replace(/\r$/, ""));
+	let current = null;
+	for (let at = 0; at + 1 < words.length; at += 2) {
+		const [key, value] = [words[at], words[at + 1]];
+		if (key === "name" || current === null) {
+			current = {};
+			out.push(current);
+		}
+		current[key] = value;
+	}
+	return out;
+};
+/** This function reads the INFO of a server into its fields. */
+const infoOf = (text) => {
+	const out = {};
+	for (const line of text.split("\n")) {
+		const at = line.indexOf(":");
+		if (line.startsWith("#") || at === -1) continue;
+		out[line.slice(0, at)] = line.slice(at + 1).trim();
+	}
+	return out;
+};
+/** This function judges the views of the Sentinels and of the members: the primary that they agree on, if the group is healthy. */
+const viewOf = (group, ips, sentinels, members) => {
+	const others = String(group.sentinels.length - 1);
+	const replicas = group.members.length - 1;
+	const addresses = new Set(sentinels.map((each) => each.master["ip"]));
+	if (addresses.size !== 1) return {
+		primary: null,
+		healthy: false,
+		why: `the Sentinels name ${addresses.size} primaries`
+	};
+	const ip = [...addresses][0];
+	const primary = Object.entries(ips).find(([, at]) => at === ip)?.[0] ?? null;
+	if (primary === null) return {
+		primary: null,
+		healthy: false,
+		why: `no member has the address ${ip}`
+	};
+	for (const [at, each] of sentinels.entries()) {
+		if (each.master["flags"] !== "master") return {
+			primary,
+			healthy: false,
+			why: `Sentinel ${at} flags the primary ${each.master["flags"]}`
+		};
+		if (each.master["num-other-sentinels"] !== others) return {
+			primary,
+			healthy: false,
+			why: `Sentinel ${at} knows ${each.master["num-other-sentinels"]} other Sentinels`
+		};
+		const good = each.replicas.filter((replica) => replica["flags"] === "slave" && replica["master-link-status"] === "ok" && replica["master-host"] === ip);
+		if (good.length !== replicas || each.replicas.length !== replicas) return {
+			primary,
+			healthy: false,
+			why: `Sentinel ${at} has ${good.length} healthy replicas of ${each.replicas.length}`
+		};
+	}
+	const info = members[primary] ?? {};
+	if (info["role"] !== "master") return {
+		primary,
+		healthy: false,
+		why: `${primary} reports the role ${info["role"]}`
+	};
+	if (info["master_failover_state"] !== void 0 && info["master_failover_state"] !== "no-failover") return {
+		primary,
+		healthy: false,
+		why: `${primary} is in the failover state ${info["master_failover_state"]}`
+	};
+	const online = Object.entries(info).filter(([key, value]) => /^slave\d+$/.test(key) && value.includes("state=online")).length;
+	if (online !== replicas) return {
+		primary,
+		healthy: false,
+		why: `${primary} has ${online} replicas online`
+	};
+	for (const name of group.members) {
+		if (name === primary) continue;
+		const each = members[name] ?? {};
+		if (each["role"] !== "slave" || each["master_host"] !== ip || each["master_link_status"] !== "up") return {
+			primary,
+			healthy: false,
+			why: `${name} is not a replica of ${primary} with its link up`
+		};
+	}
+	return {
+		primary,
+		healthy: true,
+		why: "healthy"
+	};
+};
+const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/** This time is the longest that a group takes to converge before a run, or after it. */
+const CONVERGE_MS = 24e4;
+/**
+* This function opens one environment of a group cell:
+* - one new network, with each member, each Sentinel, the client and the observer on it.
+* - each member: the stock image, its server as the child of a shell, thus a fin can stop the
+*   server and the container stays up. The first member is the primary, and the others follow it.
+* - each Sentinel: the stock image, its Sentinel as the child of a shell, which monitors the first member.
+* - an injector sidecar for each part, in its network and process namespaces.
+* - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
+* - the observer of the Sentinels, in a namespace of its own.
+* - the prober, which watches the namespace of the client, and of each Sentinel for a calibration.
+*/
+const openGroupLab = async (group, options = {}) => {
+	const image = await harnessImage();
+	const node = await nodeClientImage();
+	const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();
+	const runtime = await getContainerRuntimeClient();
+	const opened = [];
+	let network;
+	const close = async () => {
+		for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
+		await network?.stop().catch(() => void 0);
+	};
+	const ipOf = async (container) => {
+		const ip = (await runtime.container.inspect(runtime.container.getById(container.getId()))).NetworkSettings.Networks[network.getName()]?.IPAddress;
+		if (ip === void 0 || ip === "") throw new Error(`${container.getName()} has no address on ${network.getName()}`);
+		return ip;
+	};
+	try {
+		network = await new Network().start();
+		const boxes = /* @__PURE__ */ new Map();
+		const injectors = /* @__PURE__ */ new Map();
+		const parts = {
+			members: {},
+			sentinels: {}
+		};
+		/** This function starts one part: the stock image with its configuration, its program as the child of a shell, and its injector. */
+		const start = async (name, program, file, content, ready) => {
+			const box = await new GenericContainer(group.image).withNetwork(network).withNetworkAliases(name).withCopyContentToContainer([{
+				content,
+				target: file
+			}]).withCommand([
+				"sh",
+				"-c",
+				`${program} ${file} & while :; do sleep 3600; done`
+			]).withWaitStrategy(Wait.forLogMessage(ready)).start();
+			opened.push(box);
+			boxes.set(name, box);
+			const injector = await withPid(new GenericContainer(image).withNetworkMode(`container:${box.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW"), `container:${box.getId()}`).start();
+			opened.push(injector);
+			injectors.set(name, injector);
+			return {
+				ip: await ipOf(box),
+				netns: await netnsOf(injector)
+			};
+		};
+		const [first, ...rest] = group.members;
+		if (first === void 0) throw new Error("a group has no member");
+		parts.members[first] = await start(first, group.programs.server, "/data/server.conf", memberConfig(group, null), group.ready.member);
+		const primaryIp = parts.members[first].ip;
+		for (const name of rest) parts.members[name] = await start(name, group.programs.server, "/data/server.conf", memberConfig(group, primaryIp), group.ready.member);
+		for (const name of group.sentinels) parts.sentinels[name] = await start(name, group.programs.sentinel, "/data/sentinel.conf", sentinelConfig(group, primaryIp), group.ready.sentinel);
+		const holder = await new GenericContainer(image).withNetwork(network).start();
+		opened.push(holder);
+		const admin = await new GenericContainer(image).withNetworkMode(`container:${holder.getId()}`).withPrivilegedMode().start();
+		opened.push(admin);
+		const clientIp = await ipOf(holder);
+		const sentinelAddresses = group.sentinels.map((name) => `${parts.sentinels[name].ip}:${group.sentinelPort}`);
+		const timeline = lines();
+		const observer = await new GenericContainer(node).withNetwork(network).withCommand([
+			"node",
+			"/opt/harness/node/observer.mjs",
+			sentinelAddresses.join(",")
+		]).withLogConsumer(timeline.consume).start();
+		opened.push(observer);
+		const observerIp = await ipOf(observer);
+		for (const address of sentinelAddresses) await new Promise((resolve, reject) => {
+			const started = Date.now();
+			const look = () => {
+				if (timeline.seen.some((line) => line.ev === "subscribed" && line["sentinel"] === address)) resolve();
+				else if (Date.now() - started > 3e4) reject(/* @__PURE__ */ new Error(`the observer never subscribed to ${address}`));
+				else setTimeout(look, 50);
+			};
+			look();
+		});
+		const clientNetns = await netnsOf(admin);
+		const watched = [clientNetns, ...options.watchSentinels === true ? group.sentinels.map((name) => parts.sentinels[name].netns) : []];
+		const records = lines();
+		const prober = await new GenericContainer(image).withPrivilegedMode().withNetworkMode("none").withCommand(["/opt/harness/prober", ...watched.map(String)]).withLogConsumer(records.consume).start();
+		opened.push(prober);
+		const begun = await records.until(["begin", "fail"], 3e4, "prober");
+		if (begun.ev === "fail") throw new Error(`the prober failed: ${String(begun["why"])}`);
+		const box = (name) => {
+			const found = boxes.get(name);
+			if (found === void 0) throw new Error(`the group has no part ${name}`);
+			return found;
+		};
+		const injectorOf = (name) => {
+			const found = injectors.get(name);
+			if (found === void 0) throw new Error(`the group has no part ${name}`);
+			return found;
+		};
+		const cli = (name, args) => {
+			const port = group.sentinels.includes(name) ? group.sentinelPort : group.port;
+			return output(box(name), [
+				group.programs.cli,
+				"-p",
+				String(port),
+				...args
+			]);
+		};
+		const stamped = async (where, argv) => {
+			const stamp = JSON.parse((await output(where, argv)).split("\n").at(-1));
+			return {
+				t0: stamp.t0,
+				t1: stamp.t1
+			};
+		};
+		const ips = Object.fromEntries(Object.entries(parts.members).map(([name, at]) => [name, at.ip]));
+		const view = async () => {
+			const sentinels = [];
+			for (const name of group.sentinels) {
+				const master = fieldsOf(await cli(name, [
+					"SENTINEL",
+					"MASTER",
+					group.name
+				]))[0] ?? {};
+				const replicas = fieldsOf(await cli(name, [
+					"SENTINEL",
+					"REPLICAS",
+					group.name
+				]));
+				sentinels.push({
+					master,
+					replicas
+				});
+			}
+			const members = {};
+			for (const name of group.members) members[name] = infoOf(await cli(name, ["INFO", "replication"]));
+			return viewOf(group, ips, sentinels, members);
+		};
+		const after = (at) => at === void 0 ? [] : ["--at", String(Math.round(at))];
+		const roles = (role, primary) => {
+			if (role === "primary") return primary;
+			if (role === "sentinel") return group.sentinels[0];
+			const replica = group.members.find((name) => name !== primary);
+			if (replica === void 0) throw new Error("the group has no replica");
+			return replica;
+		};
+		return {
+			group,
+			clientIp,
+			observerIp,
+			members: parts.members,
+			sentinels: parts.sentinels,
+			netns: {
+				client: clientNetns,
+				watched
+			},
+			kernel: records.seen,
+			timeline: timeline.seen,
+			async sysctl(values) {
+				const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
+				if (settings.length > 0) await output(admin, [
+					"sysctl",
+					"-q",
+					"-w",
+					...settings
+				]);
+				const read = {};
+				for (const key of Object.keys(values)) read[key] = await output(admin, [
+					"sysctl",
+					"-n",
+					key
+				]);
+				return read;
+			},
+			async converge(within = CONVERGE_MS) {
+				const started = Date.now();
+				let calm = 0;
+				let last = {
+					primary: null,
+					healthy: false,
+					why: "no view yet"
+				};
+				while (Date.now() - started < within) {
+					last = await view().catch((error) => ({
+						primary: null,
+						healthy: false,
+						why: error.message
+					}));
+					calm = last.healthy ? calm + 1 : 0;
+					if (calm >= 2 && last.primary !== null) return last.primary;
+					await sleep$1(1e3);
+				}
+				throw new Error(`the group did not converge in ${within} ms: ${last.why}`);
+			},
+			async flush(primary) {
+				await cli(primary, ["FLUSHALL"]);
+				const acked = Number(await cli(primary, [
+					"WAIT",
+					String(group.members.length - 1),
+					"5000"
+				]));
+				if (acked !== group.members.length - 1) throw new Error(`${acked} replicas acknowledged the flush`);
+			},
+			async reset() {
+				for (const name of group.sentinels) await cli(name, [
+					"SENTINEL",
+					"RESET",
+					group.name
+				]);
+			},
+			async holds(primary) {
+				return (await cli(primary, [
+					"--scan",
+					"--pattern",
+					"w:*"
+				])).split("\n").map((key) => Number(key.trim().slice(2))).filter((n) => Number.isInteger(n) && n > 0).sort((a, b) => a - b);
+			},
+			partOf: roles,
+			async client(args) {
+				const events = lines();
+				const container = await new GenericContainer(node).withNetworkMode(`container:${holder.getId()}`).withCommand([
+					"node",
+					"/opt/harness/node/client-sentinel.mjs",
+					...args
+				]).withLogConsumer(events.consume).start();
+				return {
+					events: events.seen,
+					until: (kinds, within) => events.until(kinds, within, "client"),
+					stop: () => container.stop({ timeout: 2 }).then(() => void 0).catch(() => void 0)
+				};
+			},
+			fault(kind, part, cuts, at) {
+				const peers = peersOf(parts, clientIp, part, cuts).join(",");
+				return stamped(injectorOf(part), [
+					"/opt/harness/inject",
+					kind,
+					peers,
+					...after(at)
+				]);
+			},
+			lift(kind, part, cuts, at) {
+				const peers = peersOf(parts, clientIp, part, cuts).join(",");
+				return stamped(injectorOf(part), [
+					"/opt/harness/inject",
+					kind === "rst" ? "clear-rst" : "clear",
+					peers,
+					...after(at)
+				]);
+			},
+			signal(part, signal, at) {
+				const program = group.sentinels.includes(part) ? group.programs.sentinel : group.programs.server;
+				return stamped(injectorOf(part), [
+					"/opt/harness/inject",
+					"signal",
+					program,
+					signal,
+					...after(at)
+				]);
+			},
+			async restart(member) {
+				await output(box(member), [
+					"sh",
+					"-c",
+					`${group.programs.server} /data/server.conf --daemonize yes`
+				]);
+				const started = Date.now();
+				while (await cli(member, ["PING"]).catch(() => "") !== "PONG") {
+					if (Date.now() - started > 3e4) throw new Error(`${member} did not start again`);
+					await sleep$1(200);
+				}
+			},
+			failover(coordinated) {
+				const sentinel = group.sentinels[0];
+				return stamped(injectorOf(sentinel), [
+					"/opt/harness/inject",
+					"resp",
+					`127.0.0.1:${group.sentinelPort}`,
+					"SENTINEL",
+					"FAILOVER",
+					group.name,
+					...coordinated ? ["COORDINATED"] : []
+				]);
+			},
+			async forget() {
+				await output(admin, [
+					"ip",
+					"tcp_metrics",
+					"flush",
+					"all"
+				]);
+			},
+			async settle() {
+				const from = records.seen.length;
+				const clock = await new Promise((resolve) => {
+					const started = Date.now();
+					const look = () => {
+						const line = records.seen.slice(from).find((record) => record.ev === "clock");
+						if (line !== void 0) resolve(line);
+						else if (Date.now() - started > 5e3) resolve(null);
+						else setTimeout(look, 50);
+					};
+					look();
+				});
+				return clock === null ? null : Number(clock["lost"] ?? 0);
+			},
+			async manifest(sysctls) {
+				const config = await output(admin, ["zcat", "/proc/config.gz"]).catch(() => "");
+				const inspect = async (container) => runtime.container.inspect(runtime.container.getById(container.getId()));
+				const memberSettings = async (name) => {
+					const read = {};
+					for (const key of Object.keys(group.memberSettings)) read[key] = (await cli(name, [
+						"CONFIG",
+						"GET",
+						key
+					])).split("\n")[1] ?? "";
+					return read;
+				};
+				const sentinelSettings = async (name) => {
+					const master = fieldsOf(await cli(name, [
+						"SENTINEL",
+						"MASTER",
+						group.name
+					]))[0] ?? {};
+					const read = { quorum: master["quorum"] ?? "" };
+					for (const key of Object.keys(group.sentinelSettings)) read[key] = master[key] ?? "";
+					return read;
+				};
+				const members = [];
+				for (const name of group.members) members.push({
+					name,
+					ip: parts.members[name].ip,
+					id: (await inspect(box(name))).Image,
+					settings: await memberSettings(name)
+				});
+				const sentinels = [];
+				for (const name of group.sentinels) sentinels.push({
+					name,
+					ip: parts.sentinels[name].ip,
+					id: (await inspect(box(name))).Image,
+					settings: await sentinelSettings(name)
+				});
+				return {
+					kernel: await output(holder, ["uname", "-r"]),
+					architecture: await output(holder, ["uname", "-m"]),
+					hz: /^CONFIG_HZ=(\d+)$/m.exec(config)?.[1] ?? null,
+					boot_id: await output(holder, ["cat", "/proc/sys/kernel/random/boot_id"]),
+					docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
+					target: {
+						image: group.image,
+						id: members[0].id,
+						settings: members[0].settings,
+						sysctls: {}
+					},
+					harness: {
+						image,
+						id: (await inspect(holder)).Image,
+						packages: (await output(holder, [
+							"apk",
+							"info",
+							"-v"
+						])).split("\n").sort()
+					},
+					client: {
+						kind: "ioredis",
+						image: node
+					},
+					netns: {
+						client: clientNetns,
+						target: parts.members[first].netns
+					},
+					sysctls,
+					group: {
+						members,
+						sentinels,
+						observer: { image: node }
+					}
+				};
+			},
+			close
+		};
+	} catch (error) {
+		await close();
+		throw error;
+	}
+};
+/** This count is the number of accepted writes that a run waits for before its fault: the stream is steady then. */
+const STEADY = 20;
+/** This time is how long a run continues after its outcome, so that the record shows what came next. */
+const TAIL_MS = 2e3;
+/** This function gives the address of a member, with the port of the group. */
+const addressOf = (lab, member) => `${lab.members[member].ip}:${lab.group.port}`;
+/** This function gives the latest time that a group cell waits for its outcome, in ms from the fault. */
+const deadlineOf = (cell) => {
+	const { prediction } = cell;
+	return (prediction.detects === true ? valueIn(prediction.high, "s") : prediction.detects === false && prediction.recovers !== void 0 ? valueIn(prediction.recovers.high, "s") : valueIn(prediction.window, "s")) * 1e3 + 1e4;
+};
+/** This function gives the arguments of the client of a group cell. */
+const clientArgs = (cell, lab) => {
+	return [
+		"--sentinels",
+		cell.group.sentinels.map((name) => `${lab.sentinels[name].ip}:${cell.group.sentinelPort}`).join(","),
+		"--name",
+		cell.group.name,
+		"--interval-ms",
+		String(cell.probe.interval === void 0 ? 50 : valueIn(cell.probe.interval, "ms")),
+		"--max-s",
+		String(Math.ceil(deadlineOf(cell) / 1e3 + 120)),
+		...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
+	];
+};
+/** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
+const waitFor = (client, match, within) => new Promise((resolve) => {
+	const started = Date.now();
+	const look = () => {
+		const found = client.events.find(match);
+		if (found !== void 0 || Date.now() - started > within) resolve(found);
+		else setTimeout(look, 20);
+	};
+	look();
+});
+/** This function gives the latest subscription of the observer to each Sentinel, in some lines of the observer. */
+const subscriptionsOf = (timeline) => {
+	const latest = /* @__PURE__ */ new Map();
+	for (const line of timeline) if (line.ev === "subscribed") latest.set(String(line.sentinel), line);
+	return [...latest.values()];
+};
+/** This function gives the count of lost records in the latest clock line of the prober. */
+const droppedSoFar$1 = (records) => {
+	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
+	return 0;
+};
+/**
+* This function does one run of a group cell:
+* - The group converges, and its keys go. The client connects through the Sentinels, and its
+*   stream of writes gets steady.
+* - The fault occurs on its part, as the primary of this run makes the roles. A fault with a
+*   hold lifts at its instant in the VM. A stall lifts at its length after the first write into it.
+* - The run waits for its outcome: the first write that a new primary accepts, or the first one
+*   after the lift on the same primary. Then it continues for a moment.
+* - Each fault goes, the client stops, and the group converges again. The keys that it then holds
+*   tell which writes it kept.
+*/
+const groupOnce = async (lab, cell, environment, index) => {
+	const group = cell.group;
+	const primary = await lab.converge();
+	await lab.flush(primary);
+	await lab.forget();
+	const from = lab.kernel.length;
+	const timelineFrom = lab.timeline.length;
+	const lostBefore = droppedSoFar$1(lab.kernel);
+	const client = await lab.client(clientArgs(cell, lab));
+	const lands = cell.prediction.group?.lands ?? "new";
+	const place = cell.faultAt ?? { role: "primary" };
+	let fault = null;
+	let heal = null;
+	let at = null;
+	const undo = [];
+	try {
+		await client.until(["ready"], 6e4);
+		if (await waitFor(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY, 3e4) === void 0) throw new Error(`the client did not write ${STEADY} times in 30 s`);
+		let healing = null;
+		switch (cell.fault) {
+			case "blackhole":
+			case "partition":
+			case "rst": {
+				const kind = cell.fault === "rst" ? "rst" : "blackhole";
+				const cuts = place.cuts ?? (cell.fault === "partition" ? "group" : "all");
+				const part = lab.partOf(place.role, primary);
+				at = part;
+				fault = await lab.fault(kind, part, cuts);
+				undo.push(() => lab.lift(kind, part, cuts));
+				if (cell.stall !== void 0) {
+					const t1 = fault.t1;
+					const sent = await waitFor(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
+					if (sent !== void 0) healing = lab.lift(kind, part, cuts, sent.t + valueIn(cell.stall, "s") * 1e9);
+				} else if (cell.hold !== void 0) healing = lab.lift(kind, part, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
+				break;
+			}
+			case "freeze-process": {
+				const part = lab.partOf(place.role, primary);
+				at = part;
+				fault = await lab.signal(part, "STOP");
+				undo.push(() => lab.signal(part, "CONT"));
+				if (cell.hold !== void 0) healing = lab.signal(part, "CONT", fault.t0 + valueIn(cell.hold, "s") * 1e9);
+				break;
+			}
+			case "fin": {
+				const part = lab.partOf(place.role, primary);
+				at = part;
+				fault = await lab.signal(part, "KILL");
+				if (group.members.includes(part)) undo.push(() => lab.restart(part));
+				break;
+			}
+			case "planned-failover":
+				fault = await lab.failover(cell.coordinated === true);
+				break;
+			default: throw new Error(`a group cell has no fault ${cell.fault}`);
+		}
+		const healed = healing === null ? Promise.resolve(null) : healing.then((stamp) => {
+			heal = stamp;
+			undo.length = 0;
+			return stamp;
+		});
+		healed.catch(() => void 0);
+		const t0 = fault.t0;
+		const old = addressOf(lab, primary);
+		const deadline = deadlineOf(cell);
+		if ((lands === "new" ? await waitFor(client, (event) => event.ev === "ack" && event.t > t0 && typeof event["at"] === "string" && event["at"] !== old, deadline) : await (async () => {
+			await healed;
+			const after = heal?.t0 ?? t0;
+			return waitFor(client, (event) => event.ev === "ack" && event.t > after, deadline);
+		})()) !== void 0) await sleep$1(TAIL_MS);
+		await healed;
+	} finally {
+		for (const step of undo.reverse()) await step().catch(() => void 0);
+		await client.stop();
+	}
+	const settled = await lab.settle();
+	const lost = settled === null ? null : settled - lostBefore;
+	const after = await lab.converge().catch(() => null);
+	const holds = after === null ? null : await lab.holds(after);
+	if (lab.timeline.slice(timelineFrom).some((line) => line.ev === "sentinel" && String(line.event).startsWith("-failover-abort"))) await lab.reset();
+	return {
+		environment,
+		index,
+		events: [...client.events],
+		fault,
+		...heal === null ? {} : { heal },
+		kernel: lab.kernel.slice(from),
+		netns: lab.netns.client,
+		targetNetns: lab.members[primary].netns,
+		lost,
+		group: {
+			members: lab.members,
+			sentinels: lab.sentinels,
+			primary,
+			at,
+			timeline: [...subscriptionsOf(lab.timeline.slice(0, timelineFrom)), ...lab.timeline.slice(timelineFrom)],
+			after,
+			holds
+		}
+	};
+};
+//#endregion
 //#region src/run.ts
 /**
 * This function measures and judges the raw records of a cell, with the code of the harness at
@@ -1679,9 +2725,9 @@ const once = async (lab, cell, environment, index, dry = false) => {
 			if (replyInFlight) {
 				await go();
 				await sleep(200);
-			}
+			} else await sleep(250);
 			const faultAt = cell.faultAt ?? (server ? "client" : "target");
-			const towardServerComesIn = faultAt === "target" || typeof faultAt === "object" && faultAt.toward === "client";
+			const towardServerComesIn = faultAt === "target" || isHopPlace(faultAt) && faultAt.toward === "client";
 			const direction = cell.drops === void 0 ? void 0 : cell.drops === "to-server" === towardServerComesIn ? "in" : "out";
 			const options = {
 				...direction === void 0 ? {} : { direction },
@@ -1808,6 +2854,7 @@ const line = (cell, run, read) => `${cell.name}: env ${run.environment} ${run.wa
 * of a cell whose fault ends the environment is a dry run, without the fault.
 */
 const inEnvironment = async (cell, environment, count, say) => {
+	if (cell.group !== void 0) return inGroupEnvironment(cell, environment, count, say);
 	const lab = await openLab(cell.target, cell.client ?? cell.probe.driver?.name ?? "tcp", cell.topology);
 	try {
 		const settings = await lab.configure(cell.targetSettings ?? {});
@@ -1816,7 +2863,7 @@ const inEnvironment = async (cell, environment, count, say) => {
 		if (cell.fault === "mtu-blackhole" || cell.fault === "netem") {
 			await lab.offload(false, "target");
 			await lab.offload(false, "client");
-			if (typeof cell.faultAt === "object") await lab.offload(false, cell.faultAt);
+			if (isHopPlace(cell.faultAt)) await lab.offload(false, cell.faultAt);
 		}
 		const latency = netemSpec(cell, void 0);
 		if (latency !== null) await lab.netem(latency, "target");
@@ -1843,6 +2890,35 @@ const inEnvironment = async (cell, environment, count, say) => {
 	}
 };
 /**
+* This function does the part of the runs of a group cell in one environment. The warm-up is a
+* whole run with its fault, because no fault of a group ends its environment. A server that a
+* fin stopped starts again, and the group converges before each run.
+*/
+const inGroupEnvironment = async (cell, environment, count, say) => {
+	const lab = await openGroupLab(cell.group, { watchSentinels: cell.calibrates === "sentinels" });
+	try {
+		const manifest = await lab.manifest(await lab.sysctl(cell.sysctls));
+		const warmup = {
+			...await groupOnce(lab, cell, environment, -1),
+			warmup: true
+		};
+		say(line(cell, warmup, measure(cell, warmup)));
+		const runs = [];
+		for (let index = 0; index < count; index += 1) {
+			const run = await groupOnce(lab, cell, environment, index);
+			runs.push(run);
+			say(line(cell, run, measure(cell, run)));
+		}
+		return {
+			manifest,
+			runs,
+			warmup
+		};
+	} finally {
+		await lab.close();
+	}
+};
+/**
 * This function does the runs of a cell. They go across new environments, and each environment records
 * its manifest. Then the verdict judges the runs. A median across the edge of the band extends
 * the cell from 11 runs to 22, and then to 44, each extension in new environments. With a
@@ -1891,6 +2967,6 @@ const runCell = async (cell, say = () => {}) => {
 	}
 };
 //#endregion
-export { LISTING, NOISE, PUMP, RELATIVE, STEPS, crossingsOf, deviceOf, endpointAfter, harnessImage, judge, measure, medianInterval, nodeClientImage, openLab, quantileOf, reorderingOf, routerImage, routesOf, runCell, socketOf, spread, spreadOf, templated, toxicBody, verdictOf };
+export { CONVERGE_MS, LISTING, NOISE, PUMP, RELATIVE, STAGES, STEPS, crossingsOf, deadlineOf, deviceOf, endpointAfter, errorsOf, fieldsOf, groupOnce, groupVerdictOf, harnessImage, infoOf, isHopPlace, judge, lagsOf, measure, measureGroup, medianInterval, memberConfig, nodeClientImage, openGroupLab, openLab, peersOf, quantileOf, reorderingOf, routerImage, routesOf, runCell, sentinelConfig, socketOf, spread, spreadOf, stagesOf, subscriptionsOf, templated, toxicBody, verdictOf, viewOf, writesOf };
 
 //# sourceMappingURL=index.js.map