@mark1russell7/harness 0.5.0 → 0.7.0

package/build/index.js

@@ -10,6 +10,8 @@ import { fileURLToPath } from "node:url";
 const isHopPlace = (place) => typeof place === "object" && "hop" in place;
 //#endregion
 //#region src/verdict.ts
+/** This function gives the words of the outcome of a second request: its connection, a retry, and its end. */
+const requestWords = (outcome) => `${outcome.connection}${outcome.retried ? ", a retry" : ""}, ${outcome.pending === true ? "pending" : outcome.error ?? "a reply"}`;
 /**
 * This amount is the noise of the pipeline itself, δ_abs, as the calibration measured it. It is
 * the 99.9th percentile of the whole latency of a reset, at the concurrency cap or less. Its
@@ -115,14 +117,32 @@ const middle = (sorted) => {
 * which does not exist at this time.
 */
 const verdictOf = (prediction, runs) => {
-	const verdict = verdictOfBand(prediction, runs);
+	const band = verdictOfBand(prediction, runs);
 	const counted = runs.filter((run) => run.void === null);
+	const verdict = counted.some((run) => run.request !== void 0) ? withRequests(band, counted) : band;
 	if (prediction.group === void 0 && !counted.some((run) => run.group !== void 0)) return verdict;
 	return {
 		...verdict,
 		group: groupVerdictOf(prediction, counted)
 	};
 };
+/**
+* This function adds the outcomes of the second request of an HTTP client to a verdict. A cell
+* whose outcome is a distribution gets them in its reason too.
+*/
+const withRequests = (verdict, counted) => {
+	const requests = {};
+	for (const run of counted) {
+		const words = run.request === null || run.request === void 0 ? "no second request" : requestWords(run.request);
+		requests[words] = (requests[words] ?? 0) + 1;
+	}
+	const listed = Object.entries(requests).map(([words, n]) => `${n} ${words}`).join("; ");
+	return {
+		...verdict,
+		requests,
+		why: verdict.decision === "measured" ? `${verdict.why} Second requests: ${listed}.` : verdict.why
+	};
+};
 /** This function gives the median of some counts, or null for none. */
 const medianCount = (values) => values.length === 0 ? null : middle([...values].sort((a, b) => a - b));
 /** This function sums what the group of a group cell did in its counted runs, and tests each stage that has a band. */
@@ -139,9 +159,10 @@ const groupVerdictOf = (prediction, counted) => {
 		for (const run of counted) {
 			const at = run.group?.stages[band.stage];
 			if (at === void 0) {
+				const who = band.stage.startsWith("+") ? "no Sentinel announced" : "no member or client marked";
 				outside.push({
 					run: name(run),
-					why: `no Sentinel announced ${band.stage}`
+					why: `${who} ${band.stage}`
 				});
 				continue;
 			}
@@ -362,13 +383,367 @@ const verdictOfBand = (prediction, runs) => {
 	};
 };
 //#endregion
+//#region src/driver.ts
+/** These are the errno numbers that the kernel reports to a socket, by the names that predictions use. */
+const ERRNO$1 = {
+	32: "EPIPE",
+	101: "ENETUNREACH",
+	103: "ECONNABORTED",
+	104: "ECONNRESET",
+	110: "ETIMEDOUT",
+	111: "ECONNREFUSED",
+	113: "EHOSTUNREACH"
+};
+const between$2 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
+const round = (seconds) => Math.round(seconds * 1e6) / 1e6;
+/** This function gives the value of a field of a line by its name. A dotted name is a field in a field. */
+const fieldOf = (line, path) => path.split(".").reduce((value, key) => value !== null && typeof value === "object" ? value[key] : void 0, line);
+/** This function tells if a line of the client matches a line match: its `ev`, and each field that the match names. */
+const matches = (line, match) => line.ev === match.ev && Object.entries(match.where ?? {}).every(([path, value]) => fieldOf(line, path) === value);
+/** This function says a line match in words, for a finding. */
+const wordsOf = (match) => {
+	const where = Object.entries(match.where ?? {}).map(([path, value]) => `${path} ${String(value)}`);
+	return where.length === 0 ? `a ${match.ev} line` : `a ${match.ev} line with ${where.join(", ")}`;
+};
+/** This function gives the sockets of the driver in a run, from the `socket` and `socket-end` lines of the client. */
+const socketsOf = (events) => events.filter((event) => event.ev === "socket").map((event) => {
+	const local = String(event["local"]);
+	const end = events.find((each) => each.ev === "socket-end" && each["local"] === local && each.t >= event.t);
+	return {
+		role: String(event["role"]),
+		local,
+		server: String(event["server"]),
+		opened: event.t,
+		ended: end?.t ?? null
+	};
+});
+/**
+* This function finds the socket of a role at an instant. The pool's socket is the first one, which
+* the client names in its `connected` line. Each other role has the latest socket that opened at
+* the instant or before it.
+*/
+const socketAt = (events, role, at) => {
+	const sockets = socketsOf(events);
+	if (role === "pool") {
+		const local = events.find((event) => event.ev === "connected")?.["local"];
+		return sockets.find((socket) => socket.local === local);
+	}
+	return sockets.filter((socket) => socket.role === role && socket.opened <= at).at(-1);
+};
+const placeOf = (run, netns, find) => {
+	const cookie = run.kernel.find((record) => record.netns === netns && find(record))?.cookie;
+	if (cookie === void 0) return void 0;
+	return {
+		netns,
+		cookie,
+		records: run.kernel.filter((record) => record.netns === netns && record.cookie === cookie)
+	};
+};
+const portOf = (address) => address.split(":").at(-1) ?? "";
+/**
+* This function gives the socket of the target for a socket of the client. In a direct pair, its
+* peer is the address of the client socket. Behind a router, its peer is the address of the router
+* with the port of the client socket. The router keeps the port when it is free.
+*/
+const peerPlace = (run, local) => {
+	const peer = run.topology === void 0 ? local : `${run.topology.targetPeer}:${portOf(local)}`;
+	return placeOf(run, run.targetNetns, (record) => record.dst === peer);
+};
+/** This function gives the kernel place of a socket of a role at an instant. The role `server` is a socket of the target. */
+const kernelOf = (run, role, at) => {
+	if (role === "server") {
+		const pool = socketAt(run.events, "pool", at);
+		return {
+			socket: pool,
+			place: pool === void 0 ? void 0 : peerPlace(run, pool.local)
+		};
+	}
+	const socket = socketAt(run.events, role, at);
+	return {
+		socket,
+		place: socket === void 0 ? void 0 : placeOf(run, run.netns, (record) => record.src === socket.local)
+	};
+};
+const newState$1 = (record) => record.new;
+const oldState$1 = (record) => record.old;
+/** This function gives the instant that a run starts to look for its outcome. It is the fault, the reply before it, or the start of the client. */
+const outsetOf = (run) => {
+	const ready = run.events.find((event) => event.ev === "ready");
+	return run.fault?.t0 ?? ready?.t ?? run.events[0]?.t ?? 0;
+};
+/**
+* This function gives the instant of a reference in a run, on a socket of the driver:
+* - `last-received`: the last segment that the socket received before the fault held. A band of
+*   the kernel counts from the last segment of any kind. A band that a line ends is a timer of
+*   the driver. It counts from the last segment with data that the socket received or sent.
+* - `request-sent`: the send of the operation into the fault, the second `sent` line.
+* - `first-retransmission`: the first retransmission of the RTO timer on the socket.
+* - `peer-closed`: the change of state of the socket of the target, when its process dies.
+* - `fault`: the start of the commit of the fault. `client-start`: the `connecting` line.
+*/
+const referenceOf = (run, from, role, data) => {
+	const ready = run.events.find((event) => event.ev === "ready");
+	const before = run.fault?.t1 ?? ready?.t ?? Infinity;
+	if (from === "fault") return run.fault?.t0;
+	if (from === "client-start") return (run.events.find((event) => event.ev === "connecting") ?? run.events[0])?.t;
+	if (from === "request-sent") return run.events.filter((event) => event.ev === "sent")[1]?.t;
+	const { socket, place } = kernelOf(run, role, before);
+	if (place === void 0) return void 0;
+	if (from === "last-received") {
+		const counts = (record) => data ? (record.ev === "rx" || record.ev === "tx") && (record.seg?.len.value ?? 0) > 0 : record.ev === "rx";
+		return place.records.filter((record) => record.mono <= before && counts(record)).at(-1)?.mono;
+	}
+	if (from === "first-retransmission") {
+		const request = run.events.filter((event) => event.ev === "sent")[1]?.t ?? before;
+		return place.records.find((record) => record.ev === "retrans" && record.why === "rto" && record.mono > request)?.mono;
+	}
+	return (socket === void 0 ? void 0 : peerPlace(run, socket.local))?.records.find((record) => record.ev === "state" && oldState$1(record) === 1)?.mono;
+};
+/** This function gives the first line of the client that matches, after an instant. */
+const lineAfter = (events, match, after) => events.find((event) => event.t > after && matches(event, match));
+/** This function gives the name of the error of a line: the field that the match names, or `name`. */
+const nameOf = (line, match) => String(fieldOf(line, match.name ?? "name") ?? line.ev);
+/** This function gives the report of the kernel on a socket after an instant: an error, or the end of the file (CLOSE_WAIT). */
+const reportOf = (place, after) => {
+	const report = place.records.find((record) => record.mono > after && (record.ev === "error" && (record.err ?? 0) !== 0 || record.ev === "state" && newState$1(record) === 8));
+	if (report === void 0) return void 0;
+	return report.ev === "error" ? {
+		at: report.mono,
+		outcome: {
+			kind: "error",
+			errno: ERRNO$1[report.err] ?? `E${report.err}`
+		}
+	} : {
+		at: report.mono,
+		outcome: { kind: "eof" }
+	};
+};
+/** This function gives the latest instant of the bands of a driver prediction, in seconds from their references. */
+const longestOf = (cell) => {
+	const { prediction } = cell;
+	if (prediction.detects === null) return valueIn(prediction.window, "s");
+	const steps = (prediction.driver?.then ?? []).map((step) => valueIn(step.high, "s"));
+	const own = prediction.detects ? valueIn(prediction.high, "s") : valueIn(prediction.window, "s");
+	return Math.max(own, ...steps);
+};
+/**
+* This function tells if a run of a driver cell has its outcome, so that its wait can stop. For a
+* detection, the outcome is the end of its band and of each later band. A silent prediction waits
+* for its window.
+*/
+const settled = (cell, run) => {
+	const { prediction } = cell;
+	if (prediction.detects !== true || prediction.driver === void 0) return false;
+	const driver = prediction.driver;
+	const after = outsetOf(run);
+	return (driver.ends === void 0 ? (() => {
+		const { place } = kernelOf(run, driver.socket, run.fault?.t1 ?? after);
+		return place !== void 0 && reportOf(place, after) !== void 0;
+	})() : lineAfter(run.events, driver.ends, after) !== void 0) && (driver.then ?? []).every((step) => lineAfter(run.events, step.ends, after) !== void 0);
+};
+/**
+* This function examines the keepalive timer of a socket between two instants. A firing two
+* jiffies or more behind its wheel slot was delayed by the host, which stopped a vCPU. A firing
+* that found the socket held by its owner put itself off by HZ/20. Either way, the run measured
+* something other than the timer.
+*/
+const delayed = (place, from, to) => {
+	const firings = place.records.filter((record) => record.ev === "timer" && record.what === "keepalive" && record.mono >= from && record.mono <= to);
+	const late = firings.find((record) => (record.behind?.value ?? 0) >= 2);
+	if (late !== void 0) return `its keepalive timer ran ${late.behind.value} jiffies behind its wheel slot: the VM delayed it, not the kernel`;
+	return firings.find((record) => record.owned === true) === void 0 ? null : "its keepalive timer found the socket held by its owner and put itself off by HZ/20: the run measured the owner, not the timer";
+};
+/**
+* This function reads one run of a driver cell.
+* - The band of the cell counts from its reference on its socket. It ends at its line, or at the
+*   report of the kernel on that socket.
+* - Each later band (`then`) gets the same test. A later band outside its band, or absent, is a finding.
+* - A line that the prediction names as quiet, after the fault and in its window, is a finding.
+*   So is a flow of an open socket of the driver that a router forgot, for a silent prediction.
+* - A silent prediction can time a reply (`reply`), and the verdict tests it against `replies`.
+*
+* A run is void when the prober lost records, or when the client did not get its first reply. It
+* is also void when the operation went out before the fault held. A late lift of a hold and a
+* keepalive probe before the fault also make it void. So do a delayed timer and a round trip
+* above the assumption.
+*/
+const measureDriver = (cell, run) => {
+	const { prediction } = cell;
+	if (prediction.detects === null || prediction.driver === void 0) throw new Error(`${cell.name} has no driver band`);
+	const driver = prediction.driver;
+	const { events } = run;
+	const ready = events.find((event) => event.ev === "ready");
+	const request = events.filter((event) => event.ev === "sent")[1];
+	const after = outsetOf(run);
+	const before = run.fault?.t1 ?? ready?.t ?? after;
+	const { place } = kernelOf(run, driver.socket, before);
+	const records = place?.records ?? [];
+	const srtt = kernelOf(run, "pool", before).place?.records.find((record) => record.tcp?.srtt !== void 0)?.tcp?.srtt;
+	const rtt = srtt === void 0 ? null : quantity(valueIn(srtt, "us") / 1e3, "ms");
+	const base = {
+		environment: run.environment,
+		index: run.index,
+		probes: records.filter((record) => record.ev === "probe" && record.what === "keepalive").length,
+		retransmissions: records.filter((record) => record.ev === "retrans").length,
+		acks: request === void 0 ? 0 : records.filter((record) => record.ev === "rx" && record.mono > request.t).length,
+		rtt
+	};
+	const nothing = {
+		outcome: { kind: "silent" },
+		elapsed: null,
+		socket: null,
+		noticed: null,
+		steps: [],
+		finding: null
+	};
+	if (cell.before !== true && ready === void 0) return {
+		...base,
+		...nothing,
+		void: "the client never had its reply"
+	};
+	if (run.lost === null) return {
+		...base,
+		...nothing,
+		void: "the prober wrote no clock line after the run, so nothing says what it dropped"
+	};
+	if (run.lost > 0) return {
+		...base,
+		...nothing,
+		void: `the prober dropped ${run.lost} records`
+	};
+	let voided = null;
+	if (cell.probe.mode !== "idle" && cell.before !== true) {
+		if (request === void 0) voided = "the operation was never sent into the fault";
+		else if (run.fault !== null && run.fault.t1 > request.t) voided = "the operation went out before the fault held";
+	}
+	const keepalive = records.find((record) => record.ev === "probe" && record.what === "keepalive");
+	if (voided === null && run.fault !== null && keepalive !== void 0 && keepalive.mono < run.fault.t1) voided = "a keepalive probe went out before the fault held";
+	if (voided === null && cell.hold !== void 0 && run.fault !== null) {
+		const heal = run.heal ?? null;
+		if (heal === null) voided = "the fault never lifted";
+		else {
+			const late = (heal.t0 - run.fault.t0) / 1e9 - valueIn(cell.hold, "s");
+			if (late > valueIn(NOISE, "s")) voided = `the lift came ${Math.round(late * 1e4) / 10} ms after its hold: the injector was late, not the client`;
+		}
+	}
+	const assumed = prediction.rtt;
+	if (voided === null && assumed !== void 0 && rtt !== null && valueIn(rtt, "ms") > valueIn(assumed.high, "ms")) voided = `rtt-over-assumption: its first round trip was ${rtt.value} ms, above the ${valueIn(assumed.high, "ms")} ms that the prediction assumes`;
+	const findings = [];
+	const noise = valueIn(NOISE, "s");
+	const until = prediction.detects ? Infinity : after + valueIn(prediction.window, "s") * 1e9;
+	for (const match of driver.quiet ?? []) {
+		const line = events.find((event) => event.t > after && event.t <= until && matches(event, match));
+		if (line !== void 0) findings.push(`${wordsOf(match)} came ${between$2(after, line.t).value} s after the fault, where none should`);
+	}
+	if (!prediction.detects && run.middlebox !== void 0) for (const socket of socketsOf(events)) {
+		const forgot = run.middlebox.find((line) => line.ev === "ct" && line["type"] === "DESTROY" && String(line["orig"] ?? "").startsWith(`${socket.local}>`) && line.t > socket.opened && (socket.ended === null || line.t < socket.ended) && line.t <= until);
+		if (forgot !== void 0) findings.push(`a router forgot the flow of the ${socket.role} socket ${between$2(after, forgot.t).value} s after the fault, where it should keep it`);
+	}
+	const stepTimes = (driver.then ?? []).map((step) => {
+		const line = lineAfter(events, step.ends, after);
+		const from = referenceOf(run, step.from, step.socket ?? driver.socket, true);
+		if (line === void 0 || from === void 0) {
+			findings.push(line === void 0 ? `${step.what}: ${wordsOf(step.ends)} never came` : `${step.what}: its reference (${step.from}) never came`);
+			return {
+				what: step.what,
+				name: line === void 0 ? null : nameOf(line, step.ends),
+				elapsed: null
+			};
+		}
+		const elapsed = between$2(from, line.t);
+		const name = nameOf(line, step.ends);
+		if (step.name !== "OK" && name !== step.name) findings.push(`${step.what}: it ended with ${name}, not ${step.name}`);
+		const [low, high] = [round(valueIn(step.low, "s") - noise), round(valueIn(step.high, "s") + noise)];
+		if (elapsed.value < low || elapsed.value > high) findings.push(`${step.what}: ${elapsed.value} s is outside [${low}, ${high}] s`);
+		return {
+			what: step.what,
+			name,
+			elapsed
+		};
+	});
+	const stepped = stepTimes.length === 0 ? {} : { driver: { steps: stepTimes } };
+	if (!prediction.detects) {
+		if (driver.reply === void 0 || prediction.replies === void 0) return {
+			...base,
+			...nothing,
+			...stepped,
+			void: voided,
+			finding: findings[0] ?? null
+		};
+		const line = lineAfter(events, driver.reply.ends, after);
+		const from = referenceOf(run, driver.reply.from, driver.socket, true);
+		if (line === void 0 || from === void 0) return {
+			...base,
+			...nothing,
+			...stepped,
+			void: voided,
+			finding: findings[0] ?? `the reply never came: ${wordsOf(driver.reply.ends)}`
+		};
+		return {
+			...base,
+			...nothing,
+			...stepped,
+			outcome: { kind: "answered" },
+			elapsed: between$2(from, line.t),
+			void: voided,
+			finding: findings[0] ?? null
+		};
+	}
+	const line = driver.ends === void 0 ? void 0 : lineAfter(events, driver.ends, after);
+	const report = driver.ends === void 0 && place !== void 0 ? reportOf(place, after) : void 0;
+	if (driver.ends === void 0 && place === void 0) return {
+		...base,
+		...nothing,
+		...stepped,
+		void: voided ?? `the prober saw nothing of the ${driver.socket} socket`,
+		finding: findings[0] ?? null
+	};
+	const ended = line?.t ?? report?.at;
+	const outcome = line !== void 0 && driver.ends !== void 0 ? {
+		kind: "error",
+		errno: nameOf(line, driver.ends)
+	} : report?.outcome ?? { kind: "silent" };
+	if (ended === void 0) return {
+		...base,
+		...nothing,
+		...stepped,
+		void: voided,
+		finding: findings[0] ?? `nothing detected it: ${prediction.timer} never fired`
+	};
+	const reference = referenceOf(run, prediction.from, driver.socket, driver.ends !== void 0);
+	if (reference === void 0) return {
+		...base,
+		...nothing,
+		...stepped,
+		outcome,
+		void: voided,
+		finding: `it ended without its reference event (${prediction.from} on the ${driver.socket} socket)`
+	};
+	if (driver.ends === void 0 && prediction.from === "last-received" && place !== void 0) voided ??= delayed(place, reference, ended);
+	const name = outcome.kind === "error" ? outcome.errno : outcome.kind === "eof" ? "EOF" : "silent";
+	if (prediction.errno !== "OK" && name !== prediction.errno) findings.unshift(`it ended with ${name}, not ${prediction.errno}`);
+	const seen = report === void 0 ? void 0 : events.find((event) => event.t >= report.at && (event.ev === "error" || event.ev === "socket-error"));
+	return {
+		...base,
+		...stepped,
+		outcome,
+		elapsed: between$2(reference, ended),
+		socket: seen === void 0 ? null : between$2(reference, seen.t),
+		noticed: report === void 0 || seen === void 0 ? null : between$2(report.at, seen.t),
+		steps: (prediction.from === "last-received" ? records.filter((record) => record.ev === "probe" && record.what === "keepalive" && record.mono > reference) : []).map((record) => between$2(reference, record.mono)),
+		void: voided,
+		finding: findings[0] ?? null
+	};
+};
+//#endregion
 //#region src/timeline.ts
 /**
 * This module measures a run of a group cell. It gives the first write that a primary accepted
-* after the fault, and the timeline of the group as its Sentinels announced it. It also gives what
+* after the fault, and the timeline of the group. The Sentinels of a group announce its timeline.
+* The logs of the members of a replica set and its client tell it. The module also gives what
 * became of each write.
 */
-/** These are the stages of a timeline, in their sequence after a fault. */
+/** These are the stages of a timeline under Sentinels, in their sequence after a fault. */
 const STAGES = [
 	"+sdown",
 	"+odown",
@@ -398,6 +773,105 @@ const stagesOf = (timeline, from) => {
 	}
 	return out;
 };
+/** These are the stages of the timeline of a replica set, in their usual sequence after a fault. */
+const REPLICA_SET_STAGES = [
+	"member-down",
+	"election-start",
+	"elected",
+	"caught-up",
+	"writable",
+	"old-primary-steps-down",
+	"discovered",
+	"written-off",
+	"first-write"
+];
+/** These are the log lines of mongod that mark each stage of a member. */
+const MEMBER_MARKS = {
+	"member-down": {
+		ids: [21216],
+		message: /now in state DOWN/,
+		at: "other"
+	},
+	"election-start": {
+		ids: [4615652, 21437],
+		message: /^Starting an election|Skipping dry run|Conducting a dry run/,
+		at: "other"
+	},
+	elected: {
+		ids: [21450],
+		message: /^Election succeeded/,
+		at: "other"
+	},
+	"caught-up": {
+		ids: [21363],
+		message: /^Exited primary catch-up mode/,
+		at: "other"
+	},
+	writable: {
+		ids: [21331],
+		message: /^Transition to primary complete/,
+		at: "other"
+	},
+	"old-primary-steps-down": {
+		ids: [
+			21809,
+			21475,
+			962660
+		],
+		message: /relinquishing primary|Stepping down from primary/,
+		at: "old"
+	}
+};
+/** This function tells if a log line of a member marks a stage, with the old primary of the run. */
+const marks = (line, stage, old, oldAddress) => {
+	const mark = MEMBER_MARKS[stage];
+	const member = String(line["member"] ?? "");
+	if (mark.at === "old" !== (member === old)) return false;
+	const attr = line["attr"] ?? {};
+	if (stage === "member-down" && !JSON.stringify(attr).includes(oldAddress)) return false;
+	if (stage === "old-primary-steps-down" && Number(line["id"]) === 21358 && attr["oldState"] === "PRIMARY") return true;
+	return mark.ids.includes(Number(line["id"])) || mark.message.test(String(line["msg"] ?? ""));
+};
+/**
+* This function gives the time of each stage of a replica set from an instant, in a run. The stages
+* of the members come from their log lines (`ev` "mongod"), whose times the lab puts on the
+* monotonic clock. The stages of the client come from its lines:
+* - `discovered`: a member other than the old primary becomes RSPrimary in the view of the client.
+* - `written-off`: the first heartbeat of the client to the old primary that fails.
+* - `first-write`: the first write that a primary accepts, of a write that the client sent after
+*   the commit of the fault (`sentAfter`). It is on a new primary when `lands` is `new`.
+*/
+const replicaSetStagesOf = (run, from, sentAfter, lands) => {
+	const out = {};
+	const old = run.group?.primary ?? "";
+	const oldAddress = `${run.group?.members[old]?.ip ?? ""}:`;
+	for (const line of run.group?.timeline ?? []) {
+		if (line.ev !== "mongod" || line.t < from) continue;
+		for (const stage of Object.keys(MEMBER_MARKS)) if (out[stage] === void 0 && marks(line, stage, old, oldAddress)) out[stage] = between$1(from, line.t);
+	}
+	const isOld = (address) => typeof address === "string" && address.startsWith(oldAddress);
+	for (const event of run.events) {
+		if (event.t < from) continue;
+		if (out.discovered === void 0 && event.ev === "sdam" && event["event"] === "server-changed" && !isOld(event["address"])) {
+			if (event["next"]?.type === "RSPrimary") out.discovered = between$1(from, event.t);
+		}
+		if (out["written-off"] === void 0 && event.ev === "sdam" && event["event"] === "heartbeat-failed" && isOld(event["address"])) out["written-off"] = between$1(from, event.t);
+	}
+	const accepted = acceptedAfter(run, sentAfter, lands);
+	if (accepted !== void 0) out["first-write"] = between$1(from, accepted.t);
+	return out;
+};
+/** This function gives the first write that a primary accepted, of a write sent after an instant. It is on a new primary for `new`. */
+const acceptedAfter = (run, sentAfter, lands) => {
+	const sent = /* @__PURE__ */ new Map();
+	for (const event of run.events) if (event.ev === "sent") sent.set(Number(event["n"]), event.t);
+	const primary = run.group?.primary ?? "";
+	return run.events.find((event) => {
+		if (event.ev !== "ack" || (sent.get(Number(event["n"])) ?? -Infinity) <= sentAfter) return false;
+		const member = memberAt(run, event["at"]);
+		return member !== null && (lands === "new" ? member !== primary : true);
+	});
+};
 /** This function gives what became of each write of a run, from the lines of the client and the keys that the group kept. */
 const writesOf = (run, after, primary) => {
 	const { events } = run;
@@ -470,7 +944,10 @@ const lagsOf = (run, kernel) => {
 * - The reference is the start of the commit of the fault, as the injector stamped it. For a
 *   stall, it is the first write into the fault, as for the stall of a direct pair.
 * - The outcome is the first write that a primary accepted after the fault. The prediction says
-*   which primary: a new one, or the same one after the lift.
+*   which primary: a new one, or the same one after the lift. In a replica set, it is the first
+*   accepted write of a write that the client sent after the commit of the fault.
+* - The stages come from the Sentinels, or from the logs of the members of a replica set and the
+*   lines of its client (`replicaSetStagesOf`).
 * - Its time is the last segment with data that the client's socket to that primary received
 *   before the client saw the acceptance. The stamp of the client is the fallback.
 * - A run is void when the prober lost records, or when the client did not become ready. It is
@@ -526,16 +1003,25 @@ const measureGroup = (cell, run) => {
 		group: empty
 	};
 	const timeline = run.group?.timeline ?? [];
-	const stages = stagesOf(timeline, t0);
 	const heal = run.heal ?? null;
 	const lands = expected?.lands ?? "new";
-	const after = lands === "same" && heal !== null ? heal.t0 : t0;
+	const replica = cell.replicaSet !== void 0;
+	const stall = prediction.detects === false && prediction.recovers !== void 0;
+	const stages = replica ? replicaSetStagesOf(run, t0, run.fault.t1, lands) : stagesOf(timeline, t0);
+	const after = lands === "same" && heal !== null && (!replica || stall) ? heal.t0 : t0;
+	const sentAt = /* @__PURE__ */ new Map();
+	for (const event of events) if (event.ev === "sent") sentAt.set(Number(event["n"]), event.t);
 	const accepted = events.find((event) => {
 		if (event.ev !== "ack" || event.t <= after) return false;
+		if (replica && (sentAt.get(Number(event["n"])) ?? -Infinity) <= run.fault.t1) return false;
 		const member = memberAt(run, event["at"]);
 		return member !== null && (lands === "new" ? member !== primary : true);
 	});
 	const to = accepted === void 0 ? null : memberAt(run, accepted["at"]);
+	if (replica) {
+		delete stages["first-write"];
+		if (accepted !== void 0) stages["first-write"] = between$1(t0, accepted.t);
+	}
 	const group = {
 		from: primary,
 		to,
@@ -560,7 +1046,7 @@ const measureGroup = (cell, run) => {
 		}
 	}
 	const assumed = prediction.detects === null ? void 0 : prediction.rtt;
-	const first = kernel.find((record) => record.netns === run.netns && record.dst === `${run.group?.members[primary]?.ip}:${cell.group?.port}` && record.tcp?.srtt !== void 0);
+	const first = kernel.find((record) => record.netns === run.netns && record.dst === `${run.group?.members[primary]?.ip}:${(cell.group ?? cell.replicaSet)?.port}` && record.tcp?.srtt !== void 0);
 	const rtt = first?.tcp?.srtt === void 0 ? null : valueIn(first.tcp.srtt, "us") / 1e3;
 	if (voided === null && assumed !== void 0 && rtt !== null && rtt > valueIn(assumed.high, "ms")) voided = `rtt-over-assumption: its first round trip was ${rtt} ms, above the ${valueIn(assumed.high, "ms")} ms that the prediction assumes`;
 	if (voided === null && prediction.detects === true && prediction.from === "fault" && (run.fault.t1 - run.fault.t0) / 1e6 > valueIn(NOISE, "ms")) voided = `the commit of the fault took ${Math.round((run.fault.t1 - run.fault.t0) / 1e5) / 10} ms, longer than the noise of the pipeline: its start is not known`;
@@ -576,8 +1062,12 @@ const measureGroup = (cell, run) => {
 		}
 	}
 	const findings = [];
-	const switched = stages["+switch-master"];
+	const switched = replica ? stages["elected"] : stages["+switch-master"];
 	if (expected?.failover === false && switched !== void 0) findings.push(`the group failed over ${switched.value} s after the fault, where it should not`);
+	for (const stage of expected?.quiet ?? []) {
+		const at = stages[stage];
+		if (at !== void 0) findings.push(`${stage} came ${at.value} s after the fault, where it should not`);
+	}
 	if (accepted !== void 0 && lands === "same" && to !== primary) findings.push(`the client wrote to ${to ?? String(accepted["at"])}, where it should stay on ${primary}`);
 	for (const error of expected?.errors ?? []) {
 		const met = group.errors.find((each) => each.name === error.name);
@@ -595,7 +1085,8 @@ const measureGroup = (cell, run) => {
 		};
 	}
 	const address = String(accepted["at"]);
-	const segment = kernel.filter((record) => record.netns === run.netns && record.ev === "rx" && record.dst === address && (record.seg?.len.value ?? 0) > 0 && record.mono <= accepted.t && record.mono > after).at(-1);
+	const others = new Set(socketsOf(events).filter((socket) => socket.role !== "pool").map((socket) => socket.local));
+	const segment = kernel.filter((record) => record.netns === run.netns && record.ev === "rx" && record.dst === address && !others.has(String(record.src)) && (record.seg?.len.value ?? 0) > 0 && record.mono <= accepted.t && record.mono > after).at(-1);
 	const at = segment?.mono ?? accepted.t;
 	return {
 		outcome: { kind: "answered" },
@@ -761,9 +1252,61 @@ const crossingsOf = (run, hop, toClient, toTarget) => {
 *   it, and before the first keepalive probe. A live peer answers that probe, and the answer
 *   starts the idle time again. A run where the fault was late is void. A run whose records the
 *   prober lost is also void.
+* - For an HTTP client, it also reads what the second request met (`requestOf`). An outcome other
+*   than the one that the prediction gives is a finding.
 */
 const measure = (cell, run) => {
-	if (cell.group !== void 0) {
+	const read = measureSocket(cell, run);
+	const expected = cell.prediction.request;
+	if (cell.probe.http === void 0 && expected === void 0) return read;
+	const request = requestOf(run.events);
+	if (expected === void 0 || read.void !== null) return {
+		...read,
+		request
+	};
+	const differs = requestFinding(expected, request);
+	const finding = read.finding === null ? differs : differs === null ? read.finding : `${read.finding}. ${differs}`;
+	return {
+		...read,
+		request,
+		finding
+	};
+};
+/** This function tells how a request reading differs from its prediction, or gives null. */
+const requestFinding = (expected, read) => {
+	if (read === null) return `the second request never went out, where it should meet: ${requestWords(expected)}`;
+	return read.connection === expected.connection && read.retried === expected.retried && read.error === expected.error && read.pending === (expected.pending === true) ? null : `the second request met: ${requestWords(read)}, not ${requestWords(expected)}`;
+};
+/**
+* This function reads the second request of an HTTP client from its lines. Its first try is its
+* first `request` line, and each `retry` line is a new try. Its end is its `error` line, or its
+* `replied` line. A request with neither is pending. It gives null when the client wrote no try.
+*
+* A new try that gets no connection writes no `retry` line. Its dial writes a `dial-error` line.
+* Thus a `dial-error` line after the first try and before the end is also a new try.
+*/
+const requestOf = (events) => {
+	const of = (event) => Number(event["i"]) === 2;
+	const tries = events.filter((event) => (event.ev === "request" || event.ev === "retry") && of(event));
+	const first = tries[0];
+	if (first === void 0) return null;
+	const failed = events.find((event) => event.ev === "error" && of(event));
+	const replied = events.some((event) => event.ev === "replied" && of(event));
+	const ready = events.find((event) => event.ev === "ready");
+	const sent = events.find((event) => event.ev === "sent" && of(event));
+	const end = failed ?? events.find((event) => event.ev === "done" && of(event));
+	const dialed = events.some((event) => event.ev === "dial-error" && event.t > first.t && (end === void 0 || event.t <= end.t));
+	return {
+		connection: first["reused"] === true ? "reused" : "new",
+		retried: dialed || tries.some((each) => each.ev === "retry"),
+		error: failed === void 0 ? null : String(failed["errno"]),
+		pending: failed === void 0 && !replied,
+		gap: ready === void 0 || sent === void 0 ? null : between(ready.t, sent.t)
+	};
+};
+/** This function reads the socket of one run against its cell, as `measure` does without the second request of an HTTP client. */
+const measureSocket = (cell, run) => {
+	if (cell.group !== void 0 || cell.replicaSet !== void 0) {
 		const read = measureGroup(cell, run);
 		return {
 			environment: run.environment,
@@ -781,10 +1324,11 @@ const measure = (cell, run) => {
 			group: read.group
 		};
 	}
+	if (cell.prediction.detects !== null && cell.prediction.driver !== void 0) return measureDriver(cell, run);
 	const { events, kernel } = run;
 	const ready = events.find((event) => event.ev === "ready");
 	const local = events.find((event) => event.ev === "connected")?.["local"];
-	const inflight = cell.probe.mode !== "idle";
+	const inflight = cell.probe.mode !== "idle" && cell.probe.mode !== "gap";
 	const request = inflight ? events.filter((event) => event.ev === "sent")[1] : ready;
 	const server = cell.observer === "server";
 	const place = socketOf(cell, run);
@@ -873,20 +1417,22 @@ const measure = (cell, run) => {
 	const reported = socket.find((record) => record.ev === "error" && (record.err ?? 0) !== 0);
 	const seen = remote ? void 0 : events.find((event) => event.ev === "error" || event.ev === "eof");
 	const closeWait = socket.find((record) => record.ev === "state" && newState(record) === 8);
+	const own = cell.prediction.detects === true && cell.prediction.errno === "CLOSED" ? socket.find((record) => record.ev === "state" && oldState(record) === 1 && newState(record) === 4) : void 0;
 	const outcome = reported !== void 0 ? {
 		kind: "error",
 		errno: ERRNO[reported.err] ?? `E${reported.err}`
-	} : closeWait !== void 0 || seen?.ev === "eof" ? { kind: "eof" } : seen !== void 0 ? {
+	} : closeWait !== void 0 || seen?.ev === "eof" ? { kind: "eof" } : own !== void 0 ? { kind: "closed" } : seen !== void 0 ? {
 		kind: "error",
 		errno: String(seen["errno"])
 	} : { kind: "silent" };
-	const ended = reported?.mono ?? closeWait?.mono ?? seen?.t;
+	const ended = reported?.mono ?? closeWait?.mono ?? own?.mono ?? seen?.t;
 	const noticed = reported !== void 0 && seen !== void 0 ? between(reported.mono, seen.t) : null;
 	const { prediction } = cell;
 	const before = run.fault?.t1 ?? ready.t;
 	const referenceAt = (from) => {
 		if (from === "request-sent") return request.t;
 		if (from === "fault") return run.fault?.t0;
+		if (from === "client-start") return (events.find((event) => event.ev === "connecting") ?? events[0])?.t;
 		return (from === "last-received" ? socket.filter((each) => each.ev === "rx" && each.mono <= before).at(-1) : from === "first-retransmission" ? retransmitted.find((each) => each.why === "rto") : kernel.find((each) => each.netns === peerNetns && each.cookie === peerCookie && each.ev === "state" && oldState(each) === 1))?.mono;
 	};
 	const lastRx = kernel.filter((each) => each.netns === run.netns && each.src === local && each.ev === "rx" && each.mono <= before).at(-1)?.mono ?? ready.t;
@@ -1056,8 +1602,9 @@ const measure = (cell, run) => {
 		finding: `it ended without its reference event (${prediction.from})`
 	};
 	const steps = (prediction.from === "last-received" ? keepalives : prediction.from === "first-retransmission" ? retransmitted.filter((record) => record.why === "rto" && record.mono > reference) : []).map((record) => between(reference, record.mono));
-	const finding = (prediction.errno === "EOF" ? outcome.kind === "eof" : outcome.kind === "error" && outcome.errno === prediction.errno) ? null : `it ended with ${outcome.kind === "error" ? outcome.errno : "end of file"}, not ${prediction.errno}`;
-	if (prediction.from === "last-received" || prediction.from === "first-retransmission") voided ??= delayed(reference, ended, prediction.from === "last-received" ? "keepalive" : "retransmit");
+	const finding = (prediction.errno === "EOF" ? outcome.kind === "eof" : prediction.errno === "CLOSED" ? outcome.kind === "closed" : outcome.kind === "error" && outcome.errno === prediction.errno) ? null : `it ended with ${outcome.kind === "error" ? outcome.errno : outcome.kind === "closed" ? "its own close" : "end of file"}, not ${prediction.errno}`;
+	const programTimer = cell.probe.http !== void 0 && prediction.from === "last-received" && prediction.timer !== "keepalive";
+	if (prediction.from === "last-received" && !programTimer || prediction.from === "first-retransmission") voided ??= delayed(reference, ended, prediction.from === "last-received" ? "keepalive" : "retransmit");
 	const lag = (valueIn(prediction.high, "s") - valueIn(prediction.low, "s")) * 1e9;
 	const slow = prediction.timer.startsWith("expire_") ? (run.middlebox ?? []).find((line) => line.ev === "pump-slow" && line.t <= ended && line.t + Number(line["took"]) * 1e9 >= ended - lag) : void 0;
 	if (slow !== void 0) voided ??= `a listing of the router took ${Math.round(Number(slow["took"]) * 1e4) / 10} ms, longer than the bound of its prediction`;
@@ -1114,6 +1661,24 @@ const NODE = {
 	dockerfile: "harness/image/node/Dockerfile",
 	from: ["harness/image/node"]
 };
+/** This image holds the Go HTTP client: the http.Transport of Go 1.27.2. */
+const GO_HTTP = {
+	name: "loom-harness-go-http",
+	dockerfile: "harness/image/go-http/Dockerfile",
+	from: ["harness/image/go-http"]
+};
+/** This image holds the Node HTTP clients: the http.Agent of Node 24.21.0, and undici 8.12.2. */
+const NODE_HTTP = {
+	name: "loom-harness-node-http",
+	dockerfile: "harness/image/node-http/Dockerfile",
+	from: ["harness/image/node-http"]
+};
+/** This image holds the MongoDB client: the driver mongodb 7.7.0 on Node 24.21.0, and its step-down script. */
+const MONGO = {
+	name: "loom-harness-mongo",
+	dockerfile: "harness/image/mongo/Dockerfile",
+	from: ["harness/image/mongo"]
+};
 /** This image is the router of a middlebox chain: NAT and conntrack, with a form for a forgotten flow. */
 const ROUTER = {
 	name: "loom-harness-router",
@@ -1180,6 +1745,12 @@ const harnessImage = () => build(HARNESS);
 const nodeClientImage = () => build(NODE);
 /** This function gives the image of the router. */
 const routerImage = () => build(ROUTER);
+/** This function gives the image of the Go HTTP client. */
+const goHttpImage = () => build(GO_HTTP);
+/** This function gives the image of the Node HTTP clients. */
+const nodeHttpImage = () => build(NODE_HTTP);
+/** This function gives the image of the MongoDB client. */
+const mongoClientImage = () => build(MONGO);
 /**
 * This time is the interval of the listing of a router, when its hop does not set one. A listing
 * evicts each expired flow that it walks past, and the DESTROY event of that flow follows.
@@ -1322,6 +1893,21 @@ const clientOf = async (kind, image) => {
 		command: ["node", "/opt/harness/node/client-ioredis.mjs"],
 		clientKind: kind
 	};
+	if (kind === "go-http") return {
+		clientImage: await goHttpImage(),
+		command: ["/opt/harness/client-go"],
+		clientKind: kind
+	};
+	if (kind === "http-agent" || kind === "undici") return {
+		clientImage: await nodeHttpImage(),
+		command: ["node", `/opt/harness/node-http/client-${kind === "undici" ? "undici" : "agent"}.mjs`],
+		clientKind: kind
+	};
+	if (kind === "mongodb") return {
+		clientImage: await mongoClientImage(),
+		command: ["node", "/opt/harness/mongo/client-mongo.mjs"],
+		clientKind: kind
+	};
 	return {
 		clientImage: image,
 		command: ["/opt/harness/client"],
@@ -1362,6 +1948,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 	const runtime = await getContainerRuntimeClient();
 	const opened = [];
 	const networks = [];
+	const serverLines = lines();
 	const close = async () => {
 		for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
 		for (const network of networks.reverse()) await network.stop().catch(() => void 0);
@@ -1381,7 +1968,12 @@ const openLab = async (target, kind = "tcp", topology) => {
 	try {
 		for (let at = 0; at <= hops.length; at += 1) networks.push(await new Network().start());
 		const last = networks[hops.length];
-		const stock = new GenericContainer(target.image).withNetwork(last).withWaitStrategy(Wait.forLogMessage(target.ready));
+		let stock = new GenericContainer(target.image).withNetwork(last).withWaitStrategy(Wait.forLogMessage(target.ready));
+		if (target.files !== void 0) stock = stock.withCopyContentToContainer(Object.entries(target.files).map(([path, content]) => ({
+			content,
+			target: path
+		})));
+		if (target.lines === true) stock = stock.withLogConsumer(serverLines.consume);
 		const server = await (target.command === void 0 ? stock : stock.withCommand([...target.command])).start();
 		opened.push(server);
 		const serverIp = await ipOn(server, last);
@@ -1569,6 +2161,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 		};
 		const paused = runtime.container.getById(server.getId());
 		let targetAdmin;
+		let signaller;
 		const hopNamed = (name) => {
 			const found = inChain.find((hop) => hop.hop.name === name);
 			if (found === void 0) throw new Error(`the topology has no hop ${name}`);
@@ -1633,6 +2226,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 			netns,
 			topology: placed,
 			middlebox,
+			server: serverLines.seen,
 			kernel: records.seen,
 			async sysctl(values) {
 				const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
@@ -1830,6 +2424,19 @@ const openLab = async (target, kind = "tcp", topology) => {
 			},
 			freeze: () => paused.pause().then(() => void 0),
 			thaw: () => paused.unpause().then(() => void 0),
+			async signal(signal, at) {
+				if (target.program === void 0) throw new Error(`${target.image} names no server process to signal`);
+				signaller ??= await withPid(new GenericContainer(image).withNetworkMode(`container:${server.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW"), `container:${server.getId()}`).start();
+				if (!opened.includes(signaller)) opened.push(signaller);
+				const when = at === void 0 ? [] : ["--at", String(Math.round(at))];
+				return stamped(signaller, [
+					"/opt/harness/inject",
+					"signal",
+					target.program,
+					signal,
+					...when
+				]);
+			},
 			async probe(args) {
 				const events = lines();
 				const container = await new GenericContainer(clientImage).withNetworkMode(`container:${client.getId()}`).withCommand([...command, ...args]).withLogConsumer(events.consume).start();
@@ -1898,7 +2505,9 @@ const openLab = async (target, kind = "tcp", topology) => {
 						image: target.image,
 						id: (await inspect(server)).Image,
 						settings,
-						sysctls: targetSysctls
+						sysctls: targetSysctls,
+						...target.files === void 0 ? {} : { files: Object.fromEntries(Object.entries(target.files).map(([path, content]) => [path, createHash("sha256").update(content).digest("hex")])) },
+						...serverLines.seen[0] === void 0 ? {} : { first: serverLines.seen[0] }
 					},
 					harness: {
 						image,
@@ -2044,7 +2653,7 @@ const viewOf = (group, ips, sentinels, members) => {
 		why: "healthy"
 	};
 };
-const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const sleep$2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** This time is the longest that a group takes to converge before a run, or after it. */
 const CONVERGE_MS = 24e4;
 /**
@@ -2239,7 +2848,7 @@ const openGroupLab = async (group, options = {}) => {
 					}));
 					calm = last.healthy ? calm + 1 : 0;
 					if (calm >= 2 && last.primary !== null) return last.primary;
-					await sleep$1(1e3);
+					await sleep$2(1e3);
 				}
 				throw new Error(`the group did not converge in ${within} ms: ${last.why}`);
 			},
@@ -2317,7 +2926,7 @@ const openGroupLab = async (group, options = {}) => {
 				const started = Date.now();
 				while (await cli(member, ["PING"]).catch(() => "") !== "PONG") {
 					if (Date.now() - started > 3e4) throw new Error(`${member} did not start again`);
-					await sleep$1(200);
+					await sleep$2(200);
 				}
 			},
 			failover(coordinated) {
@@ -2435,9 +3044,9 @@ const openGroupLab = async (group, options = {}) => {
 	}
 };
 /** This count is the number of accepted writes that a run waits for before its fault: the stream is steady then. */
-const STEADY = 20;
+const STEADY$1 = 20;
 /** This time is how long a run continues after its outcome, so that the record shows what came next. */
-const TAIL_MS = 2e3;
+const TAIL_MS$2 = 2e3;
 /** This function gives the address of a member, with the port of the group. */
 const addressOf = (lab, member) => `${lab.members[member].ip}:${lab.group.port}`;
 /** This function gives the latest time that a group cell waits for its outcome, in ms from the fault. */
@@ -2460,7 +3069,7 @@ const clientArgs = (cell, lab) => {
 	];
 };
 /** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
-const waitFor = (client, match, within) => new Promise((resolve) => {
+const waitFor$1 = (client, match, within) => new Promise((resolve) => {
 	const started = Date.now();
 	const look = () => {
 		const found = client.events.find(match);
@@ -2476,7 +3085,7 @@ const subscriptionsOf = (timeline) => {
 	return [...latest.values()];
 };
 /** This function gives the count of lost records in the latest clock line of the prober. */
-const droppedSoFar$1 = (records) => {
+const droppedSoFar$2 = (records) => {
 	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
 	return 0;
 };
@@ -2498,7 +3107,7 @@ const groupOnce = async (lab, cell, environment, index) => {
 	await lab.forget();
 	const from = lab.kernel.length;
 	const timelineFrom = lab.timeline.length;
-	const lostBefore = droppedSoFar$1(lab.kernel);
+	const lostBefore = droppedSoFar$2(lab.kernel);
 	const client = await lab.client(clientArgs(cell, lab));
 	const lands = cell.prediction.group?.lands ?? "new";
 	const place = cell.faultAt ?? { role: "primary" };
@@ -2508,7 +3117,7 @@ const groupOnce = async (lab, cell, environment, index) => {
 	const undo = [];
 	try {
 		await client.until(["ready"], 6e4);
-		if (await waitFor(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY, 3e4) === void 0) throw new Error(`the client did not write ${STEADY} times in 30 s`);
+		if (await waitFor$1(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$1, 3e4) === void 0) throw new Error(`the client did not write ${STEADY$1} times in 30 s`);
 		let healing = null;
 		switch (cell.fault) {
 			case "blackhole":
@@ -2522,7 +3131,7 @@ const groupOnce = async (lab, cell, environment, index) => {
 				undo.push(() => lab.lift(kind, part, cuts));
 				if (cell.stall !== void 0) {
 					const t1 = fault.t1;
-					const sent = await waitFor(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
+					const sent = await waitFor$1(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
 					if (sent !== void 0) healing = lab.lift(kind, part, cuts, sent.t + valueIn(cell.stall, "s") * 1e9);
 				} else if (cell.hold !== void 0) healing = lab.lift(kind, part, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
 				break;
@@ -2556,11 +3165,11 @@ const groupOnce = async (lab, cell, environment, index) => {
 		const t0 = fault.t0;
 		const old = addressOf(lab, primary);
 		const deadline = deadlineOf(cell);
-		if ((lands === "new" ? await waitFor(client, (event) => event.ev === "ack" && event.t > t0 && typeof event["at"] === "string" && event["at"] !== old, deadline) : await (async () => {
+		if ((lands === "new" ? await waitFor$1(client, (event) => event.ev === "ack" && event.t > t0 && typeof event["at"] === "string" && event["at"] !== old, deadline) : await (async () => {
 			await healed;
 			const after = heal?.t0 ?? t0;
-			return waitFor(client, (event) => event.ev === "ack" && event.t > after, deadline);
-		})()) !== void 0) await sleep$1(TAIL_MS);
+			return waitFor$1(client, (event) => event.ev === "ack" && event.t > after, deadline);
+		})()) !== void 0) await sleep$2(TAIL_MS$2);
 		await healed;
 	} finally {
 		for (const step of undo.reverse()) await step().catch(() => void 0);
@@ -2593,66 +3202,730 @@ const groupOnce = async (lab, cell, environment, index) => {
 	};
 };
 //#endregion
-//#region src/run.ts
-/**
-* This function measures and judges the raw records of a cell, with the code of the harness at
-* this time. `runCell` uses it during a cell. A holder of the raw records of a cell can use it
-* again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.
-*/
-const judge = (cell, runs) => {
-	const measured = runs.filter((run) => run.warmup !== true).map((run) => measure(cell, run));
+//#region src/replset.ts
+/** These are the components of the log of mongod that a run keeps: replication, its heartbeats, the elections, the rollbacks and the commands. */
+const KEPT = /* @__PURE__ */ new Set([
+	"REPL",
+	"REPL_HB",
+	"ELECTION",
+	"ROLLBACK",
+	"COMMAND"
+]);
+/** This function reads one log line of a member, or gives null for a line of another component or a line that is not JSON. */
+const memberLineOf = (member, text) => {
+	if (!text.startsWith("{")) return null;
+	let line;
+	try {
+		line = JSON.parse(text);
+	} catch {
+		return null;
+	}
+	const real = Date.parse(line.t?.$date ?? "");
+	if (!Number.isFinite(real) || !KEPT.has(line.c ?? "")) return null;
 	return {
-		measured,
-		verdict: verdictOf(cell.prediction, measured)
+		ev: "mongod",
+		t: 0,
+		member,
+		real,
+		id: Number(line.id ?? 0),
+		c: line.c ?? "",
+		msg: line.msg ?? "",
+		attr: line.attr ?? {}
 	};
 };
-/** This function gives the count of lost records in the latest clock line of the prober, since it began. */
-const droppedSoFar = (records) => {
-	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
-	return 0;
-};
-/** This function gives the argv of the client for a cell, with a deadline far after all that it waits for. */
-const probeArgs = (cell, lab) => {
-	const { prediction, probe } = cell;
-	const longest = (prediction.detects ? valueIn(prediction.high, "s") : valueIn(prediction.window, "s")) + (cell.idleBefore === void 0 ? 0 : valueIn(cell.idleBefore, "s"));
-	const args = [
-		lab.targetIp,
-		String(lab.targetPort),
-		"--mode",
-		probe.mode
-	];
-	if (probe.request !== void 0) args.push("--request", probe.request);
-	if (probe.count !== void 0) args.push("--count", String(probe.count));
-	if (probe.interval !== void 0) args.push("--interval-ms", String(valueIn(probe.interval, "ms")));
-	if (probe.driver !== void 0) {
-		args.push("--keepalive-ms", String(valueIn(probe.driver.keepAlive, "ms")));
-		args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
-		return args;
-	}
-	if (probe.keepalive) args.push("--keepalive");
-	if (probe.userTimeout !== void 0) args.push("--user-timeout-ms", String(valueIn(probe.userTimeout, "ms")));
-	args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
-	return args;
-};
 /**
-* This function waits for an error on the socket of the observer, or for its end of file
-* (CLOSE_WAIT, state 8). The prober records both. A cell whose observer is the server, or a proxy
-* of a topology, waits for that. The promise resolves after `within` ms at the latest.
+* This function gives the offset of the wall clock from the monotonic clock, in ns, from the clock
+* lines of a client. Each line pairs its stamp with the wall clock in whole ms, which truncates.
+* Thus each pair is the offset or less, and the largest pair is the nearest. It is null without a clock line.
 */
-const observerError = (cell, partial, within) => new Promise((resolve) => {
-	const started = Date.now();
-	const look = () => {
-		const run = partial();
-		const at = socketOf(cell, run);
-		if (at !== void 0 && run.kernel.some((record) => record.netns === at.netns && record.cookie === at.cookie && (record.ev === "error" && (record.err ?? 0) !== 0 || record.ev === "state" && record.new === 8)) || Date.now() - started > within) resolve();
-		else setTimeout(look, 50);
-	};
-	look();
-});
-const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const clockOffsetOf = (events) => {
+	const offsets = events.filter((event) => event.ev === "clocks").map((event) => Number(event["real"]) * 1e6 - event.t);
+	return offsets.length === 0 ? null : Math.max(...offsets);
+};
+/** This function puts the log lines of the members on the monotonic clock, with the offset of a client. */
+const onMonotonic = (log, offset) => log.map((line) => ({
+	...line,
+	t: offset === null ? 0 : Math.round(line.real * 1e6 - offset)
+}));
 /**
-* This function waits for the request that an in-flight client sends on the signal: its second
-* send, after the PING that made it ready. Its stamp is when the request first met the fault.
+* This function gives the peers that a fault on a member of a replica set cuts it from. `all` is
+* each other member and the client, `group` is the other members, and `clients` is the client.
+* `one` is the first other member in the sequence of the set, and `one-way` is the same member for
+* the segments from the part only.
+*/
+const cutOf = (set, members, client, part, cuts) => {
+	const others = set.members.filter((name) => name !== part).map((name) => members[name].ip);
+	if (cuts === "clients") return { peers: [client] };
+	if (cuts === "group") return { peers: others };
+	if (cuts === "one") return { peers: others.slice(0, 1) };
+	if (cuts === "one-way") return {
+		peers: others.slice(0, 1),
+		direction: "out"
+	};
+	return { peers: [...others, client] };
+};
+/** This function gives the command of the server of each member. The server is the child of a shell, thus a fin can stop it and the container stays up. */
+const memberCommand = (set) => `${set.programs.server} --replSet ${set.name} --bind_ip_all --port ${set.port} ${(set.args ?? []).join(" ")}`.trim();
+/** This function gives the script of the shell that starts the set: each member by its address, and the settings of the set. */
+const initiateScript = (set, ips) => {
+	const config = {
+		_id: set.name,
+		members: set.members.map((name, at) => ({
+			_id: at,
+			host: `${ips[name]}:${set.port}`
+		})),
+		settings: set.settings
+	};
+	return `print(JSON.stringify(rs.initiate(${JSON.stringify(config)})))`;
+};
+/** This script prints the view of a member: itself, if it accepts writes, and the state of each member as it sees them. */
+const STATUS_SCRIPT = "const s = rs.status(); const h = db.hello(); print(JSON.stringify({ me: h.me, writable: h.isWritablePrimary === true, members: s.members.map((m) => ({ name: m.name, state: m.stateStr, health: m.health, optime: m.optimeDate ? m.optimeDate.getTime() : null })) }))";
+/**
+* This function judges the view of a member: the primary of the set, and if the set is healthy.
+* The set is healthy when the member is the one primary and accepts writes. Each other member must
+* be a healthy secondary at the optime of the primary.
+*/
+const statusViewOf = (set, ips, status) => {
+	const byAddress = new Map(set.members.map((name) => [`${ips[name]}:${set.port}`, name]));
+	const primaries = status.members.filter((member) => member.state === "PRIMARY");
+	if (primaries.length !== 1) return {
+		primary: null,
+		healthy: false,
+		why: `the set has ${primaries.length} primaries`
+	};
+	const primary = byAddress.get(primaries[0].name) ?? null;
+	if (primary === null) return {
+		primary: null,
+		healthy: false,
+		why: `no member has the address ${primaries[0].name}`
+	};
+	if (status.me !== primaries[0].name || !status.writable) return {
+		primary,
+		healthy: false,
+		why: `${primary} does not accept writes yet`
+	};
+	if (status.members.length !== set.members.length) return {
+		primary,
+		healthy: false,
+		why: `the set has ${status.members.length} members`
+	};
+	for (const member of status.members) {
+		if (member.health !== 1) return {
+			primary,
+			healthy: false,
+			why: `${member.name} is not healthy`
+		};
+		if (member.state !== "PRIMARY" && member.state !== "SECONDARY") return {
+			primary,
+			healthy: false,
+			why: `${member.name} is ${member.state}`
+		};
+		if (member.optime !== primaries[0].optime) return {
+			primary,
+			healthy: false,
+			why: `${member.name} is behind the primary`
+		};
+	}
+	return {
+		primary,
+		healthy: true,
+		why: "healthy"
+	};
+};
+const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/** These are the arguments of replSetStepDown: the time that the old primary cannot stand again, and the wait for a secondary to catch up. */
+const STEP_DOWN = {
+	seconds: 60,
+	catchUp: 10
+};
+/**
+* This function opens one environment of a replica set cell:
+* - one new network, with each member, the client and the sender of a step-down on it.
+* - each member: the stock image, its server as the child of a shell. A log consumer reads its
+*   JSON log. The first member starts the set with the address of each member, and the lab
+*   waits for a primary.
+* - an injector sidecar for each member, in its network and process namespaces.
+* - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
+* - the prober, which watches the namespace of the client.
+*/
+const openReplicaSetLab = async (set) => {
+	const image = await harnessImage();
+	const mongo = await mongoClientImage();
+	const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();
+	const runtime = await getContainerRuntimeClient();
+	const opened = [];
+	let network;
+	const close = async () => {
+		for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
+		await network?.stop().catch(() => void 0);
+	};
+	const ipOf = async (container) => {
+		const ip = (await runtime.container.inspect(runtime.container.getById(container.getId()))).NetworkSettings.Networks[network.getName()]?.IPAddress;
+		if (ip === void 0 || ip === "") throw new Error(`${container.getName()} has no address on ${network.getName()}`);
+		return ip;
+	};
+	try {
+		network = await new Network().start();
+		const boxes = /* @__PURE__ */ new Map();
+		const injectors = /* @__PURE__ */ new Map();
+		const members = {};
+		const log = [];
+		for (const name of set.members) {
+			let partial = "";
+			const box = await new GenericContainer(set.image).withNetwork(network).withNetworkAliases(name).withCommand([
+				"sh",
+				"-c",
+				`${memberCommand(set)} & echo $! > /tmp/server.pid; while :; do sleep 3600; done`
+			]).withWaitStrategy(Wait.forLogMessage(set.ready)).withLogConsumer((stream) => stream.on("data", (chunk) => {
+				partial += chunk.toString();
+				const split = partial.split("\n");
+				partial = split.pop() ?? "";
+				for (const text of split) {
+					const line = memberLineOf(name, text.trim());
+					if (line !== null) log.push(line);
+				}
+			})).start();
+			opened.push(box);
+			boxes.set(name, box);
+			const injector = await withPid(new GenericContainer(image).withNetworkMode(`container:${box.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW"), `container:${box.getId()}`).start();
+			opened.push(injector);
+			injectors.set(name, injector);
+			members[name] = {
+				ip: await ipOf(box),
+				netns: await netnsOf(injector)
+			};
+		}
+		const ips = Object.fromEntries(Object.entries(members).map(([name, at]) => [name, at.ip]));
+		const holder = await new GenericContainer(image).withNetwork(network).start();
+		opened.push(holder);
+		const admin = await new GenericContainer(image).withNetworkMode(`container:${holder.getId()}`).withPrivilegedMode().start();
+		opened.push(admin);
+		const clientIp = await ipOf(holder);
+		const commander = await new GenericContainer(mongo).withNetwork(network).withCommand([
+			"sh",
+			"-c",
+			"while :; do sleep 3600; done"
+		]).start();
+		opened.push(commander);
+		const clientNetns = await netnsOf(admin);
+		const records = lines();
+		const prober = await new GenericContainer(image).withPrivilegedMode().withNetworkMode("none").withCommand(["/opt/harness/prober", String(clientNetns)]).withLogConsumer(records.consume).start();
+		opened.push(prober);
+		const begun = await records.until(["begin", "fail"], 3e4, "prober");
+		if (begun.ev === "fail") throw new Error(`the prober failed: ${String(begun["why"])}`);
+		const box = (name) => {
+			const found = boxes.get(name);
+			if (found === void 0) throw new Error(`the set has no member ${name}`);
+			return found;
+		};
+		const injectorOf = (name) => {
+			const found = injectors.get(name);
+			if (found === void 0) throw new Error(`the set has no member ${name}`);
+			return found;
+		};
+		/** This function starts a script of the shell on a member, against its own server only. */
+		const shell = (name, script) => output(box(name), [
+			set.programs.shell,
+			"--quiet",
+			"--norc",
+			`mongodb://127.0.0.1:${set.port}/?directConnection=true&serverSelectionTimeoutMS=3000`,
+			"--eval",
+			script
+		]);
+		const lastLine = (text) => text.split("\n").filter((line) => line.trim() !== "").at(-1) ?? "";
+		const stamped = async (where, argv) => {
+			const stamp = JSON.parse(lastLine(await output(where, argv)));
+			return {
+				t0: stamp.t0,
+				t1: stamp.t1
+			};
+		};
+		const after = (at) => at === void 0 ? [] : ["--at", String(Math.round(at))];
+		let known = set.members[0];
+		/** This function gives the view of the set: from the member that was the primary last, then from each other member. */
+		const view = async () => {
+			let last = {
+				primary: null,
+				healthy: false,
+				why: "no member answered"
+			};
+			for (const name of [known, ...set.members.filter((each) => each !== known)]) {
+				const read = await shell(name, STATUS_SCRIPT).then((text) => statusViewOf(set, ips, JSON.parse(lastLine(text)))).catch((error) => ({
+					primary: null,
+					healthy: false,
+					why: `${name}: ${error.message.slice(0, 200)}`
+				}));
+				if (read.primary !== null && read.primary === name) return read;
+				last = read;
+			}
+			return last;
+		};
+		const converge = async (within = CONVERGE_MS) => {
+			const started = Date.now();
+			let calm = 0;
+			let last = {
+				primary: null,
+				healthy: false,
+				why: "no view yet"
+			};
+			while (Date.now() - started < within) {
+				last = await view();
+				calm = last.healthy ? calm + 1 : 0;
+				if (last.primary !== null) known = last.primary;
+				if (calm >= 2 && last.primary !== null) return last.primary;
+				await sleep$1(1e3);
+			}
+			throw new Error(`the set did not converge in ${within} ms: ${last.why}`);
+		};
+		const initiated = await shell(set.members[0], initiateScript(set, ips));
+		if (!lastLine(initiated).includes("\"ok\":1")) throw new Error(`rs.initiate failed: ${lastLine(initiated)}`);
+		await converge();
+		return {
+			set,
+			clientIp,
+			members,
+			netns: { client: clientNetns },
+			kernel: records.seen,
+			log,
+			async sysctl(values) {
+				const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
+				if (settings.length > 0) await output(admin, [
+					"sysctl",
+					"-q",
+					"-w",
+					...settings
+				]);
+				const read = {};
+				for (const key of Object.keys(values)) read[key] = await output(admin, [
+					"sysctl",
+					"-n",
+					key
+				]);
+				return read;
+			},
+			converge,
+			async flush(primary) {
+				const script = `const r = db.getSiblingDB("cells").w.deleteMany({}, { writeConcern: { w: ${set.members.length}, wtimeout: 10000 } }); print(JSON.stringify({ ok: r.acknowledged === true }))`;
+				const reply = lastLine(await shell(primary, script));
+				if (!reply.includes("\"ok\":true")) throw new Error(`the flush failed: ${reply}`);
+			},
+			async unfreeze(primary) {
+				for (const name of set.members) {
+					if (name === primary) continue;
+					const reply = lastLine(await shell(name, "print(JSON.stringify(rs.freeze(0)))"));
+					if (!reply.includes("\"ok\":1")) throw new Error(`${name} did not unfreeze: ${reply}`);
+				}
+			},
+			async holds(primary) {
+				return JSON.parse(lastLine(await shell(primary, "print(JSON.stringify(db.getSiblingDB(\"cells\").w.find({}, { n: 1, _id: 0 }).toArray().map((d) => d.n)))"))).map(Number).filter((n) => Number.isInteger(n) && n > 0).sort((a, b) => a - b);
+			},
+			partOf(role, primary) {
+				if (role === "primary") return primary;
+				if (role === "sentinel") throw new Error("a replica set has no Sentinel");
+				const secondary = set.members.find((name) => name !== primary);
+				if (secondary === void 0) throw new Error("the set has no secondary");
+				return secondary;
+			},
+			async client(args) {
+				const events = lines();
+				const container = await new GenericContainer(mongo).withNetworkMode(`container:${holder.getId()}`).withCommand([
+					"node",
+					"/opt/harness/mongo/client-mongo.mjs",
+					...args
+				]).withLogConsumer(events.consume).start();
+				return {
+					events: events.seen,
+					until: (kinds, within) => events.until(kinds, within, "client"),
+					stop: () => container.stop({ timeout: 2 }).then(() => void 0).catch(() => void 0)
+				};
+			},
+			fault(kind, part, cuts, at) {
+				const cut = cutOf(set, members, clientIp, part, cuts);
+				const direction = cut.direction === void 0 ? [] : ["--direction", cut.direction];
+				return stamped(injectorOf(part), [
+					"/opt/harness/inject",
+					kind,
+					cut.peers.join(","),
+					...direction,
+					...after(at)
+				]);
+			},
+			lift(kind, part, cuts, at) {
+				const cut = cutOf(set, members, clientIp, part, cuts);
+				const direction = cut.direction === void 0 ? [] : ["--direction", cut.direction];
+				return stamped(injectorOf(part), [
+					"/opt/harness/inject",
+					kind === "rst" ? "clear-rst" : "clear",
+					cut.peers.join(","),
+					...direction,
+					...after(at)
+				]);
+			},
+			signal(part, signal, at) {
+				return stamped(injectorOf(part), [
+					"/opt/harness/inject",
+					"signal",
+					set.programs.server,
+					signal,
+					...after(at)
+				]);
+			},
+			async restart(member) {
+				await output(box(member), [
+					"sh",
+					"-c",
+					`${memberCommand(set)} > /proc/1/fd/1 2>&1 & echo $! > /tmp/server.pid`
+				]);
+				const started = Date.now();
+				while (!(await shell(member, "print(db.hello().ok)").catch(() => "")).includes("1")) {
+					if (Date.now() - started > 6e4) throw new Error(`${member} did not start again`);
+					await sleep$1(500);
+				}
+			},
+			stepDown(primary) {
+				return stamped(commander, [
+					"node",
+					"/opt/harness/mongo/stepdown-mongo.mjs",
+					`${members[primary].ip}:${set.port}`,
+					String(STEP_DOWN.seconds),
+					String(STEP_DOWN.catchUp)
+				]);
+			},
+			async forget() {
+				await output(admin, [
+					"ip",
+					"tcp_metrics",
+					"flush",
+					"all"
+				]);
+			},
+			async settle() {
+				const from = records.seen.length;
+				const clock = await new Promise((resolve) => {
+					const started = Date.now();
+					const look = () => {
+						const line = records.seen.slice(from).find((record) => record.ev === "clock");
+						if (line !== void 0) resolve(line);
+						else if (Date.now() - started > 5e3) resolve(null);
+						else setTimeout(look, 50);
+					};
+					look();
+				});
+				return clock === null ? null : Number(clock["lost"] ?? 0);
+			},
+			async manifest(sysctls) {
+				const config = await output(admin, ["zcat", "/proc/config.gz"]).catch(() => "");
+				const inspect = async (container) => runtime.container.inspect(runtime.container.getById(container.getId()));
+				const script = "print(JSON.stringify({ version: db.version(), settings: rs.conf().settings }))";
+				const list = [];
+				for (const name of set.members) {
+					const read = JSON.parse(lastLine(await shell(name, script)));
+					const settings = { version: read.version };
+					for (const key of Object.keys(set.settings)) settings[key] = String(read.settings[key] ?? "");
+					list.push({
+						name,
+						ip: members[name].ip,
+						id: (await inspect(box(name))).Image,
+						settings
+					});
+				}
+				return {
+					kernel: await output(holder, ["uname", "-r"]),
+					architecture: await output(holder, ["uname", "-m"]),
+					hz: /^CONFIG_HZ=(\d+)$/m.exec(config)?.[1] ?? null,
+					boot_id: await output(holder, ["cat", "/proc/sys/kernel/random/boot_id"]),
+					docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
+					target: {
+						image: set.image,
+						id: list[0].id,
+						settings: list[0].settings,
+						sysctls: {}
+					},
+					harness: {
+						image,
+						id: (await inspect(holder)).Image,
+						packages: (await output(holder, [
+							"apk",
+							"info",
+							"-v"
+						])).split("\n").sort()
+					},
+					client: {
+						kind: "mongodb",
+						image: mongo
+					},
+					netns: {
+						client: clientNetns,
+						target: members[set.members[0]].netns
+					},
+					sysctls,
+					group: {
+						members: list,
+						sentinels: [],
+						observer: { image: mongo }
+					}
+				};
+			},
+			close
+		};
+	} catch (error) {
+		await close();
+		throw error;
+	}
+};
+/** This count is the number of accepted writes that a run waits for before its fault: the stream is steady then. */
+const STEADY = 20;
+/** This time is how long a run continues after its outcome, so that the record shows what came next. */
+const TAIL_MS$1 = 2e3;
+/** This function gives the arguments of the client of a replica set cell: each member, and the stream of writes. */
+const replicaSetArgs = (cell, members) => {
+	const set = cell.replicaSet;
+	const hosts = set.members.map((name) => `${members[name].ip}:${set.port}`);
+	return [
+		members[set.members[0]].ip,
+		String(set.port),
+		"--mode",
+		"stream",
+		"--hosts",
+		hosts.join(","),
+		"--set",
+		set.name,
+		"--interval-ms",
+		String(cell.probe.interval === void 0 ? 50 : valueIn(cell.probe.interval, "ms")),
+		"--max-s",
+		String(Math.ceil(deadlineOf(cell) / 1e3 + (cell.hold === void 0 ? 0 : valueIn(cell.hold, "s")) + 120)),
+		"--w",
+		String(cell.writeConcern ?? "majority"),
+		...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
+	];
+};
+/** This function waits until a condition on the lines of the client holds, or for `within` ms. It gives the first line that matches. */
+const waitFor = (client, match, within) => new Promise((resolve) => {
+	const started = Date.now();
+	const look = () => {
+		const found = client.events.find(match);
+		if (found !== void 0 || Date.now() - started > within) resolve(found);
+		else setTimeout(look, 20);
+	};
+	look();
+});
+/** This function gives the count of lost records in the latest clock line of the prober. */
+const droppedSoFar$1 = (records) => {
+	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
+	return 0;
+};
+/**
+* This function does one run of a replica set cell:
+* - The set converges, and its writes go. The client finds the primary with its monitors, and its
+*   stream of writes gets steady.
+* - The fault occurs on its member, as the primary of this run makes the roles. A fault with a
+*   hold lifts at its instant in the VM. A stall lifts at its length after the first write into
+*   it. A step-down is replSetStepDown to the primary.
+* - The run waits for its outcome: the first accepted write of a write sent after the commit of
+*   the fault. It is on a new primary, or on any primary. Then the run waits for the end of the
+*   hold, thus the record shows what the set did in the whole hold.
+* - Each fault goes, the client stops, and the set converges again. The writes that it then holds
+*   tell which writes it kept. The log lines of the members go onto the monotonic clock.
+*/
+const replicaSetOnce = async (lab, cell, environment, index) => {
+	const set = cell.replicaSet;
+	const primary = await lab.converge();
+	await lab.flush(primary);
+	await lab.unfreeze(primary);
+	await lab.forget();
+	const from = lab.kernel.length;
+	const logFrom = lab.log.length;
+	const lostBefore = droppedSoFar$1(lab.kernel);
+	const client = await lab.client(replicaSetArgs(cell, lab.members));
+	const lands = cell.prediction.group?.lands ?? "new";
+	const place = cell.faultAt ?? { role: "primary" };
+	let fault = null;
+	let heal = null;
+	let at = null;
+	const undo = [];
+	try {
+		await client.until(["ready"], 6e4);
+		if (await waitFor(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY, 3e4) === void 0) throw new Error(`the client did not write ${STEADY} times in 30 s`);
+		let healing = null;
+		switch (cell.fault) {
+			case "blackhole":
+			case "partition":
+			case "rst": {
+				const kind = cell.fault === "rst" ? "rst" : "blackhole";
+				const cuts = place.cuts ?? (cell.fault === "partition" ? "group" : "all");
+				const part = lab.partOf(place.role, primary);
+				at = part;
+				fault = await lab.fault(kind, part, cuts);
+				undo.push(() => lab.lift(kind, part, cuts));
+				if (cell.stall !== void 0) {
+					const t1 = fault.t1;
+					const sent = await waitFor(client, (event) => event.ev === "sent" && event.t > t1, 5e3);
+					healing = lab.lift(kind, part, cuts, (sent?.t ?? t1) + valueIn(cell.stall, "s") * 1e9);
+				} else if (cell.hold !== void 0) healing = lab.lift(kind, part, cuts, fault.t0 + valueIn(cell.hold, "s") * 1e9);
+				break;
+			}
+			case "freeze-process": {
+				const part = lab.partOf(place.role, primary);
+				at = part;
+				fault = await lab.signal(part, "STOP");
+				undo.push(() => lab.signal(part, "CONT"));
+				if (cell.hold !== void 0) healing = lab.signal(part, "CONT", fault.t0 + valueIn(cell.hold, "s") * 1e9);
+				break;
+			}
+			case "fin": {
+				const part = lab.partOf(place.role, primary);
+				at = part;
+				fault = await lab.signal(part, "KILL");
+				undo.push(() => lab.restart(part));
+				break;
+			}
+			case "step-down":
+				at = primary;
+				fault = await lab.stepDown(primary);
+				break;
+			default: throw new Error(`a replica set cell has no fault ${cell.fault}`);
+		}
+		const healed = healing === null ? Promise.resolve(null) : healing.then((stamp) => {
+			heal = stamp;
+			undo.length = 0;
+			return stamp;
+		});
+		healed.catch(() => void 0);
+		const t1 = fault.t1;
+		const old = `${lab.members[primary].ip}:${set.port}`;
+		const sentAt = (n) => client.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
+		if (await waitFor(client, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > t1 && (lands === "same" || typeof event["at"] === "string" && event["at"] !== old), deadlineOf(cell)) !== void 0) await sleep$1(TAIL_MS$1);
+		if (healing !== null) {
+			await healed.catch(() => void 0);
+			await sleep$1(TAIL_MS$1);
+		}
+	} finally {
+		for (const step of undo.reverse()) await step().catch(() => void 0);
+		await client.stop();
+	}
+	const settled = await lab.settle();
+	const lost = settled === null ? null : settled - lostBefore;
+	const after = await lab.converge().catch(() => null);
+	const holds = after === null ? null : await lab.holds(after);
+	return {
+		environment,
+		index,
+		events: [...client.events],
+		fault,
+		...heal === null ? {} : { heal },
+		kernel: lab.kernel.slice(from),
+		netns: lab.netns.client,
+		targetNetns: lab.members[primary].netns,
+		lost,
+		group: {
+			members: lab.members,
+			sentinels: {},
+			primary,
+			at,
+			timeline: onMonotonic(lab.log.slice(logFrom), clockOffsetOf(client.events)),
+			after,
+			holds
+		}
+	};
+};
+//#endregion
+//#region src/run.ts
+/**
+* This function measures and judges the raw records of a cell, with the code of the harness at
+* this time. `runCell` uses it during a cell. A holder of the raw records of a cell can use it
+* again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.
+*/
+const judge = (cell, runs) => {
+	const measured = runs.filter((run) => run.warmup !== true).map((run) => measure(cell, run));
+	return {
+		measured,
+		verdict: verdictOf(cell.prediction, measured)
+	};
+};
+/** This function gives the count of lost records in the latest clock line of the prober, since it began. */
+const droppedSoFar = (records) => {
+	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
+	return 0;
+};
+/** This function gives the argv of the client for a cell, with a deadline far after all that it waits for. */
+const probeArgs = (cell, lab) => {
+	const { prediction, probe } = cell;
+	const longest = (prediction.detects === true && prediction.driver !== void 0 ? longestOf(cell) : prediction.detects ? valueIn(prediction.high, "s") : valueIn(prediction.window, "s")) + (cell.idleBefore === void 0 ? 0 : valueIn(cell.idleBefore, "s"));
+	const args = [
+		lab.targetIp,
+		String(lab.targetPort),
+		"--mode",
+		probe.mode
+	];
+	if (probe.http !== void 0) {
+		const { http } = probe;
+		const gap = http.gap === void 0 ? 0 : valueIn(http.gap, "s");
+		args.push("--method", http.method);
+		if (http.gap !== void 0) args.push("--gap-ms", String(valueIn(http.gap, "ms")));
+		if (http.hold !== void 0) args.push("--hold-ms", String(valueIn(http.hold, "ms")));
+		if (http.bytes !== void 0) args.push("--bytes", String(http.bytes));
+		for (const [name, value] of Object.entries(http.options ?? {})) args.push(`--${name}`, String(value));
+		args.push("--linger-ms", String(LINGER_MS));
+		args.push("--max-s", String(Math.ceil(longest * 2 + gap + 10)));
+		return args;
+	}
+	if (probe.request !== void 0) args.push("--request", probe.request);
+	if (probe.count !== void 0) args.push("--count", String(probe.count));
+	if (probe.interval !== void 0) args.push("--interval-ms", String(valueIn(probe.interval, "ms")));
+	if (probe.driver?.name === "mongodb") {
+		args.push("--set", probe.driver.set);
+		if (probe.driver.op !== void 0) args.push("--op", probe.driver.op);
+		if (cell.writeConcern !== void 0) args.push("--w", String(cell.writeConcern));
+		if (cell.before === true) args.push("--before");
+		if (cell.driverOptions !== void 0) args.push("--options", JSON.stringify(cell.driverOptions));
+		args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
+		return args;
+	}
+	if (probe.driver !== void 0) {
+		args.push("--keepalive-ms", String(valueIn(probe.driver.keepAlive, "ms")));
+		args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
+		return args;
+	}
+	if (probe.keepalive) args.push("--keepalive");
+	if (probe.userTimeout !== void 0) args.push("--user-timeout-ms", String(valueIn(probe.userTimeout, "ms")));
+	args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
+	return args;
+};
+/**
+* This function waits for an error on the socket of the observer, or for its end of file
+* (CLOSE_WAIT, state 8). The prober records both. A cell whose observer is the server, or a proxy
+* of a topology, waits for that. The promise resolves after `within` ms at the latest.
+*/
+const observerError = (cell, partial, within) => new Promise((resolve) => {
+	const started = Date.now();
+	const look = () => {
+		const run = partial();
+		const at = socketOf(cell, run);
+		if (at !== void 0 && run.kernel.some((record) => record.netns === at.netns && record.cookie === at.cookie && (record.ev === "error" && (record.err ?? 0) !== 0 || record.ev === "state" && record.new === 8)) || Date.now() - started > within) resolve();
+		else setTimeout(look, 50);
+	};
+	look();
+});
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/**
+* This time is the pause between the client's first reply and the fault. It is more than Linux's
+* TCP_DELACK_MAX of 200 ms. Thus the delayed ACK of the reply is out before the fault.
+*/
+const QUIET_MS = 250;
+/**
+* This time is how long an HTTP client waits after its second request, before it exits. The exit
+* closes its sockets, thus that close comes after the window of the cell.
+*/
+const LINGER_MS = 1e3;
+/** These are the lines that end the workload of an HTTP client: the end of its second request, its deadline, or its failure. */
+const HTTP_DONE = [
+	"done",
+	"deadline",
+	"fail"
+];
+/**
+* This function waits for the request that an in-flight client sends on the signal: its second
+* send, after the PING that made it ready. Its stamp is when the request first met the fault.
 */
 const requestSent = (probe, within) => new Promise((resolve, reject) => {
 	const started = Date.now();
@@ -2664,6 +3937,69 @@ const requestSent = (probe, within) => new Promise((resolve, reject) => {
 	};
 	look();
 });
+/** This time is how long a run of a driver cell continues after its outcome, so that the record shows what came next. */
+const TAIL_MS = 1e3;
+/** This function waits for the outcome of a run of a driver cell, or for `within` ms. The outcome is the end of its band and of each later band. */
+const driverOutcome = async (cell, current, within) => {
+	const started = Date.now();
+	while (Date.now() - started < within) {
+		if (settled(cell, current())) return;
+		await sleep(250);
+	}
+};
+/** This function gives the longest wait of a run of a driver cell for its outcome, in ms. */
+const driverWithin = (cell) => (longestOf(cell) * 1.2 + 10) * 1e3;
+/**
+* This function does one run of a cell whose fault holds before the client starts (`Cell.before`).
+* The fault is a network fault at its place. The client then starts into it, and the run waits for
+* each band of the driver. Then the fault lifts.
+*/
+const beforeOnce = async (lab, cell, environment, index) => {
+	if (cell.fault !== "rst" && cell.fault !== "blackhole" && cell.fault !== "unreachable") throw new Error(`${cell.name}: a fault before the client starts is a network fault, not ${cell.fault}`);
+	await lab.forget();
+	const from = lab.kernel.length;
+	const middleboxFrom = lab.middlebox.length;
+	const lostBefore = droppedSoFar(lab.kernel);
+	const faultAt = cell.faultAt ?? "target";
+	const kind = cell.fault === "unreachable" ? "reject" : cell.fault;
+	const options = cell.fault === "unreachable" ? { code: cell.icmp ?? "host" } : {};
+	const fault = await lab.fault(kind, faultAt, options);
+	let probe;
+	try {
+		probe = await lab.probe(probeArgs(cell, lab));
+		const started = probe;
+		const current = () => ({
+			environment,
+			index,
+			events: started.events,
+			fault,
+			kernel: lab.kernel.slice(from),
+			netns: lab.netns.client,
+			targetNetns: lab.netns.target,
+			lost: 0
+		});
+		await driverOutcome(cell, current, driverWithin(cell));
+		await sleep(TAIL_MS);
+	} finally {
+		await lab.lift(kind, faultAt, options);
+		await probe?.stop();
+	}
+	const settled = await lab.settle();
+	return {
+		environment,
+		index,
+		events: [...probe?.events ?? []],
+		fault,
+		kernel: lab.kernel.slice(from),
+		netns: lab.netns.client,
+		targetNetns: lab.netns.target,
+		lost: settled === null ? null : settled - lostBefore,
+		...lab.topology === null ? {} : {
+			topology: lab.topology,
+			middlebox: lab.middlebox.slice(middleboxFrom)
+		}
+	};
+};
 /** This function gives the netem specification of a cell: its latency, then the netem of its fault when it has one. */
 const netemSpec = (cell, fault) => {
 	const parts = [...cell.latency === void 0 ? [] : [`delay ${valueIn(cell.latency, "ms")}ms`], ...fault === void 0 ? [] : [fault]];
@@ -2683,9 +4019,11 @@ const netemSpec = (cell, fault) => {
 * A dry run does all of this but the fault: the warm-up of a cell whose fault ends its environment.
 */
 const once = async (lab, cell, environment, index, dry = false) => {
+	if (cell.before === true) return beforeOnce(lab, cell, environment, index);
 	await lab.forget();
 	const from = lab.kernel.length;
 	const middleboxFrom = lab.middlebox.length;
+	const serverFrom = lab.server.length;
 	const lostBefore = droppedSoFar(lab.kernel);
 	const probe = await lab.probe(probeArgs(cell, lab));
 	const server = cell.observer === "server";
@@ -2715,6 +4053,8 @@ const once = async (lab, cell, environment, index, dry = false) => {
 	});
 	let fault = null;
 	let heal = null;
+	const signalled = !server && cell.target.program !== void 0;
+	let thawing = null;
 	try {
 		await probe.until(["ready"], 15e3);
 		if (dry) {
@@ -2741,6 +4081,17 @@ const once = async (lab, cell, environment, index, dry = false) => {
 			const apply = async () => {
 				switch (cell.fault) {
 					case "freeze-process":
+						if (signalled) {
+							const stamp = await lab.signal("STOP");
+							if (cell.hold !== void 0) {
+								thawing = lab.signal("CONT", stamp.t0 + valueIn(cell.hold, "s") * 1e9).then((thawed) => {
+									heal = thawed;
+									return thawed;
+								});
+								thawing.catch(() => void 0);
+							}
+							return stamp;
+						}
 						await peer.freeze();
 						return null;
 					case "fin":
@@ -2748,11 +4099,12 @@ const once = async (lab, cell, environment, index, dry = false) => {
 						return null;
 					case "unreachable": return lab.fault("reject", faultAt, { code: cell.icmp ?? "host" });
 					case "simplex": return lab.fault("blackhole", faultAt, options);
-					case "mtu-blackhole": return lab.fault("mtu", "client", { above: cell.mtu ?? 1200 });
+					case "mtu-blackhole": return lab.fault("mtu", cell.faultAt ?? "client", { above: cell.mtu ?? 1200 });
 					case "netem": return netemOn(cell.netem);
 					case "rst": return lab.fault("rst", faultAt);
 					case "middlebox-forget":
 					case "proxy-idle":
+					case "app-stall":
 					case "none": return null;
 					case "toxic":
 						if (cell.toxic === void 0) throw new Error(`${cell.name} names no toxic`);
@@ -2765,15 +4117,22 @@ const once = async (lab, cell, environment, index, dry = false) => {
 			};
 			const lift = async () => {
 				switch (cell.fault) {
-					case "freeze-process": return peer.thaw();
+					case "freeze-process":
+						if (signalled) {
+							if (thawing !== null) await thawing.catch(() => lab.signal("CONT"));
+							else await lab.signal("CONT");
+							return;
+						}
+						return peer.thaw();
 					case "fin":
 					case "middlebox-forget":
 					case "proxy-idle":
 					case "server-close":
+					case "app-stall":
 					case "none": return;
 					case "unreachable": return lab.lift("reject", faultAt, { code: cell.icmp ?? "host" });
 					case "simplex": return lab.lift("blackhole", faultAt, options);
-					case "mtu-blackhole": return lab.lift("mtu", "client", { above: cell.mtu ?? 1200 });
+					case "mtu-blackhole": return lab.lift("mtu", cell.faultAt ?? "client", { above: cell.mtu ?? 1200 });
 					case "netem":
 						await netemOn(void 0);
 						return;
@@ -2787,9 +4146,21 @@ const once = async (lab, cell, environment, index, dry = false) => {
 			fault = await apply();
 			try {
 				if (cell.idleBefore !== void 0 && cell.probe.mode !== "idle") await sleep(valueIn(cell.idleBefore, "ms"));
-				if (cell.probe.mode !== "idle" && !replyInFlight) await go();
+				if ((cell.probe.mode === "inflight" || cell.probe.mode === "stream") && !replyInFlight) await go();
 				const { prediction } = cell;
-				if (cell.fault === "stall") {
+				const http = cell.probe.http !== void 0;
+				if (prediction.detects !== null && prediction.driver !== void 0) {
+					if (prediction.detects) await driverOutcome(cell, () => ({
+						...partial(),
+						fault
+					}), driverWithin(cell));
+					else await sleep(valueIn(prediction.window, "s") * 1e3);
+					await sleep(TAIL_MS);
+				} else if (cell.probe.mode === "gap") {
+					const gap = cell.probe.http?.gap === void 0 ? 0 : valueIn(cell.probe.http.gap, "s");
+					const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
+					await probe.until(HTTP_DONE, (gap + window) * 2e3 + 1e4).catch(() => void 0);
+				} else if (cell.fault === "stall") {
 					if (cell.stall === void 0) throw new Error(`${cell.name} says no stall's length`);
 					const sent = await requestSent(probe, 5e3);
 					heal = await lab.heal(faultAt, sent.t + valueIn(cell.stall, "s") * 1e9);
@@ -2810,10 +4181,12 @@ const once = async (lab, cell, environment, index, dry = false) => {
 				} else if (prediction.detects) {
 					const within = valueIn(prediction.high, "s") * 2e3 + 1e4;
 					if (cell.observer !== void 0 && cell.observer !== "client") await observerError(cell, partial, within);
+					else if (http && cell.probe.mode !== "idle") await probe.until(HTTP_DONE, within).catch(() => void 0);
 					else await probe.until([
 						"error",
 						"eof",
-						"deadline"
+						"deadline",
+						...http ? ["closed"] : []
 					], within).catch(() => void 0);
 				} else await sleep(valueIn(prediction.window, "s") * 1e3);
 			} finally {
@@ -2832,13 +4205,14 @@ const once = async (lab, cell, environment, index, dry = false) => {
 		index,
 		events: [...probe.events],
 		fault,
-		...cell.fault === "stall" ? { heal } : {},
+		...cell.fault === "stall" || thawing !== null ? { heal } : {},
 		kernel: lab.kernel.slice(from),
 		netns: lab.netns.client,
 		targetNetns: lab.netns.target,
 		lost,
 		...placed,
-		...lab.topology === null ? {} : { middlebox: lab.middlebox.slice(middleboxFrom) }
+		...lab.topology === null ? {} : { middlebox: lab.middlebox.slice(middleboxFrom) },
+		...cell.target.lines === true ? { server: lab.server.slice(serverFrom) } : {}
 	};
 };
 /** This function divides a count of runs across environments, as equally as possible. */
@@ -2855,6 +4229,7 @@ const line = (cell, run, read) => `${cell.name}: env ${run.environment} ${run.wa
 */
 const inEnvironment = async (cell, environment, count, say) => {
 	if (cell.group !== void 0) return inGroupEnvironment(cell, environment, count, say);
+	if (cell.replicaSet !== void 0) return inReplicaSetEnvironment(cell, environment, count, say);
 	const lab = await openLab(cell.target, cell.client ?? cell.probe.driver?.name ?? "tcp", cell.topology);
 	try {
 		const settings = await lab.configure(cell.targetSettings ?? {});
@@ -2919,6 +4294,34 @@ const inGroupEnvironment = async (cell, environment, count, say) => {
 	}
 };
 /**
+* This function does the part of the runs of a replica set cell in one environment. The warm-up
+* is a whole run with its fault, as for a group under Sentinels. The set converges before each run.
+*/
+const inReplicaSetEnvironment = async (cell, environment, count, say) => {
+	const lab = await openReplicaSetLab(cell.replicaSet);
+	try {
+		const manifest = await lab.manifest(await lab.sysctl(cell.sysctls));
+		const warmup = {
+			...await replicaSetOnce(lab, cell, environment, -1),
+			warmup: true
+		};
+		say(line(cell, warmup, measure(cell, warmup)));
+		const runs = [];
+		for (let index = 0; index < count; index += 1) {
+			const run = await replicaSetOnce(lab, cell, environment, index);
+			runs.push(run);
+			say(line(cell, run, measure(cell, run)));
+		}
+		return {
+			manifest,
+			runs,
+			warmup
+		};
+	} finally {
+		await lab.close();
+	}
+};
+/**
 * This function does the runs of a cell. They go across new environments, and each environment records
 * its manifest. Then the verdict judges the runs. A median across the edge of the band extends
 * the cell from 11 runs to 22, and then to 44, each extension in new environments. With a
@@ -2967,6 +4370,6 @@ const runCell = async (cell, say = () => {}) => {
 	}
 };
 //#endregion
-export { CONVERGE_MS, LISTING, NOISE, PUMP, RELATIVE, STAGES, STEPS, crossingsOf, deadlineOf, deviceOf, endpointAfter, errorsOf, fieldsOf, groupOnce, groupVerdictOf, harnessImage, infoOf, isHopPlace, judge, lagsOf, measure, measureGroup, medianInterval, memberConfig, nodeClientImage, openGroupLab, openLab, peersOf, quantileOf, reorderingOf, routerImage, routesOf, runCell, sentinelConfig, socketOf, spread, spreadOf, stagesOf, subscriptionsOf, templated, toxicBody, verdictOf, viewOf, writesOf };
+export { CONVERGE_MS, KEPT, LINGER_MS, LISTING, MEMBER_MARKS, NOISE, PUMP, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, STAGES, STATUS_SCRIPT, STEPS, STEP_DOWN, clockOffsetOf, crossingsOf, cutOf, deadlineOf, deviceOf, endpointAfter, errorsOf, fieldOf, fieldsOf, goHttpImage, groupOnce, groupVerdictOf, harnessImage, infoOf, initiateScript, isHopPlace, judge, lagsOf, matches, measure, measureDriver, measureGroup, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, nodeClientImage, nodeHttpImage, onMonotonic, openGroupLab, openLab, openReplicaSetLab, peersOf, quantileOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, routerImage, routesOf, runCell, sentinelConfig, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, statusViewOf, subscriptionsOf, templated, toxicBody, verdictOf, viewOf, writesOf };
 
 //# sourceMappingURL=index.js.map