@mark1russell7/harness 0.2.0 → 0.3.0

package/build/index.js

@@ -5,6 +5,192 @@ import { createRequire } from "node:module";
 import { tmpdir } from "node:os";
 import { dirname, join, relative } from "node:path";
 import { fileURLToPath } from "node:url";
+//#region src/verdict.ts
+/**
+* This amount is the noise of the pipeline itself, δ_abs, as the calibration measured it. It is
+* the 99.9th percentile of the whole latency of a reset, at the concurrency cap or less. Its
+* minimum is one jiffy. On 2026-10-12 the worst of 60 resets at 1 and 4 environments took
+* 1.141 ms, thus the minimum sets it to 4 ms. The calibration occurs again after each change to
+* the harness, the kernel or Docker.
+*/
+const NOISE = quantity(4, "ms");
+/** This fraction, δ_rel, is how far the median of a cell can be outside the prediction and stay equivalent. */
+const RELATIVE = .05;
+/** These are the run counts of a cell: 11, then 22 and 44 while its median crosses the edge of the band. */
+const STEPS = [
+	11,
+	22,
+	44
+];
+const s = (value) => quantity(Math.round(value * 1e6) / 1e6, "s");
+const name = (run) => `env ${run.environment} run ${run.index}`;
+/** This function gives P(X ≤ j) for X ~ Binomial(n, 1/2), exactly. */
+const below = (n, j) => {
+	let term = 1;
+	let sum = 0;
+	for (let i = 0; i <= j; i += 1) {
+		sum += term;
+		term = term * (n - i) / (i + 1);
+	}
+	return sum / 2 ** n;
+};
+/**
+* This function gives the distribution-free interval of a median: [x_(k), x_(n+1−k)], for the
+* largest k whose two tails each hold 2.5 % or less. For eleven runs, it gives [x_(2), x_(10)],
+* at 98.8 %.
+*/
+const medianInterval = (sorted) => {
+	const n = sorted.length;
+	let k = 0;
+	while (k + 1 <= n / 2 && below(n, k) <= .025) k += 1;
+	return k === 0 ? [sorted[0], sorted[n - 1]] : [sorted[k - 1], sorted[n - k]];
+};
+const middle = (sorted) => {
+	const half = Math.floor(sorted.length / 2);
+	return sorted.length % 2 === 1 ? sorted[half] : (sorted[half - 1] + sorted[half]) / 2;
+};
+/**
+* This function gives the verdict of a cell, from two tests:
+* - Containment: each run lands in [T_lo − δ_abs, T_hi + δ_abs]. A run outside is a finding.
+* - Median equivalence: the interval of the median is in the band, made wider by δ_rel. An
+*   interval outside the band fails. An interval across its edge extends the cell to 22 runs, then
+*   44, and then the verdict is inconclusive.
+*
+* A silent prediction holds only if no run detected the fault in its window. After a stall, the
+* return of the reply gets the same test as a detection, by its interval, and a detection is a
+* finding. The distribution test (a DKW band) needs a predicted distribution from a simulator,
+* which does not exist at this time.
+*/
+const verdictOf = (prediction, runs) => {
+	const counted = runs.filter((run) => run.void === null);
+	const voided = runs.length - counted.length;
+	const outside = counted.filter((run) => run.finding !== null).map((run) => ({
+		run: name(run),
+		why: run.finding
+	}));
+	const empty = {
+		counted: counted.length,
+		voided,
+		median: null,
+		medianInterval: null
+	};
+	if (counted.length === 0) return {
+		...empty,
+		decision: "inconclusive",
+		band: null,
+		outside,
+		why: "no run counted"
+	};
+	if (prediction.detects === null) {
+		if (prediction.from !== void 0) {
+			const times = counted.flatMap((run) => run.elapsed === null ? [] : [valueIn(run.elapsed, "s")]).sort((a, b) => a - b);
+			const kinds = /* @__PURE__ */ new Map();
+			for (const run of counted) {
+				const kind = run.outcome.kind === "error" ? run.outcome.errno : run.outcome.kind;
+				kinds.set(kind, (kinds.get(kind) ?? 0) + 1);
+			}
+			const outcomes = [...kinds].map(([kind, n]) => `${n} ${kind}`).join(", ");
+			const timed = times.length === 0 ? "no time" : `median ${s(middle(times)).value} s from ${prediction.from}, range [${s(times[0]).value}, ${s(times.at(-1)).value}] s`;
+			return {
+				...empty,
+				decision: "measured",
+				band: null,
+				outside: [],
+				median: times.length === 0 ? null : s(middle(times)),
+				why: `${counted.length} runs measured: ${outcomes}; ${timed}. ${prediction.why}`
+			};
+		}
+		const samples = counted.flatMap((run) => (run.samples ?? []).map((sample) => valueIn(sample, "s"))).sort((a, b) => a - b);
+		const retransmissions = counted.reduce((sum, run) => sum + run.retransmissions, 0);
+		const ended = counted.filter((run) => run.outcome.kind === "error" || run.outcome.kind === "eof");
+		const why = samples.length === 0 ? `${counted.length} runs measured, with no reply: ${prediction.why}` : `${counted.length} runs, ${samples.length} replies: median ${s(middle(samples)).value} s, slowest ${s(samples.at(-1)).value} s, ${retransmissions} retransmissions, ${ended.length} connections ended. ${prediction.why}`;
+		return {
+			...empty,
+			decision: "measured",
+			band: null,
+			outside: [],
+			median: samples.length === 0 ? null : s(middle(samples)),
+			why
+		};
+	}
+	if (!prediction.detects && prediction.recovers === void 0 && prediction.replies === void 0) {
+		const decision = outside.length === 0 ? "holds" : "fails";
+		const why = decision === "holds" ? `nothing detected it in ${counted.length} runs of ${valueIn(prediction.window, "s")} s` : `${outside.length} of ${counted.length} runs detected it`;
+		return {
+			...empty,
+			decision,
+			band: null,
+			outside,
+			why
+		};
+	}
+	const timed = prediction.detects ? prediction : prediction.recovers ?? prediction.replies;
+	const noise = valueIn(NOISE, "s");
+	const low = valueIn(timed.low, "s");
+	const high = valueIn(timed.high, "s");
+	const band = {
+		low: s(low - noise),
+		high: s(high + noise)
+	};
+	const values = [];
+	for (const run of counted) {
+		if (run.elapsed === null) continue;
+		const value = valueIn(run.elapsed, "s");
+		values.push(value);
+		if (value < low - noise || value > high + noise) outside.push({
+			run: name(run),
+			why: `${value} s is outside [${band.low.value}, ${band.high.value}] s`
+		});
+	}
+	values.sort((a, b) => a - b);
+	if (values.length === 0) return {
+		...empty,
+		decision: "fails",
+		band,
+		outside,
+		why: "no run measured its time"
+	};
+	const [from, to] = medianInterval(values);
+	const median = s(middle(values));
+	const interval = {
+		low: s(from),
+		high: s(to)
+	};
+	const wide = {
+		low: low * .95,
+		high: high * 1.05
+	};
+	const inside = from >= wide.low && to <= wide.high;
+	const disjoint = to < wide.low || from > wide.high;
+	const off = new Set(outside.map((each) => each.run)).size;
+	const counts = `${counted.length - off} of ${counted.length} runs inside [${band.low.value}, ${band.high.value}] s`;
+	const centre = `median ${median.value} s, its interval [${interval.low.value}, ${interval.high.value}] s`;
+	const result = {
+		counted: counted.length,
+		voided,
+		band,
+		outside,
+		median,
+		medianInterval: interval
+	};
+	if (outside.length > 0 || disjoint) return {
+		...result,
+		decision: "fails",
+		why: `${counts}; ${centre}`
+	};
+	if (inside) return {
+		...result,
+		decision: "holds",
+		why: `${counts}; ${centre}`
+	};
+	const decision = counted.length < STEPS[2] ? "extend" : "inconclusive";
+	return {
+		...result,
+		decision,
+		why: `${counts}; ${centre} straddles the band`
+	};
+};
+//#endregion
 //#region src/observe.ts
 /** These are the errno numbers that the kernel reports to a socket, by the names that predictions use. */
 const ERRNO = {
@@ -24,14 +210,77 @@ const newState = (record) => record.new;
 const count = (event, field) => event === void 0 ? 0 : Number(event[field] ?? 0);
 /** This function gives the view of the client on the same interval: TCP_INFO, with a sample each 2 ms. */
 const socketVantage = (cell, events, ready, request) => {
-	if (!cell.prediction.detects || cell.prediction.from === "peer-closed") return null;
+	if (!cell.prediction.detects || cell.prediction.from === "peer-closed" || cell.prediction.from === "fault") return null;
 	const ended = events.find((event) => event.ev === "error" || event.ev === "eof");
 	const reference = cell.prediction.from === "last-received" ? ready : cell.prediction.from === "request-sent" ? request : events.find((event) => event.ev === "kernel" && event.t >= request.t && count(event, "retransmits") > 0);
 	return ended === void 0 || reference === void 0 ? null : between(reference.t, ended.t);
 };
+const ipOf = (address) => address.split(":")[0];
+/**
+* This function finds the socket that became ESTABLISHED in a run, in a namespace, whose peer
+* matches. In a topology, an address of a hop serves each run, thus only the socket that the
+* run opened is the socket of the run.
+*/
+const opened = (run, netns, peer) => {
+	const record = run.kernel.find((each) => each.netns === netns && each.ev === "state" && newState(each) === 1 && each.dst !== void 0 && peer(each.dst));
+	return record?.cookie === void 0 ? void 0 : {
+		netns,
+		cookie: record.cookie
+	};
+};
+/** This function finds the socket of the target for the connection of a run. In a topology, its peer is the last hop. */
+const targetSocket = (run, local) => {
+	if (run.topology === void 0) {
+		const cookie = run.kernel.find((record) => record.netns === run.targetNetns && record.dst === local)?.cookie;
+		return cookie === void 0 ? void 0 : {
+			netns: run.targetNetns,
+			cookie
+		};
+	}
+	const peer = run.topology.targetPeer;
+	return opened(run, run.targetNetns, (dst) => ipOf(dst) === peer);
+};
+/**
+* This function finds the socket of the observer of a cell in a run:
+* - the client socket, by the address that it connected from, in its namespace.
+* - the socket of the target for the same connection, in its namespace.
+* - a socket of a proxy of the topology, toward the client or toward the target.
+*/
+const socketOf = (cell, run) => {
+	const local = run.events.find((event) => event.ev === "connected")?.["local"];
+	const observer = cell.observer ?? "client";
+	if (observer === "client") {
+		const cookie = run.kernel.find((record) => record.netns === run.netns && record.src === local)?.cookie;
+		return cookie === void 0 ? void 0 : {
+			netns: run.netns,
+			cookie
+		};
+	}
+	if (observer === "server") return targetSocket(run, local);
+	const hop = run.topology?.hops[observer.hop];
+	if (hop === void 0) return void 0;
+	return observer.toward === "client" ? opened(run, hop.netns, (dst) => ipOf(dst) === hop.client) : opened(run, hop.netns, (dst) => dst === hop.target);
+};
+/**
+* This function finds the socket of the peer of the observer: its close is the reference of a FIN.
+* In a direct pair, it is the other end of the connection. In a topology, it is the socket of the
+* target, for the client.
+*/
+const peerOf = (cell, run) => {
+	const local = run.events.find((event) => event.ev === "connected")?.["local"];
+	const server = cell.observer === "server";
+	if (run.topology !== void 0) return server ? void 0 : targetSocket(run, local);
+	const netns = server ? run.netns : run.targetNetns;
+	const cookie = run.kernel.find((record) => record.netns === netns && (server ? record.src : record.dst) === local)?.cookie;
+	return cookie === void 0 ? void 0 : {
+		netns,
+		cookie
+	};
+};
 /**
 * This function reads one run against its cell, from the records of the kernel first.
-* - The socket of the run is the socket that the client connected from, in its namespace.
+* - The socket of the run is the socket of its observer. It is the socket that the client connected
+*   from, in its namespace, or the socket of the target or of a proxy for that connection.
 * - The fault must hold before the event that it must meet. That is before the request goes into
 *   it, and before the first keepalive probe. A live peer answers that probe, and the answer
 *   starts the idle time again. A run where the fault was late is void. A run whose records the
@@ -44,11 +293,16 @@ const measure = (cell, run) => {
 	const inflight = cell.probe.mode !== "idle";
 	const request = inflight ? events.filter((event) => event.ev === "sent")[1] : ready;
 	const server = cell.observer === "server";
-	const netns = server ? run.targetNetns : run.netns;
-	const cookie = kernel.find((record) => record.netns === netns && (server ? record.dst : record.src) === local)?.cookie;
+	const place = socketOf(cell, run);
+	const netns = place?.netns;
+	const cookie = place?.cookie;
 	const socket = kernel.filter((record) => record.netns === netns && record.cookie === cookie);
-	const peerNetns = server ? run.netns : run.targetNetns;
-	const peerCookie = kernel.find((record) => record.netns === peerNetns && (server ? record.src : record.dst) === local)?.cookie;
+	const peerPlace = peerOf(cell, run);
+	const peerNetns = peerPlace?.netns;
+	const peerCookie = peerPlace?.cookie;
+	const after = request === void 0 ? [] : events.filter((event) => event.ev === "data" && event.t > request.t);
+	const bytes = after.reduce((sum, event) => sum + Number(event["n"] ?? 0), 0);
+	const forgotten = (run.middlebox ?? []).find((line) => line.ev === "ct" && line["type"] === "DESTROY" && typeof line["orig"] === "string" && line["orig"].startsWith(`${String(local)}>`));
 	const keepalives = socket.filter((record) => record.ev === "probe" && record.what === "keepalive");
 	const retransmitted = socket.filter((record) => record.ev === "retrans");
 	/**
@@ -116,8 +370,9 @@ const measure = (cell, run) => {
 		else if (replyFirst && replied) voided = "the server's reply arrived before the fault held";
 		else if (keepalives[0] !== void 0 && keepalives[0].mono < run.fault.t1) voided = "a keepalive probe went out before the fault held";
 	}
+	const remote = cell.observer !== void 0 && cell.observer !== "client";
 	const reported = socket.find((record) => record.ev === "error" && (record.err ?? 0) !== 0);
-	const seen = server ? void 0 : events.find((event) => event.ev === "error" || event.ev === "eof");
+	const seen = remote ? void 0 : events.find((event) => event.ev === "error" || event.ev === "eof");
 	const closeWait = socket.find((record) => record.ev === "state" && newState(record) === 8);
 	const outcome = reported !== void 0 ? {
 		kind: "error",
@@ -129,6 +384,17 @@ const measure = (cell, run) => {
 	const ended = reported?.mono ?? closeWait?.mono ?? seen?.t;
 	const noticed = reported !== void 0 && seen !== void 0 ? between(reported.mono, seen.t) : null;
 	const { prediction } = cell;
+	const before = run.fault?.t1 ?? ready.t;
+	const referenceAt = (from) => {
+		if (from === "request-sent") return request.t;
+		if (from === "fault") return run.fault?.t0;
+		return (from === "last-received" ? socket.filter((each) => each.ev === "rx" && each.mono <= before).at(-1) : from === "first-retransmission" ? retransmitted.find((each) => each.why === "rto") : kernel.find((each) => each.netns === peerNetns && each.cookie === peerCookie && each.ev === "state" && oldState(each) === 1))?.mono;
+	};
+	const lastRx = kernel.filter((each) => each.netns === run.netns && each.src === local && each.ev === "rx" && each.mono <= before).at(-1)?.mono ?? ready.t;
+	const chain = run.topology === void 0 ? {} : {
+		bytes,
+		forgot: forgotten === void 0 ? null : between(lastRx, forgotten.t)
+	};
 	if (prediction.detects === null) {
 		const sends = events.filter((event) => event.ev === "sent").slice(1);
 		const samples = sends.flatMap((send, n) => {
@@ -136,10 +402,15 @@ const measure = (cell, run) => {
 			const reply = events.find((event) => event.ev === "data" && event.t > send.t && event.t < next);
 			return reply === void 0 ? [] : [between(send.t, reply.t)];
 		});
+		const from = prediction.from === void 0 ? void 0 : referenceAt(prediction.from);
+		const until = ended ?? after.at(-1)?.t;
 		return {
 			...base,
 			...nothing,
+			...chain,
 			outcome: ended !== void 0 ? outcome : { kind: "answered" },
+			...from === void 0 || until === void 0 ? {} : { elapsed: between(from, until) },
+			noticed: prediction.from === void 0 ? null : noticed,
 			steps: retransmitted.filter((record) => record.mono > request.t).map((record) => between(request.t, record.mono)),
 			samples,
 			void: voided
@@ -184,16 +455,54 @@ const measure = (cell, run) => {
 			finding: null
 		};
 	}
-	if (!prediction.detects) {
+	if (!prediction.detects && prediction.replies !== void 0) {
 		const window = valueIn(prediction.window, "s") * 1e9;
-		if (ended === void 0 || ended - request.t > window) return {
+		if (ended !== void 0 && ended - request.t <= window) return {
 			...base,
 			...nothing,
-			void: voided
+			...chain,
+			outcome,
+			elapsed: between(request.t, ended),
+			void: voided,
+			finding: `detected after ${between(request.t, ended).value} s, where the reply should come`
+		};
+		const answer = socket.find((record) => record.ev === "rx" && record.mono > request.t && (record.seg?.len.value ?? 0) > 0);
+		const reply = after[0];
+		if (answer === void 0) return {
+			...base,
+			...nothing,
+			...chain,
+			void: voided,
+			finding: "the reply never came"
 		};
+		return {
+			...base,
+			...chain,
+			outcome: { kind: "answered" },
+			elapsed: between(request.t, answer.mono),
+			socket: reply === void 0 ? null : between(request.t, reply.t),
+			noticed: reply === void 0 ? null : between(answer.mono, reply.t),
+			steps: [],
+			void: voided,
+			finding: null
+		};
+	}
+	if (!prediction.detects) {
+		const window = valueIn(prediction.window, "s") * 1e9;
+		if (ended === void 0 || ended - request.t > window) {
+			const forgot = forgotten !== void 0 && forgotten.t - request.t <= window ? `a router forgot the flow ${between(lastRx, forgotten.t).value} s after the last segment, where it should keep it` : null;
+			return {
+				...base,
+				...nothing,
+				...chain,
+				void: voided,
+				finding: forgot
+			};
+		}
 		return {
 			...base,
 			...nothing,
+			...chain,
 			outcome,
 			elapsed: between(request.t, ended),
 			void: voided,
@@ -203,15 +512,15 @@ const measure = (cell, run) => {
 	if (ended === void 0) return {
 		...base,
 		...nothing,
+		...chain,
 		void: voided,
 		finding: `nothing detected it: ${prediction.timer} never fired`
 	};
-	const before = run.fault?.t1 ?? ready.t;
-	const kernelReference = prediction.from === "last-received" ? socket.filter((record) => record.ev === "rx" && record.mono <= before).at(-1) : prediction.from === "first-retransmission" ? retransmitted.find((record) => record.why === "rto") : prediction.from === "peer-closed" ? kernel.find((record) => record.netns === peerNetns && record.cookie === peerCookie && record.ev === "state" && oldState(record) === 1) : void 0;
-	const reference = prediction.from === "request-sent" ? request.t : kernelReference?.mono;
+	const reference = referenceAt(prediction.from);
 	if (reference === void 0) return {
 		...base,
 		...nothing,
+		...chain,
 		outcome,
 		noticed,
 		void: voided,
@@ -219,12 +528,18 @@ const measure = (cell, run) => {
 	};
 	const steps = (prediction.from === "last-received" ? keepalives : prediction.from === "first-retransmission" ? retransmitted.filter((record) => record.why === "rto" && record.mono > reference) : []).map((record) => between(reference, record.mono));
 	const finding = (prediction.errno === "EOF" ? outcome.kind === "eof" : outcome.kind === "error" && outcome.errno === prediction.errno) ? null : `it ended with ${outcome.kind === "error" ? outcome.errno : "end of file"}, not ${prediction.errno}`;
-	if (prediction.from !== "request-sent" && prediction.from !== "peer-closed") voided ??= delayed(reference, ended, prediction.from === "last-received" ? "keepalive" : "retransmit");
+	if (prediction.from === "last-received" || prediction.from === "first-retransmission") voided ??= delayed(reference, ended, prediction.from === "last-received" ? "keepalive" : "retransmit");
+	const lag = (valueIn(prediction.high, "s") - valueIn(prediction.low, "s")) * 1e9;
+	const slow = prediction.timer.startsWith("expire_") ? (run.middlebox ?? []).find((line) => line.ev === "pump-slow" && line.t <= ended && line.t + Number(line["took"]) * 1e9 >= ended - lag) : void 0;
+	if (slow !== void 0) voided ??= `a listing of the router took ${Math.round(Number(slow["took"]) * 1e4) / 10} ms, longer than the bound of its prediction`;
+	const commit = run.fault === null ? 0 : (run.fault.t1 - run.fault.t0) / 1e6;
+	if (prediction.from === "fault" && commit > valueIn(NOISE, "ms")) voided ??= `the commit of the fault took ${Math.round(commit * 10) / 10} ms, longer than the noise of the pipeline: its start is not known`;
 	return {
 		...base,
+		...chain,
 		outcome,
 		elapsed: between(reference, ended),
-		socket: server ? null : socketVantage(cell, events, ready, request),
+		socket: remote ? null : socketVantage(cell, events, ready, request),
 		noticed,
 		steps,
 		void: voided,
@@ -232,173 +547,6 @@ const measure = (cell, run) => {
 	};
 };
 //#endregion
-//#region src/verdict.ts
-/**
-* This amount is the noise of the pipeline itself, δ_abs, as the calibration measured it. It is
-* the 99.9th percentile of the whole latency of a reset, at the concurrency cap or less. Its
-* minimum is one jiffy. On 2026-10-12 the worst of 60 resets at 1 and 4 environments took
-* 1.141 ms, thus the minimum sets it to 4 ms. The calibration occurs again after each change to
-* the harness, the kernel or Docker.
-*/
-const NOISE = quantity(4, "ms");
-/** This fraction, δ_rel, is how far the median of a cell can be outside the prediction and stay equivalent. */
-const RELATIVE = .05;
-/** These are the run counts of a cell: 11, then 22 and 44 while its median crosses the edge of the band. */
-const STEPS = [
-	11,
-	22,
-	44
-];
-const s = (value) => quantity(Math.round(value * 1e6) / 1e6, "s");
-const name = (run) => `env ${run.environment} run ${run.index}`;
-/** This function gives P(X ≤ j) for X ~ Binomial(n, 1/2), exactly. */
-const below = (n, j) => {
-	let term = 1;
-	let sum = 0;
-	for (let i = 0; i <= j; i += 1) {
-		sum += term;
-		term = term * (n - i) / (i + 1);
-	}
-	return sum / 2 ** n;
-};
-/**
-* This function gives the distribution-free interval of a median: [x_(k), x_(n+1−k)], for the
-* largest k whose two tails each hold 2.5 % or less. For eleven runs, it gives [x_(2), x_(10)],
-* at 98.8 %.
-*/
-const medianInterval = (sorted) => {
-	const n = sorted.length;
-	let k = 0;
-	while (k + 1 <= n / 2 && below(n, k) <= .025) k += 1;
-	return k === 0 ? [sorted[0], sorted[n - 1]] : [sorted[k - 1], sorted[n - k]];
-};
-const middle = (sorted) => {
-	const half = Math.floor(sorted.length / 2);
-	return sorted.length % 2 === 1 ? sorted[half] : (sorted[half - 1] + sorted[half]) / 2;
-};
-/**
-* This function gives the verdict of a cell, from two tests:
-* - Containment: each run lands in [T_lo − δ_abs, T_hi + δ_abs]. A run outside is a finding.
-* - Median equivalence: the interval of the median is in the band, made wider by δ_rel. An
-*   interval outside the band fails. An interval across its edge extends the cell to 22 runs, then
-*   44, and then the verdict is inconclusive.
-*
-* A silent prediction holds only if no run detected the fault in its window. After a stall, the
-* return of the reply gets the same test as a detection, by its interval, and a detection is a
-* finding. The distribution test (a DKW band) needs a predicted distribution from a simulator,
-* which does not exist at this time.
-*/
-const verdictOf = (prediction, runs) => {
-	const counted = runs.filter((run) => run.void === null);
-	const voided = runs.length - counted.length;
-	const outside = counted.filter((run) => run.finding !== null).map((run) => ({
-		run: name(run),
-		why: run.finding
-	}));
-	const empty = {
-		counted: counted.length,
-		voided,
-		median: null,
-		medianInterval: null
-	};
-	if (counted.length === 0) return {
-		...empty,
-		decision: "inconclusive",
-		band: null,
-		outside,
-		why: "no run counted"
-	};
-	if (prediction.detects === null) {
-		const samples = counted.flatMap((run) => (run.samples ?? []).map((sample) => valueIn(sample, "s"))).sort((a, b) => a - b);
-		const retransmissions = counted.reduce((sum, run) => sum + run.retransmissions, 0);
-		const ended = counted.filter((run) => run.outcome.kind === "error" || run.outcome.kind === "eof");
-		const why = samples.length === 0 ? `${counted.length} runs measured, with no reply: ${prediction.why}` : `${counted.length} runs, ${samples.length} replies: median ${s(middle(samples)).value} s, slowest ${s(samples.at(-1)).value} s, ${retransmissions} retransmissions, ${ended.length} connections ended. ${prediction.why}`;
-		return {
-			...empty,
-			decision: "measured",
-			band: null,
-			outside: [],
-			median: samples.length === 0 ? null : s(middle(samples)),
-			why
-		};
-	}
-	if (!prediction.detects && prediction.recovers === void 0) {
-		const decision = outside.length === 0 ? "holds" : "fails";
-		const why = decision === "holds" ? `nothing detected it in ${counted.length} runs of ${valueIn(prediction.window, "s")} s` : `${outside.length} of ${counted.length} runs detected it`;
-		return {
-			...empty,
-			decision,
-			band: null,
-			outside,
-			why
-		};
-	}
-	const timed = prediction.detects ? prediction : prediction.recovers;
-	const noise = valueIn(NOISE, "s");
-	const low = valueIn(timed.low, "s");
-	const high = valueIn(timed.high, "s");
-	const band = {
-		low: s(low - noise),
-		high: s(high + noise)
-	};
-	const values = [];
-	for (const run of counted) {
-		if (run.elapsed === null) continue;
-		const value = valueIn(run.elapsed, "s");
-		values.push(value);
-		if (value < low - noise || value > high + noise) outside.push({
-			run: name(run),
-			why: `${value} s is outside [${band.low.value}, ${band.high.value}] s`
-		});
-	}
-	values.sort((a, b) => a - b);
-	if (values.length === 0) return {
-		...empty,
-		decision: "fails",
-		band,
-		outside,
-		why: "no run measured its time"
-	};
-	const [from, to] = medianInterval(values);
-	const median = s(middle(values));
-	const interval = {
-		low: s(from),
-		high: s(to)
-	};
-	const wide = {
-		low: low * .95,
-		high: high * 1.05
-	};
-	const inside = from >= wide.low && to <= wide.high;
-	const disjoint = to < wide.low || from > wide.high;
-	const counts = `${counted.length - outside.length} of ${counted.length} runs inside [${band.low.value}, ${band.high.value}] s`;
-	const centre = `median ${median.value} s, its interval [${interval.low.value}, ${interval.high.value}] s`;
-	const result = {
-		counted: counted.length,
-		voided,
-		band,
-		outside,
-		median,
-		medianInterval: interval
-	};
-	if (outside.length > 0 || disjoint) return {
-		...result,
-		decision: "fails",
-		why: `${counts}; ${centre}`
-	};
-	if (inside) return {
-		...result,
-		decision: "holds",
-		why: `${counts}; ${centre}`
-	};
-	const decision = counted.length < STEPS[2] ? "extend" : "inconclusive";
-	return {
-		...result,
-		decision,
-		why: `${counts}; ${centre} straddles the band`
-	};
-};
-//#endregion
 //#region src/lab.ts
 /**
 * These are the roots of the two packages that the images read: this package, and the prober
@@ -437,6 +585,12 @@ const NODE = {
 	dockerfile: "harness/image/node/Dockerfile",
 	from: ["harness/image/node"]
 };
+/** This image is the router of a middlebox chain: NAT and conntrack, with a form for a forgotten flow. */
+const ROUTER = {
+	name: "loom-harness-router",
+	dockerfile: "harness/image/router/Dockerfile",
+	from: ["harness/image/router"]
+};
 /** This function gives each file under a path, in a stable sequence, but not the output of a local build. */
 const sources = (path) => {
 	if (statSync(path).isFile()) return [path];
@@ -495,6 +649,26 @@ const build = (image) => {
 const harnessImage = () => build(HARNESS);
 /** This function gives the image of the Node client. */
 const nodeClientImage = () => build(NODE);
+/** This function gives the image of the router. */
+const routerImage = () => build(ROUTER);
+/**
+* This time is the interval of the listing of a router, when its hop does not set one. A listing
+* evicts each expired flow that it walks past, and the DESTROY event of that flow follows.
+*/
+const PUMP = {
+	value: .1,
+	unit: "s"
+};
+/**
+* This time is the longest listing of the conntrack table of a router that a prediction assumes.
+* A DESTROY follows an expiry within the interval of the pump and two listings. On 2026-10-15,
+* the slowest of 500 listings in an idle router took 19.4 ms, and the median took 5.5 ms. A
+* longer listing writes a `pump-slow` line, and a run with one near its detection is void.
+*/
+const LISTING = {
+	value: 20,
+	unit: "ms"
+};
 /** This function gives the arguments of the injector for the options of a fault. */
 const optionArgs = (options) => [
 	...options.direction === void 0 ? [] : ["--direction", options.direction],
@@ -508,7 +682,8 @@ const LIFT = {
 	reject: "clear-reject",
 	mtu: "clear-mtu"
 };
-const lines = () => {
+/** This function collects the JSON lines of a program. It gives each line to `also` too, when the caller gives it. */
+const lines = (also) => {
 	const seen = [];
 	const other = [];
 	const waiting = [];
@@ -520,6 +695,7 @@ const lines = () => {
 		}
 		const line = JSON.parse(text);
 		seen.push(line);
+		also?.(line);
 		for (const wait of waiting.splice(0)) if (wait.kinds.includes(line.ev)) wait.resolve(line);
 		else waiting.push(wait);
 	};
@@ -551,6 +727,42 @@ const lines = () => {
 		}
 	};
 };
+/** This container class can share the process namespace of another container. */
+const shared = (Base) => class extends Base {
+	withPidMode(mode) {
+		this.hostConfig.PidMode = mode;
+		return this;
+	}
+};
+/** This function gives the content of a file of a proxy, with the address of the next hop for each {{next}}. */
+const templated = (content, next) => content.replaceAll("{{next}}", next);
+/** This function gives the JSON body of a toxic for the API of Toxiproxy. Each toxic of a cell has the name `fault`. */
+const toxicBody = (toxic) => JSON.stringify({
+	name: "fault",
+	type: toxic.type,
+	stream: toxic.stream,
+	toxicity: 1,
+	attributes: toxic.attributes
+});
+/**
+* This function gives the routes of a chain: for each router, the networks after it, which the
+* element before it reaches through it. Element 0 is the client, and element k is hop k - 1. Network
+* j is the network between element j and element j + 1.
+*/
+const routesOf = (hops) => hops.flatMap((hop, at) => hop.kind === "router" ? [{
+	from: at,
+	via: at + 1,
+	networks: Array.from({ length: hops.length - at }, (_, k) => at + 1 + k)
+}] : []);
+/**
+* This function gives the endpoint that an element connects to: the first element after it that
+* is not a router. Element 0 is the client, element k is hop k - 1, and the last element is the target.
+*/
+const endpointAfter = (hops, element) => {
+	let at = element + 1;
+	while (at <= hops.length && hops[at - 1].kind === "router") at += 1;
+	return at;
+};
 const STOP = { timeout: 0 };
 const output = async (container, argv) => {
 	const result = await container.exec(argv);
@@ -564,63 +776,268 @@ const netnsOf = async (container) => {
 	if (inode === void 0) throw new Error(`not a network namespace: ${link}`);
 	return Number(inode);
 };
+/** This function gives the image and the command of the client of each run, and its kind for the manifest. */
+const clientOf = async (kind, image) => {
+	if (typeof kind === "object") return {
+		clientImage: kind.image,
+		command: kind.command,
+		clientKind: "image"
+	};
+	if (kind === "ioredis") return {
+		clientImage: await nodeClientImage(),
+		command: ["node", "/opt/harness/node/client-ioredis.mjs"],
+		clientKind: kind
+	};
+	return {
+		clientImage: image,
+		command: ["/opt/harness/client"],
+		clientKind: kind
+	};
+};
 /**
 * This function opens one environment:
-* - a new network and the stock target.
+* - a chain of new networks, and the stock target on the last one. A direct pair has one network.
 * - a client that holds its own network namespace, with a privileged sidecar in it for its
-*   sysctls. Docker Desktop keeps /proc/sys read-only in a container that operates.
+*   sysctls and its routes. Docker Desktop keeps /proc/sys read-only in a container that operates
+*   without privileges.
+* - each hop of the topology, on the network of each of its sides. A hop is a router, or a stock
+*   proxy with an injector sidecar in its namespace.
 * - an injector sidecar in the namespace of the target.
-* - the prober, a privileged container that loads the kernel program and watches the two
-*   namespaces.
+* - the prober, a privileged container that loads the kernel program and watches the namespace of
+*   the client, of the target and of each hop.
 *
 * The client of each run is its own container in the namespace of the client. It is the TCP
-* client of the harness, or a driver from its own image (ioredis on Node). The two print the same
-* lines.
+* client of the harness, a driver from its own image (ioredis on Node), or an image that the
+* caller gives. Each prints the same lines.
+*
+* The client reaches the networks after a router through that router, by a route that its
+* privileged sidecar adds. A router translates the address of the side of the client, thus the
+* target and each later hop see the router as the peer.
 */
-const openLab = async (target, kind = "tcp") => {
+const openLab = async (target, kind = "tcp", topology) => {
 	const image = await harnessImage();
-	const clientImage = kind === "ioredis" ? await nodeClientImage() : image;
-	const command = kind === "ioredis" ? ["node", "/opt/harness/node/client-ioredis.mjs"] : ["/opt/harness/client"];
+	const { clientImage, command, clientKind } = await clientOf(kind, image);
+	const hops = topology?.hops ?? [];
 	const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();
+	const runtime = await getContainerRuntimeClient();
 	const opened = [];
-	let network;
+	const networks = [];
 	const close = async () => {
 		for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
-		await network?.stop().catch(() => void 0);
+		for (const network of networks.reverse()) await network.stop().catch(() => void 0);
+	};
+	/** This function gives the address of a container on a network, as Docker has it at this time. A container can join a network after its start. */
+	const ipOn = async (container, network) => {
+		const ip = (await runtime.container.inspect(runtime.container.getById(container.getId()))).NetworkSettings.Networks[network.getName()]?.IPAddress;
+		if (ip === void 0 || ip === "") throw new Error(`${container.getName()} has no address on ${network.getName()}`);
+		return ip;
+	};
+	const join = (container, network) => runtime.container.connectToNetwork(runtime.container.getById(container.getId()), runtime.network.getById(network.getId()), []);
+	const subnetOf = async (network) => {
+		const subnet = (await runtime.network.getById(network.getId()).inspect()).IPAM?.Config?.[0]?.Subnet;
+		if (subnet === void 0) throw new Error(`${network.getName()} has no subnet`);
+		return subnet;
 	};
 	try {
-		network = await new Network().start();
-		const stock = new GenericContainer(target.image).withNetwork(network).withWaitStrategy(Wait.forLogMessage(target.ready));
+		for (let at = 0; at <= hops.length; at += 1) networks.push(await new Network().start());
+		const last = networks[hops.length];
+		const stock = new GenericContainer(target.image).withNetwork(last).withWaitStrategy(Wait.forLogMessage(target.ready));
 		const server = await (target.command === void 0 ? stock : stock.withCommand([...target.command])).start();
 		opened.push(server);
-		const client = await new GenericContainer(image).withNetwork(network).start();
+		const serverIp = await ipOn(server, last);
+		const middlebox = [];
+		const inChain = [];
+		const endpointOf = (element) => {
+			const at = endpointAfter(hops, element);
+			if (at === hops.length + 1) return `${serverIp}:${target.port}`;
+			const proxy = inChain[at - 1];
+			return `${proxy.ip.client}:${proxy.hop.port}`;
+		};
+		for (let at = hops.length - 1; at >= 0; at -= 1) {
+			const hop = hops[at];
+			const [clientSide, targetSide] = [networks[at], networks[at + 1]];
+			if (hop.kind === "router") {
+				const seen = lines((line) => middlebox.push({
+					...line,
+					hop: hop.name
+				}));
+				const box = await new GenericContainer(await routerImage()).withNetwork(clientSide).withPrivilegedMode().withEnvironment({
+					FORM: hop.form,
+					IDLE: String(valueIn(hop.idle, "s")),
+					PUMP: String(valueIn(hop.pump ?? PUMP, "s")),
+					SLOW: String(valueIn(LISTING, "s"))
+				}).withLogConsumer(seen.consume).start();
+				opened.push(box);
+				await join(box, targetSide);
+				const ready = await seen.until(["mbx-ready"], 3e4, `router ${hop.name}`);
+				await seen.until(hop.form === "reset-both" ? ["resetter"] : ["ct-watch"], 3e4, `router ${hop.name}`);
+				inChain[at] = {
+					hop,
+					box,
+					injector: box,
+					ip: {
+						client: await ipOn(box, clientSide),
+						target: await ipOn(box, targetSide)
+					},
+					peer: {
+						client: "",
+						target: endpointOf(at + 1)
+					},
+					netns: await netnsOf(box),
+					sysctls: { "net.netfilter.nf_conntrack_tcp_timeout_established": String(ready["idle"]) }
+				};
+				continue;
+			}
+			const next = endpointOf(at + 1);
+			let proxy = new GenericContainer(hop.image).withNetwork(targetSide);
+			if (hop.command !== void 0) proxy = proxy.withCommand([...hop.command]);
+			if (hop.files !== void 0) proxy = proxy.withCopyContentToContainer(Object.entries(hop.files).map(([path, content]) => ({
+				content: templated(content, next),
+				target: path
+			})));
+			if (hop.ready !== void 0) proxy = proxy.withWaitStrategy(Wait.forLogMessage(hop.ready));
+			const box = await proxy.start();
+			opened.push(box);
+			await join(box, clientSide);
+			const injector = await new GenericContainer(image).withNetworkMode(`container:${box.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW").start();
+			opened.push(injector);
+			inChain[at] = {
+				hop,
+				box,
+				injector,
+				ip: {
+					client: await ipOn(box, clientSide),
+					target: await ipOn(box, targetSide)
+				},
+				peer: {
+					client: "",
+					target: next
+				},
+				netns: await netnsOf(injector),
+				sysctls: {}
+			};
+		}
+		const client = await new GenericContainer(image).withNetwork(networks[0]).start();
 		opened.push(client);
 		const admin = await new GenericContainer(image).withNetworkMode(`container:${client.getId()}`).withPrivilegedMode().start();
 		opened.push(admin);
 		const injector = await new GenericContainer(image).withNetworkMode(`container:${server.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW").start();
 		opened.push(injector);
+		const clientIp = await ipOn(client, networks[0]);
+		for (let at = 0; at < inChain.length; at += 1) {
+			const before = at === 0 ? clientIp : inChain[at - 1].ip.target;
+			inChain[at] = {
+				...inChain[at],
+				peer: {
+					...inChain[at].peer,
+					client: before
+				}
+			};
+		}
+		const admins = /* @__PURE__ */ new Map();
+		const adminOf = async (element) => {
+			if (element === 0) return admin;
+			const hop = inChain[element - 1];
+			if (hop.hop.kind === "router") return hop.box;
+			let found = admins.get(element);
+			if (found === void 0) {
+				found = await new GenericContainer(image).withNetworkMode(`container:${hop.box.getId()}`).withPrivilegedMode().start();
+				opened.push(found);
+				admins.set(element, found);
+			}
+			return found;
+		};
+		for (const route of routesOf(hops)) {
+			const where = await adminOf(route.from);
+			const via = inChain[route.via - 1].ip.client;
+			for (const network of route.networks) await output(where, [
+				"ip",
+				"route",
+				"replace",
+				await subnetOf(networks[network]),
+				"via",
+				via
+			]);
+		}
+		for (let at = 0; at < inChain.length; at += 1) {
+			const hop = inChain[at];
+			if (hop.hop.kind !== "proxy" || hop.hop.sysctls === void 0) continue;
+			const where = await adminOf(at + 1);
+			const settings = Object.entries(hop.hop.sysctls).map(([key, value]) => `${key}=${value}`);
+			await output(where, [
+				"sysctl",
+				"-q",
+				"-w",
+				...settings
+			]);
+			for (const key of Object.keys(hop.hop.sysctls)) hop.sysctls[key] = await output(where, [
+				"sysctl",
+				"-n",
+				key
+			]);
+		}
 		const netns = {
 			client: await netnsOf(admin),
-			target: await netnsOf(injector)
+			target: await netnsOf(injector),
+			...inChain.length === 0 ? {} : { hops: Object.fromEntries(inChain.map((hop) => [hop.hop.name, hop.netns])) }
 		};
 		const records = lines();
 		const prober = await new GenericContainer(image).withPrivilegedMode().withNetworkMode("none").withCommand([
 			"/opt/harness/prober",
 			String(netns.client),
-			String(netns.target)
+			String(netns.target),
+			...inChain.map((hop) => String(hop.netns))
 		]).withLogConsumer(records.consume).start();
 		opened.push(prober);
 		const begun = await records.until(["begin", "fail"], 3e4, "prober");
 		if (begun.ev === "fail") throw new Error(`the prober failed: ${String(begun["why"])}`);
-		const targetIp = server.getIpAddress(network.getName());
-		const clientIp = client.getIpAddress(network.getName());
-		const runtime = await getContainerRuntimeClient();
+		const first = endpointAfter(hops, 0);
+		const targetIp = first === hops.length + 1 ? serverIp : inChain[first - 1].ip.client;
+		const targetPort = first === hops.length + 1 ? target.port : inChain[first - 1].hop.port;
+		const targetPeer = inChain.length === 0 ? clientIp : inChain.at(-1).ip.target;
+		const placed = inChain.length === 0 ? null : {
+			targetPeer,
+			hops: Object.fromEntries(inChain.map((hop) => [hop.hop.name, {
+				netns: hop.netns,
+				client: hop.peer.client,
+				target: hop.peer.target
+			}]))
+		};
 		const paused = runtime.container.getById(server.getId());
 		let targetAdmin;
+		const hopNamed = (name) => {
+			const found = inChain.find((hop) => hop.hop.name === name);
+			if (found === void 0) throw new Error(`the topology has no hop ${name}`);
+			return found;
+		};
+		/** This function gives the namespace and the peer of the rules of a fault at a place. */
+		const placeOf = (at) => {
+			if (at === "client") return [admin, targetIp];
+			if (at === "target") return [injector, targetPeer];
+			const hop = hopNamed(at.hop);
+			if (hop.hop.kind !== "proxy") throw new Error(`a fault at ${at.hop} needs a proxy: a router forwards past INPUT and OUTPUT`);
+			return [hop.injector, at.toward === "client" ? hop.peer.client : hop.peer.target.split(":")[0]];
+		};
+		const toxiproxy = () => {
+			const found = inChain.find((hop) => hop.hop.kind === "proxy" && hop.hop.api !== void 0);
+			if (found === void 0) throw new Error("the topology has no hop with the API of Toxiproxy");
+			return found;
+		};
+		const stamped = async (where, argv) => {
+			const stamp = JSON.parse((await output(where, argv)).split("\n").at(-1));
+			return {
+				t0: stamp.t0,
+				t1: stamp.t1
+			};
+		};
+		const proxies = inChain.filter((hop) => hop.hop.kind === "proxy");
 		return {
 			targetIp,
+			targetPort,
 			clientIp,
 			netns,
+			topology: placed,
+			middlebox,
 			kernel: records.seen,
 			async sysctl(values) {
 				const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
@@ -673,21 +1090,16 @@ const openLab = async (target, kind = "tcp") => {
 				return read;
 			},
 			async fault(kind, at = "target", options = {}) {
-				const [where, peer] = at === "client" ? [admin, targetIp] : [injector, clientIp];
-				const args = [
+				const [where, peer] = placeOf(at);
+				return stamped(where, [
 					"/opt/harness/inject",
 					kind,
 					peer,
 					...optionArgs(options)
-				];
-				const stamp = JSON.parse(await output(where, args));
-				return {
-					t0: stamp.t0,
-					t1: stamp.t1
-				};
+				]);
 			},
 			async lift(kind, at = "target", options = {}) {
-				const [where, peer] = at === "client" ? [admin, targetIp] : [injector, clientIp];
+				const [where, peer] = placeOf(at);
 				await output(where, [
 					"/opt/harness/inject",
 					LIFT[kind],
@@ -707,11 +1119,7 @@ const openLab = async (target, kind = "tcp") => {
 					"eth0",
 					...spec.split(/\s+/)
 				];
-				const stamp = JSON.parse(await output(where, args));
-				return {
-					t0: stamp.t0,
-					t1: stamp.t1
-				};
+				return stamped(where, args);
 			},
 			async offload(on, at = "target") {
 				await output(at === "client" ? admin : injector, [
@@ -721,6 +1129,42 @@ const openLab = async (target, kind = "tcp") => {
 					on ? "on" : "off"
 				]);
 			},
+			async toxic(toxic) {
+				const hop = toxiproxy();
+				const api = `127.0.0.1:${hop.hop.api.port}`;
+				const path = `/proxies/${hop.hop.api.proxy}/toxics`;
+				return stamped(hop.injector, toxic === null ? [
+					"/opt/harness/inject",
+					"http",
+					"DELETE",
+					api,
+					`${path}/fault`
+				] : [
+					"/opt/harness/inject",
+					"http",
+					"POST",
+					api,
+					path,
+					toxicBody(toxic)
+				]);
+			},
+			async serverClose() {
+				if (target.close === void 0) throw new Error(`${target.image} names no way to close its clients`);
+				await output(server, [...target.close]);
+			},
+			async agent(agentImage, argv, options = {}) {
+				const Shared = shared(GenericContainer);
+				const seen = lines();
+				let started = new Shared(agentImage).withNetworkMode(`container:${server.getId()}`).withCommand([...argv]).withLogConsumer(seen.consume);
+				if (options.pid === true) started = started.withPidMode(`container:${server.getId()}`);
+				const container = await started.start();
+				opened.push(container);
+				return {
+					lines: seen,
+					exec: (args) => output(container, [...args]),
+					stop: () => container.stop(STOP).then(() => void 0).catch(() => void 0)
+				};
+			},
 			async kill() {
 				if (target.kill === void 0) throw new Error(`${target.image} names no way to kill its server`);
 				await output(server, [...target.kill]);
@@ -740,25 +1184,29 @@ const openLab = async (target, kind = "tcp") => {
 					"2",
 					"-W",
 					"2",
-					clientIp
+					targetPeer
+				]);
+				for (const hop of proxies) await output(hop.injector, [
+					"ping",
+					"-c",
+					"2",
+					"-W",
+					"2",
+					hop.peer.target.split(":")[0]
 				]);
 			},
 			async setup(commands) {
 				for (const argv of commands) await output(server, [...argv]);
 			},
 			async heal(at, when) {
-				const [where, peer] = at === "client" ? [admin, targetIp] : [injector, clientIp];
-				const stamp = JSON.parse(await output(where, [
+				const [where, peer] = placeOf(at);
+				return stamped(where, [
 					"/opt/harness/inject",
 					"clear",
 					peer,
 					"--at",
 					String(Math.round(when))
-				]));
-				return {
-					t0: stamp.t0,
-					t1: stamp.t1
-				};
+				]);
 			},
 			async forget() {
 				await output(admin, [
@@ -773,6 +1221,12 @@ const openLab = async (target, kind = "tcp") => {
 					"flush",
 					"all"
 				]);
+				for (const hop of proxies) await output(hop.injector, [
+					"ip",
+					"tcp_metrics",
+					"flush",
+					"all"
+				]);
 			},
 			freeze: () => paused.pause().then(() => void 0),
 			thaw: () => paused.unpause().then(() => void 0),
@@ -806,6 +1260,34 @@ const openLab = async (target, kind = "tcp") => {
 			async manifest(sysctls, settings = {}, targetSysctls = {}) {
 				const config = await output(admin, ["zcat", "/proc/config.gz"]).catch(() => "");
 				const inspect = async (container) => runtime.container.inspect(runtime.container.getById(container.getId()));
+				const hopsOf = async () => {
+					const out = [];
+					for (const hop of inChain) {
+						const box = hop.hop;
+						out.push({
+							name: box.name,
+							kind: box.kind,
+							image: box.kind === "router" ? await routerImage() : box.image,
+							id: (await inspect(hop.box)).Image,
+							settings: box.kind === "router" ? {
+								form: box.form,
+								idle: String(valueIn(box.idle, "s")),
+								pump: String(valueIn(box.pump ?? PUMP, "s")),
+								slow: String(valueIn(LISTING, "s")),
+								packages: (await output(hop.box, [
+									"apk",
+									"info",
+									"-v"
+								])).split("\n").sort().join(" ")
+							} : {
+								...box.settings,
+								...box.files
+							},
+							sysctls: hop.sysctls
+						});
+					}
+					return out;
+				};
 				return {
 					kernel: await output(client, ["uname", "-r"]),
 					architecture: await output(client, ["uname", "-m"]),
@@ -828,11 +1310,12 @@ const openLab = async (target, kind = "tcp") => {
 						])).split("\n").sort()
 					},
 					client: {
-						kind,
+						kind: clientKind,
 						image: clientImage
 					},
 					netns,
-					sysctls
+					sysctls,
+					...inChain.length === 0 ? {} : { hops: await hopsOf() }
 				};
 			},
 			close
@@ -850,7 +1333,7 @@ const openLab = async (target, kind = "tcp") => {
 * again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.
 */
 const judge = (cell, runs) => {
-	const measured = runs.map((run) => measure(cell, run));
+	const measured = runs.filter((run) => run.warmup !== true).map((run) => measure(cell, run));
 	return {
 		measured,
 		verdict: verdictOf(cell.prediction, measured)
@@ -864,10 +1347,10 @@ const droppedSoFar = (records) => {
 /** This function gives the argv of the client for a cell, with a deadline far after all that it waits for. */
 const probeArgs = (cell, lab) => {
 	const { prediction, probe } = cell;
-	const longest = prediction.detects ? valueIn(prediction.high, "s") : valueIn(prediction.window, "s");
+	const longest = (prediction.detects ? valueIn(prediction.high, "s") : valueIn(prediction.window, "s")) + (cell.idleBefore === void 0 ? 0 : valueIn(cell.idleBefore, "s"));
 	const args = [
 		lab.targetIp,
-		String(cell.target.port),
+		String(lab.targetPort),
 		"--mode",
 		probe.mode
 	];
@@ -885,19 +1368,21 @@ const probeArgs = (cell, lab) => {
 	return args;
 };
 /**
-* This function waits for an error on the socket of the target for the connection of the
-* client, or for its end of file (CLOSE_WAIT, state 8). The prober records both in the namespace
-* of the target. A cell on the server side waits for that. The promise resolves after `within` ms
-* at the latest.
+* This function waits for an error on the socket of the observer, or for its end of file
+* (CLOSE_WAIT, state 8). The prober records both. A cell whose observer is the server, or a proxy
+* of a topology, waits for that. The promise resolves after `within` ms at the latest.
 */
-const serverError = (lab, local, within) => new Promise((resolve) => {
+const observerError = (cell, partial, within) => new Promise((resolve) => {
 	const started = Date.now();
 	const look = () => {
-		if (lab.kernel.some((record) => record.netns === lab.netns.target && record.dst === local && (record.ev === "error" && (record.err ?? 0) !== 0 || record.ev === "state" && record.new === 8)) || Date.now() - started > within) resolve();
+		const run = partial();
+		const at = socketOf(cell, run);
+		if (at !== void 0 && run.kernel.some((record) => record.netns === at.netns && record.cookie === at.cookie && (record.ev === "error" && (record.err ?? 0) !== 0 || record.ev === "state" && record.new === 8)) || Date.now() - started > within) resolve();
 		else setTimeout(look, 50);
 	};
 	look();
 });
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /**
 * This function waits for the request that an in-flight client sends on the signal: its second
 * send, after the PING that made it ready. Its stamp is when the request first met the fault.
@@ -912,25 +1397,28 @@ const requestSent = (probe, within) => new Promise((resolve, reject) => {
 	};
 	look();
 });
+/** This function gives the netem specification of a cell: its latency, then the netem of its fault when it has one. */
+const netemSpec = (cell, fault) => {
+	const parts = [...cell.latency === void 0 ? [] : [`delay ${valueIn(cell.latency, "ms")}ms`], ...fault === void 0 ? [] : [fault]];
+	return parts.length === 0 ? null : parts.join(" ");
+};
 /**
 * This function does one run. The client connects and gets its reply, the fault occurs, and an
 * in-flight client sends into it. The harness then waits for the observer to see the fault, or for
 * the silent window to end. The observer is the client, or the socket of the target for a cell on
-* the server side.
+* the server side. It can also be a socket of a proxy of a topology.
 *
 * A stall stops by itself, at its length after the send of the request, and the
 * run waits for the reply. Then the harness removes the fault, thus the environment can serve the
 * next run. The run keeps the records of the prober from its start to its end. The frozen process
 * is the peer of the observer: the target, or the client for the server.
+*
+* A dry run does all of this but the fault: the warm-up of a cell whose fault ends its environment.
 */
-/** This function gives the netem specification of a cell: its latency, then the netem of its fault when it has one. */
-const netemSpec = (cell, fault) => {
-	const parts = [...cell.latency === void 0 ? [] : [`delay ${valueIn(cell.latency, "ms")}ms`], ...fault === void 0 ? [] : [fault]];
-	return parts.length === 0 ? null : parts.join(" ");
-};
-const once = async (lab, cell, environment, index) => {
+const once = async (lab, cell, environment, index, dry = false) => {
 	await lab.forget();
 	const from = lab.kernel.length;
+	const middleboxFrom = lab.middlebox.length;
 	const lostBefore = droppedSoFar(lab.kernel);
 	const probe = await lab.probe(probeArgs(cell, lab));
 	const server = cell.observer === "server";
@@ -945,89 +1433,124 @@ const once = async (lab, cell, environment, index) => {
 		const last = probe.events.slice(-3).map((event) => JSON.stringify(event));
 		throw new Error(`${error.message}; the client's last lines: ${last.join(" ")}`);
 	});
-	let fault;
+	const placed = lab.topology === null ? {} : { topology: lab.topology };
+	/** This function gives the run as the records show it so far: the waits read it. */
+	const partial = () => ({
+		environment,
+		index,
+		events: probe.events,
+		fault: null,
+		kernel: lab.kernel.slice(from),
+		netns: lab.netns.client,
+		targetNetns: lab.netns.target,
+		lost: 0,
+		...placed
+	});
+	let fault = null;
 	let heal = null;
 	try {
 		await probe.until(["ready"], 15e3);
-		const replyInFlight = cell.probe.mode === "inflight" && (server && cell.fault !== "mtu-blackhole" || cell.fault === "fin");
-		if (replyInFlight) {
-			await go();
-			await new Promise((resolve) => setTimeout(resolve, 200));
-		}
-		const faultAt = server ? "client" : "target";
-		const direction = cell.drops === void 0 ? void 0 : cell.drops === "to-server" === (faultAt === "target") ? "in" : "out";
-		const options = {
-			...direction === void 0 ? {} : { direction },
-			...cell.icmp === void 0 ? {} : { code: cell.icmp },
-			...cell.mtu === void 0 ? {} : { above: cell.mtu }
-		};
-		const apply = async () => {
-			switch (cell.fault) {
-				case "freeze-process":
-					await peer.freeze();
-					return null;
-				case "fin":
-					await (server ? probe.kill() : lab.kill());
-					return null;
-				case "unreachable": return lab.fault("reject", faultAt, { code: cell.icmp ?? "host" });
-				case "simplex": return lab.fault("blackhole", faultAt, options);
-				case "mtu-blackhole": return lab.fault("mtu", "client", { above: cell.mtu ?? 1200 });
-				case "netem": return lab.netem(netemSpec(cell, cell.netem), "target");
-				case "rst": return lab.fault("rst", faultAt);
-				default: return lab.fault("blackhole", faultAt);
+		if (dry) {
+			if (cell.probe.mode !== "idle") await go();
+			await sleep(500);
+		} else {
+			const replyInFlight = cell.probe.mode === "inflight" && (server && cell.fault !== "mtu-blackhole" || cell.fault === "fin");
+			if (replyInFlight) {
+				await go();
+				await sleep(200);
 			}
-		};
-		const lift = async () => {
-			switch (cell.fault) {
-				case "freeze-process": return peer.thaw();
-				case "fin": return;
-				case "unreachable": return lab.lift("reject", faultAt, { code: cell.icmp ?? "host" });
-				case "simplex": return lab.lift("blackhole", faultAt, options);
-				case "mtu-blackhole": return lab.lift("mtu", "client", { above: cell.mtu ?? 1200 });
-				case "netem":
-					await lab.netem(netemSpec(cell, void 0), "target");
-					return;
-				case "rst": return lab.lift("rst", faultAt);
-				default: return heal === null ? lab.lift("blackhole", faultAt) : void 0;
+			const faultAt = cell.faultAt ?? (server ? "client" : "target");
+			const towardServerComesIn = faultAt === "target" || typeof faultAt === "object" && faultAt.toward === "client";
+			const direction = cell.drops === void 0 ? void 0 : cell.drops === "to-server" === towardServerComesIn ? "in" : "out";
+			const options = {
+				...direction === void 0 ? {} : { direction },
+				...cell.icmp === void 0 ? {} : { code: cell.icmp },
+				...cell.mtu === void 0 ? {} : { above: cell.mtu }
+			};
+			const toxicFirst = cell.toxic !== void 0 && cell.fault !== "toxic";
+			if (toxicFirst) await lab.toxic(cell.toxic);
+			const apply = async () => {
+				switch (cell.fault) {
+					case "freeze-process":
+						await peer.freeze();
+						return null;
+					case "fin":
+						await (server ? probe.kill() : lab.kill());
+						return null;
+					case "unreachable": return lab.fault("reject", faultAt, { code: cell.icmp ?? "host" });
+					case "simplex": return lab.fault("blackhole", faultAt, options);
+					case "mtu-blackhole": return lab.fault("mtu", "client", { above: cell.mtu ?? 1200 });
+					case "netem": return lab.netem(netemSpec(cell, cell.netem), "target");
+					case "rst": return lab.fault("rst", faultAt);
+					case "middlebox-forget":
+					case "proxy-idle": return null;
+					case "toxic":
+						if (cell.toxic === void 0) throw new Error(`${cell.name} names no toxic`);
+						return lab.toxic(cell.toxic);
+					case "server-close":
+						await lab.serverClose();
+						return null;
+					default: return lab.fault("blackhole", faultAt);
+				}
+			};
+			const lift = async () => {
+				switch (cell.fault) {
+					case "freeze-process": return peer.thaw();
+					case "fin":
+					case "middlebox-forget":
+					case "proxy-idle":
+					case "server-close": return;
+					case "unreachable": return lab.lift("reject", faultAt, { code: cell.icmp ?? "host" });
+					case "simplex": return lab.lift("blackhole", faultAt, options);
+					case "mtu-blackhole": return lab.lift("mtu", "client", { above: cell.mtu ?? 1200 });
+					case "netem":
+						await lab.netem(netemSpec(cell, void 0), "target");
+						return;
+					case "rst": return lab.lift("rst", faultAt);
+					case "toxic":
+						await lab.toxic(null);
+						return;
+					default: return heal === null ? lab.lift("blackhole", faultAt) : void 0;
+				}
+			};
+			fault = await apply();
+			try {
+				if (cell.idleBefore !== void 0 && cell.probe.mode !== "idle") await sleep(valueIn(cell.idleBefore, "ms"));
+				if (cell.probe.mode !== "idle" && !replyInFlight) await go();
+				const { prediction } = cell;
+				if (cell.fault === "stall") {
+					if (cell.stall === void 0) throw new Error(`${cell.name} says no stall's length`);
+					const sent = await requestSent(probe, 5e3);
+					heal = await lab.heal(faultAt, sent.t + valueIn(cell.stall, "s") * 1e9);
+					const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
+					await probe.until([
+						"data",
+						"error",
+						"eof"
+					], window * 1e3).catch(() => void 0);
+				} else if (prediction.detects === null) {
+					const window = valueIn(prediction.window, "s") * 1e3;
+					await probe.until([
+						"done",
+						"error",
+						"eof",
+						"deadline"
+					], window).catch(() => void 0);
+				} else if (prediction.detects) {
+					const within = valueIn(prediction.high, "s") * 2e3 + 1e4;
+					if (cell.observer !== void 0 && cell.observer !== "client") await observerError(cell, partial, within);
+					else await probe.until([
+						"error",
+						"eof",
+						"deadline"
+					], within).catch(() => void 0);
+				} else await sleep(valueIn(prediction.window, "s") * 1e3);
+			} finally {
+				await lift();
+				if (toxicFirst) await lab.toxic(null);
 			}
-		};
-		fault = await apply();
-		try {
-			if (cell.probe.mode !== "idle" && !replyInFlight) await go();
-			const { prediction } = cell;
-			if (cell.fault === "stall") {
-				if (cell.stall === void 0) throw new Error(`${cell.name} says no stall's length`);
-				const sent = await requestSent(probe, 5e3);
-				heal = await lab.heal(faultAt, sent.t + valueIn(cell.stall, "s") * 1e9);
-				const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
-				await probe.until([
-					"data",
-					"error",
-					"eof"
-				], window * 1e3).catch(() => void 0);
-			} else if (prediction.detects === null) {
-				const window = valueIn(prediction.window, "s") * 1e3;
-				await probe.until([
-					"done",
-					"error",
-					"eof",
-					"deadline"
-				], window).catch(() => void 0);
-			} else if (prediction.detects) {
-				const within = valueIn(prediction.high, "s") * 2e3 + 1e4;
-				if (server) {
-					const local = probe.events.find((event) => event.ev === "connected")?.["local"];
-					await serverError(lab, local, within);
-				} else await probe.until([
-					"error",
-					"eof",
-					"deadline"
-				], within).catch(() => void 0);
-			} else await new Promise((resolve) => setTimeout(resolve, valueIn(prediction.window, "s") * 1e3));
-		} finally {
-			await lift();
+			if (cell.fault === "freeze-process" && !server) await probe.until(["data"], 5e3).catch(() => void 0);
 		}
-		if (cell.fault === "freeze-process" && !server) await probe.until(["data"], 5e3).catch(() => void 0);
 	} finally {
 		await probe.stop();
 	}
@@ -1042,14 +1565,25 @@ const once = async (lab, cell, environment, index) => {
 		kernel: lab.kernel.slice(from),
 		netns: lab.netns.client,
 		targetNetns: lab.netns.target,
-		lost
+		lost,
+		...placed,
+		...lab.topology === null ? {} : { middlebox: lab.middlebox.slice(middleboxFrom) }
 	};
 };
 /** This function divides a count of runs across environments, as equally as possible. */
 const spread = (runs, environments) => Array.from({ length: environments }, (_, at) => Math.floor(runs / environments) + (at < runs % environments ? 1 : 0)).filter((count) => count > 0);
-/** This function does the part of the runs of a cell in one environment, and records its manifest. */
+/** This function tells if the fault of a cell ends its environment: a server that stopped cannot serve the next run. */
+const endsEnvironment = (cell) => cell.fault === "fin" && (cell.observer === void 0 || cell.observer === "client");
+/** This function says one line for a run. */
+const line = (cell, run, read) => `${cell.name}: env ${run.environment} ${run.warmup === true ? "warm-up" : `run ${run.index}`} ${read.outcome.kind}${read.elapsed === null ? "" : ` ${read.elapsed.value} s`}${read.void === null ? "" : ` (void: ${read.void})`}`;
+/**
+* This function does the part of the runs of a cell in one environment, and records its manifest.
+* A warm-up goes first, and the verdict does not count it. On a loaded machine, a new environment is
+* sometimes slow at first: its first exec, its first client, its first fault. The warm-up
+* of a cell whose fault ends the environment is a dry run, without the fault.
+*/
 const inEnvironment = async (cell, environment, count, say) => {
-	const lab = await openLab(cell.target, cell.probe.driver?.name ?? "tcp");
+	const lab = await openLab(cell.target, cell.client ?? cell.probe.driver?.name ?? "tcp", cell.topology);
 	try {
 		const settings = await lab.configure(cell.targetSettings ?? {});
 		const targetSysctls = await lab.targetSysctl(cell.targetSysctls ?? {});
@@ -1062,16 +1596,21 @@ const inEnvironment = async (cell, environment, count, say) => {
 		if (latency !== null) await lab.netem(latency, "target");
 		await lab.warm();
 		const manifest = await lab.manifest(await lab.sysctl(cell.sysctls), settings, targetSysctls);
+		const warmup = {
+			...await once(lab, cell, environment, -1, endsEnvironment(cell)),
+			warmup: true
+		};
+		say(line(cell, warmup, measure(cell, warmup)));
 		const runs = [];
 		for (let index = 0; index < count; index += 1) {
 			const run = await once(lab, cell, environment, index);
 			runs.push(run);
-			const read = measure(cell, run);
-			say(`${cell.name}: env ${environment} run ${index} ${read.outcome.kind}${read.elapsed === null ? "" : ` ${read.elapsed.value} s`}${read.void === null ? "" : ` (void: ${read.void})`}`);
+			say(line(cell, run, measure(cell, run)));
 		}
 		return {
 			manifest,
-			runs
+			runs,
+			warmup
 		};
 	} finally {
 		await lab.close();
@@ -1083,11 +1622,14 @@ const inEnvironment = async (cell, environment, count, say) => {
 * the cell from 11 runs to 22, and then to 44, each extension in new environments. With a
 * concurrency of more than one, that many environments operate at the same time, each with its
 * own prober. The runs then go across at least as many environments.
+*
+* Each environment does a warm-up first, and the result keeps it apart from the counted runs.
 */
 const runCell = async (cell, say = () => {}) => {
 	const started = (/* @__PURE__ */ new Date()).toISOString();
 	const manifests = [];
 	const runs = [];
+	const warmups = [];
 	const concurrency = Math.max(1, cell.concurrency ?? 1);
 	let target = cell.runs;
 	let next = 0;
@@ -1102,6 +1644,7 @@ const runCell = async (cell, say = () => {}) => {
 			for (const each of done) {
 				manifests.push(each.manifest);
 				runs.push(...each.runs);
+				warmups.push(each.warmup);
 			}
 		}
 		const { measured, verdict } = judge(cell, runs);
@@ -1110,6 +1653,7 @@ const runCell = async (cell, say = () => {}) => {
 			cell,
 			manifests,
 			runs,
+			warmups,
 			measured,
 			verdict,
 			at: {
@@ -1121,6 +1665,6 @@ const runCell = async (cell, say = () => {}) => {
 	}
 };
 //#endregion
-export { NOISE, RELATIVE, STEPS, harnessImage, judge, measure, medianInterval, nodeClientImage, openLab, runCell, spread, verdictOf };
+export { LISTING, NOISE, PUMP, RELATIVE, STEPS, endpointAfter, harnessImage, judge, measure, medianInterval, nodeClientImage, openLab, routerImage, routesOf, runCell, socketOf, spread, templated, toxicBody, verdictOf };
 
 //# sourceMappingURL=index.js.map