@mark1russell7/harness 0.14.4 → 0.15.1

package/build/index.js

@@ -1,25 +1,308 @@
-import { A as unlockedManifests, C as programOf, D as subjectProblems, E as subjectClient, F as jarVersions, I as locksOf, L as npmVersions, M as LOCK_NAMES, N as cargoVersions, O as targetImageOf, P as goVersions, R as versionsOf, S as planOf, T as shippedSource, _ as kafkaClientImage, a as SHIPPED_NAMES, b as nodeHttpImage, c as buildRecordOf, d as harnessImage, f as harnessVersion, g as inContext, h as imageProblems, i as SHIPPED_CLIENTS, j as unpinnedBases, k as targetSourceOf, l as goHttpImage, m as imageOf, n as PINNED, o as amqpClientImage, p as hikariClientImage, r as REPOSITORY, s as basesOf, t as CLIENT_NAMES, u as grpcClientImage, v as mongoClientImage, w as routerImage, x as pgClientImage, y as nodeClientImage } from "./image-EZ8ioZ3L.js";
+import { A as unlockedManifests, B as isHopPlace, C as programOf, D as subjectProblems, E as subjectClient, F as jarVersions, H as requestFirst, I as locksOf, L as npmVersions, M as LOCK_NAMES, N as cargoVersions, O as targetImageOf, P as goVersions, R as versionsOf, S as planOf, T as shippedSource, V as latestEndOf, _ as kafkaClientImage, a as SHIPPED_NAMES, b as nodeHttpImage, c as buildRecordOf, d as harnessImage, f as harnessVersion, g as inContext, h as imageProblems, i as SHIPPED_CLIENTS, j as unpinnedBases, k as targetSourceOf, l as goHttpImage, m as imageOf, n as PINNED, o as amqpClientImage, p as hikariClientImage, r as REPOSITORY, s as basesOf, t as CLIENT_NAMES, u as grpcClientImage, v as mongoClientImage, w as routerImage, x as pgClientImage, y as nodeClientImage, z as CLIENT_CAPABILITIES } from "./image-BxVTFZ1o.js";
 import { quantity, valueIn } from "@mark1russell7/quantity";
-import { median, medianInterval as medianInterval$1, quantileSorted } from "@mark1russell7/rigor-statistics";
+import { median, medianInterval as medianInterval$1, quantile, quantileSorted } from "@mark1russell7/rigor-statistics";
+import { rungVerdict } from "@mark1russell7/rigor-bench";
+import { closedLoopPlan, measureLoad, schedule, scheduleHeld } from "@mark1russell7/rigor-chaos";
 import { createHash, randomBytes } from "node:crypto";
 import { execFile } from "node:child_process";
 import { mkdtemp, rm, writeFile } from "node:fs/promises";
 import { release, tmpdir } from "node:os";
 import { join } from "node:path";
 import { existsSync, readFileSync } from "node:fs";
-//#region src/cell.ts
-/** This function tells if a fault place is a hop of a topology. */
-const isHopPlace = (place) => typeof place === "object" && "hop" in place;
+//#region src/load.ts
+/**
+* The measure and the verdict of an open-loop cell (SYS-68). A client on a schedule sends each
+* arrival at its time, thus a stall counts in each arrival that it delayed. The package rigor-chaos
+* counts each latency from the planned time. The package rigor-bench judges a quantile by a
+* distribution-free interval.
+*/
+/** This path is where the client of an open-loop cell finds its schedule. */
+const SCHEDULE_PATH = "/tmp/schedule.json";
+/** This function gives the time of a line of the client, in ms from the start of the schedule. */
+const fromStart = (start, t) => (t - start) / 1e6;
+/**
+* This function gives the measure of one run of a schedule from the lines of its client. It gives
+* null when the client did not start the schedule. The first `sent` line of a client is its request
+* before `ready`, and it names no arrival. The end of the run is its last line.
+*/
+const loadOf = (events, plan) => {
+	const start = events.find((event) => event.ev === "scheduled")?.t;
+	if (start === void 0) return null;
+	return measureOf(events, start, plan, void 0);
+};
+/** This function tells if a status of a reply is a failure: a number outside 200 to 299. */
+const failedStatus = (status) => typeof status === "number" && (status < 200 || status > 299);
+/** This function gives the measure of the arrivals of one stream: the lines with the key `key`, or each line for no key. */
+const measureOf = (events, start, plan, key) => {
+	const sent = /* @__PURE__ */ new Map();
+	const done = /* @__PURE__ */ new Map();
+	let failed = 0;
+	for (const event of events) {
+		const index = event["i"];
+		if (typeof index !== "number" || key !== void 0 && event["k"] !== key) continue;
+		if (event.ev === "sent") sent.set(index, fromStart(start, event.t));
+		if (event.ev === "replied") {
+			done.set(index, fromStart(start, event.t));
+			if (failedStatus(event["status"])) failed += 1;
+		}
+	}
+	const completions = [...sent].map(([index, at]) => ({
+		index,
+		sent: at,
+		done: done.get(index) ?? null
+	}));
+	const end = events.reduce((latest, event) => Math.max(latest, fromStart(start, event.t)), 0);
+	return {
+		...measureLoad(plan, completions, end),
+		failed
+	};
+};
+/** This path is where the client `load` finds its plan. */
+const LOAD_PATH = "/tmp/load.json";
+/** This text is the format of the plan of the client `load`. */
+const LOAD_FORMAT = "systems-load/1";
 /**
-* This function tells if the request of a client goes before the fault of its cell. That is so for a
-* hop event that comes while the request is in flight (`HopEvent.inFlight`). Then a request after the fault is void.
+* This function gives the schedule of each stream of a cell, and the text of the plan of the client
+* `load`. It throws for no stream, for two streams with one key and for a stream with no request.
 */
-const requestFirst = (cell) => cell.fault === "hop-event" && cell.event?.inFlight === true;
+const loadPlanOf = (streams) => {
+	if (streams.length === 0) throw new RangeError("a plan of the client load has no stream");
+	const keys = /* @__PURE__ */ new Set();
+	for (const stream of streams) {
+		if (keys.has(stream.key)) throw new RangeError(`two streams have the key ${stream.key}`);
+		if (stream.requests.length === 0) throw new RangeError(`the stream ${stream.key} has no request`);
+		keys.add(stream.key);
+	}
+	const scheduled = streams.map((stream) => ({
+		key: stream.key,
+		schedule: schedule(stream.plan.profile, stream.plan.options),
+		requests: stream.requests
+	}));
+	return {
+		streams: scheduled.map(({ key, schedule: each }) => ({
+			key,
+			schedule: each
+		})),
+		content: JSON.stringify({
+			format: LOAD_FORMAT,
+			streams: scheduled
+		})
+	};
+};
+/**
+* This function gives the measure of each stream of a run of the client `load`, and their pool. Each
+* stream counts its arrivals from 0, thus the lines of a stream are the lines with its key. The pool
+* sums the counts and joins the times, in the order of the streams. It gives null when the client did
+* not start its schedule.
+*/
+const streamsLoadOf = (events, streams) => {
+	const start = events.find((event) => event.ev === "scheduled")?.t;
+	if (start === void 0) return null;
+	const each = streams.map((stream) => [stream.key, measureOf(events, start, stream.schedule, stream.key)]);
+	return {
+		pool: poolOf(each.map(([, measure]) => measure)),
+		streams: Object.fromEntries(each)
+	};
+};
+/** This function gives the pool of some measures: it sums the counts and joins the times, in the order of the measures. */
+const poolOf = (measures) => {
+	const sum = (count) => measures.reduce((total, measure) => total + count(measure), 0);
+	const join = (times) => measures.flatMap(times);
+	return {
+		planned: sum((measure) => measure.planned),
+		unsent: sum((measure) => measure.unsent),
+		unfinished: sum((measure) => measure.unfinished),
+		failed: sum((measure) => measure.failed ?? 0),
+		latency: join((measure) => measure.latency),
+		service: join((measure) => measure.service),
+		lag: join((measure) => measure.lag),
+		censored: join((measure) => measure.censored)
+	};
+};
+/**
+* This time is the tolerance of a closed-loop estimate, in ms. A send later than its expected time
+* by more than this tolerance is a pause of the client: a driver gap.
+*/
+const CLOSED_TOLERANCE = 5;
+/**
+* This function gives the estimate of one closed-loop stream from its requests, in the order of their
+* sends. Each time is a time of a line of the client. The client waits `interval` ms after each reply,
+* thus the interval is also the pause of the driver. The function gives undefined for no request, and
+* for a request with no reply before the last request.
+*/
+const estimateOf = (requests, end, interval) => {
+	const start = requests[0]?.sent;
+	if (start === void 0 || requests.slice(0, -1).some((request) => request.done === null)) return void 0;
+	const loop = requests.map((request) => ({
+		sent: fromStart(start, request.sent),
+		done: request.done === null ? null : fromStart(start, request.done)
+	}));
+	const stop = Math.max(fromStart(start, end), ...loop.map((request) => request.done ?? request.sent), interval);
+	const run = closedLoopPlan(loop, {
+		interval,
+		end: stop,
+		tolerance: 5,
+		pause: interval
+	});
+	return {
+		...measureLoad(run.plan, run.completions, stop),
+		hidden: run.hidden,
+		driverGaps: run.driverGaps,
+		driverGapTotal: run.driverGapTotal
+	};
+};
+/** This function gives the spread of a list of times in ms, or null for no time. */
+const spreadOf$1 = (values) => values.length === 0 ? null : {
+	p50: quantity(quantile(values, .5, "nearest-rank"), "ms"),
+	p99: quantity(quantile(values, .99, "nearest-rank"), "ms"),
+	max: quantity(values.reduce((most, value) => Math.max(most, value), 0), "ms")
+};
+/** This time is the date of the record of a band and the date of its verdict. The record has no age, because it comes from this run. */
+const NOW = "2026-01-01T00:00:00.000Z";
+/**
+* This function judges one band of the latency over the pooled arrivals of the counted runs. The
+* top of the band reads the interval of rigor-bench: the band holds only if the interval is under its top.
+* The latencies of one run can follow each other, thus the width of the interval is an estimate.
+* The bottom of the band reads the quantile itself. It gives the text of the verdict, and if it holds.
+*/
+const bandVerdict = (band, values, censored) => {
+	const high = valueIn(band.high, "ms");
+	const rung = {
+		name: "band",
+		at: {},
+		budgets: [{
+			kind: "quantile",
+			measure: "latency",
+			unit: "ms",
+			quantile: band.quantile,
+			limit: high
+		}],
+		test: "the cell"
+	};
+	const top = rungVerdict(rung, {
+		rung: "band",
+		commit: "the cell",
+		at: NOW,
+		real: ["the counted runs"],
+		synthetic: [],
+		evidence: [{
+			kind: "quantile",
+			measure: "latency",
+			values,
+			censored
+		}]
+	}, { now: NOW });
+	const read = values.length + censored.length === 0 ? null : quantile([...values, ...censored], band.quantile, "nearest-rank");
+	const low = band.low === void 0 ? null : valueIn(band.low, "ms");
+	const above = low === null || read !== null && read >= low;
+	const place = low === null ? `under ${high} ms` : `in [${low}, ${high}] ms`;
+	const outcome = top.outcome === "within" && !above ? "under the bottom of the band" : top.outcome;
+	return {
+		holds: top.outcome === "within" && above,
+		words: `p${band.quantile * 100} of the latency ${read === null ? "has no value" : `is ${read} ms`}, wanted ${place}: ${outcome}. ${top.reason}`
+	};
+};
 /**
-* This function gives the latest end of a band in seconds, from the reply of the client. It is
-* the high bound, after the latest reference of the band (`startsBy`).
+* This function judges the share of failed arrivals over the counted runs against a limit. Each
+* planned arrival is a trial. An arrival fails if the client did not send it. It also fails with no
+* reply at the end of its run, or with the status of a failure. The rate budget of rigor-bench
+* reads the upper end of the Wilson interval at 0.95. Thus 381 arrivals with no failure hold a limit
+* of 1 %, and fewer arrivals cannot decide.
 */
-const latestEndOf = (prediction) => valueIn(prediction.high, "s") + (prediction.startsBy === void 0 ? 0 : valueIn(prediction.startsBy, "s"));
+const failuresVerdict = (limit, failures, trials) => {
+	const verdict = rungVerdict({
+		name: "failures",
+		at: {},
+		budgets: [{
+			kind: "rate",
+			measure: "failures",
+			limit
+		}],
+		test: "the cell"
+	}, {
+		rung: "failures",
+		commit: "the cell",
+		at: NOW,
+		real: ["the counted runs"],
+		synthetic: [],
+		evidence: [{
+			kind: "rate",
+			measure: "failures",
+			failures,
+			trials
+		}]
+	}, { now: NOW });
+	return {
+		holds: verdict.outcome === "within",
+		words: `${failures} of ${trials} arrivals failed, wanted a share under ${limit * 100} %: ${verdict.outcome}. ${verdict.reason}`
+	};
+};
+/**
+* This function judges the load of an open-loop cell over its counted runs. A run that did not keep
+* its schedule measures its driver, not the target, thus it is a finding of that run. Each
+* band reads all the arrivals of the counted runs.
+*/
+const loadVerdict = (prediction, runs) => {
+	const limit = valueIn(prediction.lag, "ms");
+	const outside = runs.flatMap((run) => {
+		if (run.load === void 0) return [{
+			run: run.name,
+			why: "the client never started its schedule"
+		}];
+		const check = scheduleHeld(run.load, limit);
+		return check.held ? [] : [{
+			run: run.name,
+			why: check.reason
+		}];
+	});
+	const measures = runs.flatMap((run) => run.load === void 0 ? [] : [run.load]);
+	const latency = measures.flatMap((measure) => measure.latency);
+	const censored = measures.flatMap((measure) => measure.censored);
+	const bands = prediction.bands.map((band) => bandVerdict(band, latency, censored));
+	const total = (count) => measures.reduce((sum, measure) => sum + count(measure), 0);
+	const planned = total((measure) => measure.planned);
+	const unsent = total((measure) => measure.unsent);
+	const unfinished = total((measure) => measure.unfinished);
+	const failed = total((measure) => measure.failed ?? 0);
+	const failures = prediction.failures === void 0 ? void 0 : failuresVerdict(prediction.failures, unsent + unfinished + failed, planned);
+	return {
+		holds: runs.length > 0 && outside.length === 0 && bands.every((band) => band.holds) && (failures?.holds ?? true),
+		outside,
+		load: {
+			planned,
+			unsent,
+			unfinished,
+			failed,
+			latency: spreadOf$1([...latency, ...censored]),
+			service: spreadOf$1(measures.flatMap((measure) => measure.service)),
+			lag: spreadOf$1(measures.flatMap((measure) => measure.lag.map((lag) => Math.abs(lag)))),
+			bands: bands.map((band) => band.words),
+			...failures === void 0 ? {} : { failures: failures.words }
+		}
+	};
+};
+/** This function gives the estimate verdict of the counted runs, or undefined when no run has an estimate. */
+const estimateVerdict = (estimates) => {
+	const runs = estimates.filter((estimate) => estimate !== void 0);
+	if (runs.length === 0) return void 0;
+	const pool = poolOf(runs);
+	const total = (count) => runs.reduce((sum, estimate) => sum + count(estimate), 0);
+	const hidden = total((estimate) => estimate.hidden);
+	const driverGaps = total((estimate) => estimate.driverGaps);
+	const driverGapTotal = total((estimate) => estimate.driverGapTotal);
+	const latency = spreadOf$1([...pool.latency, ...pool.censored]);
+	const seconds = (time) => (valueIn(time, "ms") / 1e3).toFixed(3);
+	const spread = latency === null ? "no latency" : `median ${seconds(latency.p50)} s, p99 ${seconds(latency.p99)} s, slowest ${seconds(latency.max)} s`;
+	const gaps = driverGaps === 0 ? "no pause of the client" : `${driverGaps} pauses of the client, ${Math.round(driverGapTotal)} ms in all`;
+	return {
+		planned: pool.planned,
+		hidden,
+		driverGaps,
+		driverGapTotal: quantity(driverGapTotal, "ms"),
+		latency,
+		words: `The estimate of the open loop puts back ${hidden} hidden sends, with ${gaps}: ${spread}.`
+	};
+};
 //#endregion
 //#region src/verdict.ts
 /** This function gives the words of the outcome of a second request: its connection, a retry, and its end. */
@@ -345,14 +628,15 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
 		why: "no run counted"
 	};
 	if (prediction.detects === null) {
-		if (prediction.queue !== void 0 || prediction.ports !== void 0) {
+		const law = prediction.queue !== void 0 ? "queue" : prediction.ports !== void 0 ? "port range" : prediction.pause !== void 0 ? "paused consumer" : prediction.conntrack !== void 0 ? "conntrack table" : void 0;
+		if (law !== void 0) {
 			const off = new Set(outside.map((each) => each.run)).size;
 			return {
 				...empty,
 				decision: off === 0 ? "holds" : "fails",
 				band: null,
 				outside,
-				why: `${counted.length - off} of ${counted.length} runs hold the law of the ${prediction.queue === void 0 ? "port range" : "queue"}; ${prediction.why}`
+				why: `${counted.length - off} of ${counted.length} runs hold the law of the ${law}; ${prediction.why}`
 			};
 		}
 		const crossed = counted.flatMap((run) => run.crossings === void 0 ? [] : [run.crossings]);
@@ -420,6 +704,33 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
 				why: `${counted.length} runs measured: ${outcomes}; ${timed}.${late} ${prediction.why}`
 			};
 		}
+		if (counted.some((run) => run.load !== void 0)) {
+			const judged = loadVerdict(prediction.load ?? {
+				lag: quantity(Number.MAX_SAFE_INTEGER, "ms"),
+				bands: []
+			}, counted.map((run) => ({
+				name: name(run),
+				load: run.load
+			})));
+			const middleOfLoad = judged.load.latency?.p50;
+			const failed = judged.load.failed === 0 ? "" : `, ${judged.load.failed} with the status of a failure`;
+			const read = [
+				`${counted.length} runs on a schedule, ${judged.load.planned} arrivals, ${judged.load.unsent} not sent, ${judged.load.unfinished} with no reply${failed}.`,
+				...judged.load.bands,
+				...judged.load.failures === void 0 ? [] : [judged.load.failures],
+				prediction.why
+			].join(" ");
+			return {
+				...empty,
+				decision: prediction.load === void 0 ? "measured" : judged.holds ? "holds" : "fails",
+				band: null,
+				outside: judged.outside,
+				median: middleOfLoad === void 0 ? null : s(valueIn(middleOfLoad, "s")),
+				...judged.load.latency === null ? {} : { slowest: s(valueIn(judged.load.latency.max, "s")) },
+				load: judged.load,
+				why: read
+			};
+		}
 		const samples = counted.flatMap((run) => (run.samples ?? []).map((sample) => valueIn(sample, "s"))).sort((a, b) => a - b);
 		const retransmissions = counted.reduce((sum, run) => sum + run.retransmissions, 0);
 		const ended = counted.filter((run) => run.outcome.kind === "error" || run.outcome.kind === "eof");
@@ -433,7 +744,9 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
 			reordering.fast,
 			reordering.seen
 		].some((count) => (count ?? 0) > 0) ? ` Reordering: ${reorderingWords(reordering)}.` : "";
-		const why = samples.length === 0 ? `${counted.length} runs measured, with no reply: ${prediction.why}` : `${counted.length} runs, ${samples.length} replies: median ${s(median(samples)).value} s, p99 ${s(quantileOf(samples, .99)).value} s, slowest ${s(samples.at(-1)).value} s, ${retransmissions} retransmissions, ${ended.length} connections ended.${reordered} ${prediction.why}`;
+		const estimate = estimateVerdict(counted.map((run) => run.estimate));
+		const estimated = estimate === void 0 ? "" : ` ${estimate.words}`;
+		const why = samples.length === 0 ? `${counted.length} runs measured, with no reply: ${prediction.why}` : `${counted.length} runs, ${samples.length} replies: median ${s(median(samples)).value} s, p99 ${s(quantileOf(samples, .99)).value} s, slowest ${s(samples.at(-1)).value} s, ${retransmissions} retransmissions, ${ended.length} connections ended.${reordered}${estimated} ${prediction.why}`;
 		return {
 			...empty,
 			decision: "measured",
@@ -442,6 +755,7 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
 			median: samples.length === 0 ? null : s(median(samples)),
 			...samples.length === 0 ? {} : { slowest: s(samples.at(-1)) },
 			...reordering === void 0 ? {} : { reordering },
+			...estimate === void 0 ? {} : { estimate },
 			why
 		};
 	}
@@ -1351,6 +1665,285 @@ const measureQueue = (run, expected) => {
 	};
 };
 //#endregion
+//#region src/conntrack.ts
+/** This errno tells that a connect got a slot. */
+const NONE = "none";
+const numberOr = (line, field) => {
+	const value = line?.[field];
+	return typeof value === "number" ? value : null;
+};
+/** This function gives the time from one stamp to a stamp of a field of a line, when the line has it. */
+const untilField = (from, line, field) => {
+	const stamp = numberOr(line, field);
+	return from === void 0 || stamp === null ? null : between$2(from.t, stamp);
+};
+const stallOf = (line) => line === void 0 ? null : {
+	errno: String(line["errno"]),
+	time: quantity(Math.round((numberOr(line, "elapsed") ?? 0) / 1e3) / 1e6, "s"),
+	drops: numberOr(line, "drops") ?? 0,
+	retransFail: numberOr(line, "retrans_fail") ?? 0
+};
+/** This function tells where a reading differs from the law of its cell. */
+const findingsOf$1 = (read, failed, expected) => {
+	const found = [];
+	const { max, neighbor, refused, drained, pressure } = expected;
+	if (typeof failed === "string") found.push(`a connect failed with ${failed} before the client held its flows`);
+	if (read.flows !== max) found.push(`the client held ${read.flows} flows, not ${max}`);
+	if (read.count !== max) found.push(`the count read ${read.count} at the fill, not ${max}`);
+	if (neighbor !== void 0 && read.neighbor !== neighbor) found.push(`the namespace of the target read ${read.neighbor ?? "nothing"}, not ${neighbor}`);
+	if (refused !== void 0) {
+		const stall = read.stall;
+		if (stall === null) found.push("no connect after the fill ended");
+		else {
+			if (stall.errno !== refused.errno) found.push(`the connect after the fill ended with ${stall.errno}, not ${refused.errno}`);
+			if (!inside(stall.time, refused.within)) found.push(`the connect after the fill took ${valueIn(stall.time, "s")} s, outside ${bandWords(refused.within)}`);
+			if (stall.drops !== refused.drops) found.push(`the field drop rose by ${stall.drops}, not ${refused.drops}`);
+			if (stall.retransFail !== refused.retransFail) found.push(`TCPRetransFail rose by ${stall.retransFail}, not ${refused.retransFail}`);
+		}
+	}
+	if (drained !== void 0) {
+		for (const [name, time] of [["first fall", read.firstFall], ["zero", read.zero]]) if (time === null || !inside(time, drained)) found.push(`the ${name} of the count came at ${time === null ? "no time" : `${valueIn(time, "s")} s`}, outside ${bandWords(drained)}`);
+	}
+	if (pressure !== void 0) {
+		const stall = read.stall;
+		if (read.closed !== max) found.push(`the count read ${read.closed ?? "nothing"} after each side closed, not ${max}`);
+		const { highest } = pressure;
+		if (highest === void 0) {
+			if (read.lowest !== pressure.lowest) found.push(`the lowest count was ${read.lowest ?? "nothing"}, not ${pressure.lowest}`);
+			if (read.after !== pressure.after) found.push(`the count read ${read.after ?? "nothing"} after the connect, not ${pressure.after}`);
+		} else if (read.lowest === null || read.lowest < pressure.lowest || read.lowest > highest) found.push(`the lowest count was ${read.lowest ?? "nothing"}, outside ${pressure.lowest} to ${highest}`);
+		else if (read.after !== read.lowest + 1) found.push(`the count read ${read.after ?? "nothing"} after the connect, not ${read.lowest + 1}`);
+		if (stall === null) found.push("no connect after the close ended");
+		else {
+			if (stall.errno !== NONE) found.push(`the connect after the close ended with ${stall.errno}`);
+			if (!pressure.points.some((point) => inside(stall.time, point))) found.push(`the connect after the close took ${valueIn(stall.time, "s")} s, on no point of the schedule`);
+		}
+	}
+	return found;
+};
+/**
+* This function reads one run of a conntrack cell from the lines of its client and its target. An
+* `unsettled` line makes the run void. The entries of the run before it were still in the table, thus
+* the run did not start from an empty table.
+*/
+const measureConntrack = (run, expected) => {
+	const { events, kernel } = run;
+	const retransmissions = kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length;
+	const base = {
+		environment: run.environment,
+		index: run.index,
+		outcome: { kind: "answered" },
+		elapsed: null,
+		socket: null,
+		noticed: null,
+		steps: [],
+		probes: 0,
+		retransmissions,
+		acks: 0
+	};
+	const voided = (why) => ({
+		...base,
+		void: why,
+		finding: null
+	});
+	const unsettled = events.find((event) => event.ev === "unsettled");
+	if (unsettled !== void 0) return voided(`the table held ${String(unsettled["count"])} entries at the end of the wait before the run`);
+	if (events.find((event) => event.ev === "done") === void 0) {
+		const error = events.find((event) => event.ev === "error");
+		return voided(error === void 0 ? "the client did not reach its end" : `the client did not reach its end: ${String(error["what"])}`);
+	}
+	if (run.lost === null) return voided("the prober wrote no clock line after the run, so nothing says what it dropped");
+	if (run.lost > 0) return voided(`the prober dropped ${run.lost} records`);
+	const full = events.find((event) => event.ev === "full");
+	if (full === void 0) return voided("the client wrote no line for its fill");
+	const stall = events.find((event) => event.ev === "stall");
+	const closing = events.find((event) => event.ev === "closing");
+	const drained = events.find((event) => event.ev === "drained");
+	const closed = events.find((event) => event.ev === "closed");
+	const neighbor = (run.server ?? []).filter((line) => line.ev === "count" && line.t >= full.t && (stall === void 0 || line.t <= stall.t)).at(-1);
+	const conntrack = {
+		flows: numberOr(full, "flows") ?? 0,
+		count: numberOr(full, "count") ?? 0,
+		neighbor: numberOr(neighbor, "count"),
+		stall: stallOf(stall),
+		firstFall: untilField(closing, drained, "first_fall"),
+		zero: untilField(closing, drained, "zero"),
+		closed: numberOr(closed, "count"),
+		lowest: numberOr(drained ?? stall, "lowest"),
+		after: numberOr(stall, "after")
+	};
+	const found = findingsOf$1(conntrack, full["errno"], expected);
+	return {
+		...base,
+		void: null,
+		finding: found.length === 0 ? null : found.join(". "),
+		conntrack
+	};
+};
+//#endregion
+//#region src/pause.ts
+/** This expression finds the stamp of a log line of a broker, as its console writes it, on UTC. */
+const LOG_TIME = /\[(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d),(\d{3})\]/;
+/** This function gives the time of a log line on the monotonic clock of a member, in ns, from the `start` line of that member. It is null for a line with no stamp. */
+const logTimeOf = (line, start) => {
+	const found = LOG_TIME.exec(line);
+	if (found === null) return null;
+	const wall = Date.UTC(Number(found[1]), Number(found[2]) - 1, Number(found[3]), Number(found[4]), Number(found[5]), Number(found[6]), Number(found[7]));
+	return start.t + (wall - Number(start["wall"])) * 1e6;
+};
+/** This function gives the time between two instants of the monotonic clock, in s, to 1 microsecond. */
+const secondsBetween = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
+const isInside = (time, band) => {
+	const seconds = valueIn(time, "s");
+	return seconds >= valueIn(band.low, "s") && seconds <= valueIn(band.high, "s");
+};
+const bandText = (band) => `[${valueIn(band.low, "s")}, ${valueIn(band.high, "s")}] s`;
+const isChange = (event) => event.ev === "assigned" || event.ev === "revoked" || event.ev === "lost";
+/** This function gives the partitions that a line of the rebalance listener names, in its field `partitions` or `held`. */
+const partitionsOf = (event, field) => event[field] ?? [];
+/** This function gives the last line of the rebalance listener in a list of lines. */
+const lastChange = (events) => events.findLast(isChange);
+/** This function gives the offset that a commit names for a partition, or null. */
+const offsetOf = (event, partition) => {
+	const value = event["offsets"]?.[String(partition)];
+	return typeof value === "number" ? value : null;
+};
+/**
+* This function tells if the two members can take the fault. Each member holds one partition, the
+* two partitions differ, and the first member has two acknowledged commits on its partition after its
+* last change. Thus the first member reads and commits at a steady rate.
+*/
+const pauseReady = (first, other) => {
+	const mine = lastChange(first);
+	const theirs = lastChange(other);
+	if (mine === void 0 || theirs === void 0) return false;
+	const held = partitionsOf(mine, "held");
+	const others = partitionsOf(theirs, "held");
+	if (held.length !== 1 || others.length !== 1 || held[0] === others[0]) return false;
+	return first.filter((event) => event.ev === "commit" && event.t > mine.t && offsetOf(event, held[0]) !== null).length >= 2;
+};
+/** This function tells if a log line of a broker tells that a member left a group, under either protocol. */
+const isFenceLine = (line) => /Member \S+ fenced from the group because|Member \S+ in group \S+ has failed, removing it from the group/.test(line);
+/** This function tells if a fence line names a group. The two protocols write the name in different words. */
+const namesGroup = (line, group) => line.includes(`[GroupId ${group}]`) || line.includes(`in group ${group} has`);
+/** This function compares a reading to the law, and gives each disagreement. */
+const findingsOf = (reading, prediction, fences, otherReports) => {
+	const found = [];
+	const { commit, fence, takeover } = reading;
+	if (prediction.fence === null) {
+		if (fences.length > 0) found.push(`the group removed a member: ${fences[0]}`);
+		if (otherReports) found.push("the other member reported the partition");
+	} else {
+		if (fence === null) found.push("no log line tells of a fence, and the other member did not report the partition");
+		else if (!isInside(fence, prediction.fence)) found.push(`the fence is ${valueIn(fence, "s")} s, outside ${bandText(prediction.fence)}`);
+		if (reading.otherFirst !== reading.lastAck) found.push(`the other member started at offset ${reading.otherFirst}, and the last commit is ${reading.lastAck}`);
+	}
+	if (prediction.takeover !== null) {
+		if (takeover === null) found.push("the other member did not report the partition");
+		else if (!isInside(takeover, prediction.takeover)) found.push(`the other member reported the partition at ${valueIn(takeover, "s")} s, outside ${bandText(prediction.takeover)}`);
+	}
+	if (commit === null) {
+		found.push("the member made no commit after the stop");
+		return found;
+	}
+	switch (prediction.thaw) {
+		case "fails":
+			if (commit.name !== "CommitFailedException") found.push(`the commit after the stop gave ${commit.name ?? "an acknowledgment"}, not CommitFailedException`);
+			break;
+		case "acknowledged":
+			if (!commit.acked) found.push(`the commit after the stop failed with ${commit.name}`);
+			if (commit.epoch !== reading.epochBefore) found.push(`the epoch changed from ${reading.epochBefore} to ${commit.epoch}`);
+			break;
+		case "rejoin": {
+			const within = prediction.rejoinWithin;
+			if (reading.rejoin === null) found.push("the member did not get a new epoch");
+			else if (within !== void 0 && valueIn(reading.rejoin, "s") > valueIn(within, "s")) found.push(`the new epoch came at ${valueIn(reading.rejoin, "s")} s, above ${valueIn(within, "s")} s`);
+			if (!commit.acked) found.push(`the stale commit failed with ${commit.name}`);
+			if (reading.reassigned) found.push("the member held the partition again before its commit");
+			if (reading.committed !== reading.stale) found.push(`the committed offset is ${reading.committed}, and the stale commit named ${reading.stale}`);
+			if (reading.stale === null || reading.otherLast === null || reading.stale >= reading.otherLast) found.push(`the stale commit ${reading.stale} is not below the progress ${reading.otherLast} of the other member`);
+			break;
+		}
+	}
+	return found;
+};
+/** This function reads one run of a paused consumer against its law. */
+const measurePause = (run, prediction) => {
+	const retransmissions = run.kernel.filter((record) => record.netns === run.netns && record.ev === "retrans").length;
+	const base = {
+		environment: run.environment,
+		index: run.index,
+		outcome: { kind: "answered" },
+		elapsed: null,
+		socket: null,
+		noticed: null,
+		steps: [],
+		probes: 0,
+		retransmissions,
+		acks: 0
+	};
+	const voided = (why) => ({
+		...base,
+		void: why,
+		finding: null
+	});
+	const { fault, heal, pause } = run;
+	if (fault === null || pause === void 0) return voided("the partitions were not one for each member before the stop");
+	if (heal === void 0 || heal === null) return voided("the stop did not end");
+	const stop = fault.t0;
+	const thaw = heal.t0;
+	const before = run.events.filter((event) => event.t < stop);
+	const after = run.events.filter((event) => event.t >= stop);
+	const start = run.events.find((event) => event.ev === "start");
+	if (start === void 0) return voided("the member wrote no start line");
+	const mine = lastChange(before);
+	const theirs = lastChange(pause.other.filter((event) => event.t < stop));
+	const held = mine === void 0 ? [] : partitionsOf(mine, "held");
+	const others = theirs === void 0 ? [] : partitionsOf(theirs, "held");
+	const partition = held[0];
+	if (partition === void 0 || held.length !== 1 || others.length !== 1 || others[0] === partition) return voided("the partitions were not one for each member before the stop");
+	if (after.find((event) => event.ev === "pause")?.["in"] !== "process") return voided("the stop landed outside the processing of a batch");
+	if (before.findLast((event) => event.ev === "read")?.["partition"] !== partition) return voided("the last read before the stop was on another partition");
+	const lastCommit = before.findLast((event) => event.ev === "commit" && offsetOf(event, partition) !== null);
+	if (lastCommit === void 0) return voided("the member had no acknowledged commit on its partition before the stop");
+	const member = String(mine?.["member"]);
+	const fenceAt = pause.fences.filter((line) => line.includes(member)).map((line) => logTimeOf(line, start)).find((time) => time !== null && time >= stop) ?? null;
+	const taken = pause.other.find((event) => event.ev === "assigned" && event.t >= stop && partitionsOf(event, "partitions").includes(partition));
+	if (prediction.fence !== null && fenceAt === null && taken !== void 0) return voided("the broker log lost the line of the fence");
+	const thawed = after.find((event) => event.ev === "commit" || event.ev === "commit-fail");
+	const rejoined = after.find((event) => event.ev === "rejoined");
+	const ackAt = thawed?.t ?? Number.POSITIVE_INFINITY;
+	const otherCommit = pause.other.findLast((event) => event.ev === "commit" && event.t < ackAt && offsetOf(event, partition) !== null);
+	const committed = after.find((event) => event.ev === "committed");
+	const epochOf = (event) => typeof event?.["epoch"] === "number" ? event["epoch"] : null;
+	const pauseReading = {
+		partition,
+		fence: fenceAt === null ? null : secondsBetween(stop, fenceAt),
+		takeover: taken === void 0 ? null : secondsBetween(stop, taken.t),
+		commit: thawed === void 0 ? null : {
+			acked: thawed.ev === "commit",
+			name: typeof thawed["name"] === "string" ? thawed["name"] : null,
+			message: typeof thawed["message"] === "string" ? thawed["message"] : null,
+			epoch: epochOf(thawed)
+		},
+		epochBefore: epochOf(lastCommit),
+		lastAck: offsetOf(lastCommit, partition),
+		otherFirst: pause.other.find((event) => event.ev === "read" && event.t >= stop && event["partition"] === partition)?.["first"] ?? null,
+		otherLast: otherCommit === void 0 ? null : offsetOf(otherCommit, partition),
+		rejoin: rejoined === void 0 ? null : secondsBetween(thaw, rejoined.t),
+		reassigned: after.some((event) => event.ev === "assigned" && event.t >= thaw && event.t <= ackAt && partitionsOf(event, "partitions").includes(partition)),
+		committed: committed === void 0 ? null : offsetOf(committed, partition),
+		stale: thawed === void 0 ? null : offsetOf(thawed, partition)
+	};
+	const findings = findingsOf(pauseReading, prediction, pause.fences, taken !== void 0);
+	return {
+		...base,
+		void: null,
+		finding: findings.length === 0 ? null : findings.join(". "),
+		pause: pauseReading
+	};
+};
+//#endregion
 //#region src/ports.ts
 /** This errno ends a search that found no port. */
 const NO_PORT = "EADDRNOTAVAIL";
@@ -2312,6 +2905,21 @@ const requestOf = (events) => {
 };
 /** This function reads the socket of one run against its cell, as `measure` does without the second request of an HTTP client. */
 const measureSocket = (cell, run) => {
+	if (run.missing !== void 0) return {
+		environment: run.environment,
+		index: run.index,
+		outcome: { kind: "answered" },
+		elapsed: null,
+		socket: null,
+		noticed: null,
+		steps: [],
+		probes: 0,
+		retransmissions: 0,
+		acks: 0,
+		void: `the environment cannot do this cell: ${run.missing}`,
+		finding: null
+	};
+	if (cell.prediction.detects === null && cell.prediction.pause !== void 0) return measurePause(run, cell.prediction.pause);
 	if (cell.group !== void 0 || cell.replicaSet !== void 0 || cell.cluster !== void 0 || cell.pair !== void 0 || cell.kafka !== void 0) {
 		const read = measureGroup(cell, run);
 		return {
@@ -2332,6 +2940,7 @@ const measureSocket = (cell, run) => {
 	}
 	if (cell.prediction.detects === null && cell.prediction.queue !== void 0) return measureQueue(run, cell.prediction.queue);
 	if (cell.prediction.detects === null && cell.prediction.ports !== void 0) return measurePorts(run, cell.prediction.ports);
+	if (cell.prediction.detects === null && cell.prediction.conntrack !== void 0) return measureConntrack(run, cell.prediction.conntrack);
 	if (cell.prediction.detects !== null && cell.prediction.driver !== void 0) return measureDriver(cell, run);
 	const { events, kernel } = run;
 	const ready = events.find((event) => event.ev === "ready");
@@ -2462,13 +3071,26 @@ const measureSocket = (cell, run) => {
 		forgot: forgotten === void 0 ? null : between(lastRx, forgotten.t)
 	};
 	if (prediction.detects === null) {
-		const sends = events.filter((event) => event.ev === "sent").slice(1);
+		const load = run.streams !== void 0 ? streamsLoadOf(events, run.streams)?.pool ?? null : run.schedule === void 0 ? void 0 : loadOf(events, run.schedule);
+		if (load === null) return {
+			...base,
+			...nothing,
+			void: "the client never started its schedule"
+		};
+		const sends = load === void 0 ? events.filter((event) => event.ev === "sent").slice(1) : [];
 		const answers = events.some((event) => event.ev === "replied") ? "replied" : "data";
-		const samples = sends.flatMap((send, n) => {
+		const requests = sends.map((send, n) => {
 			const next = sends[n + 1]?.t ?? Infinity;
-			const reply = events.find((event) => event.ev === answers && event.t > send.t && event.t < next);
-			return reply === void 0 ? [] : [between(send.t, reply.t)];
+			return {
+				send,
+				reply: events.find((event) => event.ev === answers && event.t > send.t && event.t < next)
+			};
 		});
+		const samples = requests.flatMap(({ send, reply }) => reply === void 0 ? [] : [between(send.t, reply.t)]);
+		const estimate = cell.probe.mode === "stream" && cell.probe.driver === void 0 && cell.probe.interval !== void 0 ? estimateOf(requests.map(({ send, reply }) => ({
+			sent: send.t,
+			done: reply?.t ?? null
+		})), events.reduce((latest, event) => Math.max(latest, event.t), 0), valueIn(cell.probe.interval, "ms")) : void 0;
 		const from = prediction.from === void 0 ? void 0 : referenceAt(prediction.from);
 		const until = ended ?? after.at(-1)?.t;
 		const peerRecords = peerCookie === void 0 ? [] : kernel.filter((record) => record.netns === peerNetns && record.cookie === peerCookie);
@@ -2500,6 +3122,8 @@ const measureSocket = (cell, run) => {
 			noticed: prediction.from === void 0 ? null : noticed,
 			steps: retransmitted.filter((record) => record.mono > request.t).map((record) => between(request.t, record.mono)),
 			samples,
+			...load === void 0 ? {} : { load },
+			...estimate === void 0 ? {} : { estimate },
 			reordering: reorderingOf(socket.filter((record) => record.mono > request.t), peerRecords.filter((record) => record.mono > request.t)),
 			...crossings,
 			void: voided
@@ -3446,6 +4070,7 @@ const clientOf = async (kind) => {
 		clientImage: await imageOf(client.image),
 		command: client.command,
 		clientKind: typeof kind === "string" ? kind : "image",
+		...client.capabilities === void 0 || client.capabilities.length === 0 ? {} : { capabilities: client.capabilities },
 		...typeof client.image === "string" ? {} : { source: client.image }
 	};
 };
@@ -3477,18 +4102,21 @@ const deviceOf = (route) => {
 */
 const openLab = async (target, kind = "tcp", topology) => {
 	const image = await harnessImage();
-	const { clientImage, command, clientKind, source } = await clientOf(kind);
+	const { clientImage, command, clientKind, capabilities, source } = await clientOf(kind);
 	const hops = topology?.hops ?? [];
 	const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();
 	const runtime = await getContainerRuntimeClient();
 	const opened = [];
 	const networks = [];
 	const serverLines = lines();
+	let restoreHost = () => Promise.resolve([]);
 	let fabric;
 	const close = async () => {
+		const restored = await restoreHost().then(() => null, (error) => error);
 		await fabric?.close();
 		for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
 		for (const network of networks.reverse()) await network.stop().catch(() => void 0);
+		if (restored !== null) throw restored;
 	};
 	/** This function gives the address of a container on a network, as Docker has it at this time. A container can join a network after its start. */
 	const ipOn = async (container, network) => {
@@ -3793,6 +4421,33 @@ const openLab = async (target, kind = "tcp", topology) => {
 			};
 		};
 		const proxies = inChain.filter((hop) => hop.hop.kind === "proxy");
+		let hostAdmin;
+		const hostChanges = [];
+		let hostDone;
+		restoreHost = async () => {
+			if (hostDone !== void 0) return hostDone;
+			const restored = [];
+			for (const change of [...hostChanges].reverse()) {
+				await output(hostAdmin, [
+					"sysctl",
+					"-q",
+					"-w",
+					`${change.name}=${change.before}`
+				]);
+				const read = await output(hostAdmin, [
+					"sysctl",
+					"-n",
+					change.name
+				]);
+				if (read !== change.before) throw new Error(`${change.name} reads ${read} after the restore, not ${change.before}`);
+				restored.unshift({
+					...change,
+					restored: read
+				});
+			}
+			hostDone = restored;
+			return restored;
+		};
 		return {
 			serverIp,
 			targetIp,
@@ -3839,6 +4494,53 @@ const openLab = async (target, kind = "tcp", topology) => {
 				]);
 				return read;
 			},
+			async namespaceSetup(commands) {
+				for (const command of commands) {
+					await output(admin, [...command]);
+					await output(injector, [...command]);
+				}
+			},
+			async hostSysctl(settings) {
+				if (settings.length === 0) return;
+				if (hostAdmin === void 0) {
+					hostAdmin = await new GenericContainer(image).withNetworkMode("host").withPrivilegedMode().start();
+					opened.push(hostAdmin);
+				}
+				for (const { name, value, guard } of settings) {
+					if (guard !== void 0) {
+						const reads = Number(await output(hostAdmin, [
+							"sysctl",
+							"-n",
+							guard.name
+						]));
+						if (!(reads <= guard.atMost)) throw new Error(`${guard.name} of the host reads ${reads}, above ${guard.atMost}`);
+					}
+					const before = await output(hostAdmin, [
+						"sysctl",
+						"-n",
+						name
+					]);
+					hostChanges.push({
+						name,
+						before,
+						set: value,
+						restored: null
+					});
+					await output(hostAdmin, [
+						"sysctl",
+						"-q",
+						"-w",
+						`${name}=${value}`
+					]);
+					const set = await output(hostAdmin, [
+						"sysctl",
+						"-n",
+						name
+					]);
+					if (set !== value) throw new Error(`${name} of the host reads ${set} after the write, not ${value}`);
+				}
+			},
+			restoreHost: () => restoreHost(),
 			async configure(values) {
 				const read = {};
 				if (Object.keys(values).length === 0) return read;
@@ -4090,9 +4792,10 @@ const openLab = async (target, kind = "tcp", topology) => {
 					query
 				]);
 			},
-			async probe(args) {
+			async probe(args, files = []) {
 				const events = lines();
-				const container = await new GenericContainer(clientImage).withNetworkMode(`container:${client.getId()}`).withCommand([...command, ...args]).withLogConsumer(events.consume).start();
+				const prober = new GenericContainer(clientImage).withNetworkMode(`container:${client.getId()}`).withCommand([...command, ...args]).withCopyContentToContainer(files.map((file) => ({ ...file }))).withLogConsumer(events.consume);
+				const container = await (capabilities === void 0 ? prober : prober.withAddedCapabilities(...capabilities)).start();
 				return {
 					events: events.seen,
 					until: (kinds, within) => events.until(kinds, within, "client"),
@@ -4193,6 +4896,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 					client: {
 						kind: clientKind,
 						image: clientImage,
+						...capabilities === void 0 ? {} : { capabilities },
 						...source === void 0 ? {} : {
 							source,
 							...buildOf(source)
@@ -6978,6 +7682,7 @@ const openKafkaLab = async (kafka) => {
 		const injectors = /* @__PURE__ */ new Map();
 		const parts = {};
 		const raw = {};
+		const fences = [];
 		const openNode = async (name, environment) => {
 			let partial = "";
 			const box = await new GenericContainer(kafka.image).withNetwork(started).withNetworkAliases(name).withHostname(name).withEnvironment(environment).withWaitStrategy(Wait.forLogMessage(kafka.ready)).withStartupTimeout(18e4).withLogConsumer((stream) => stream.on("data", (chunk) => {
@@ -6988,6 +7693,7 @@ const openKafkaLab = async (kafka) => {
 					const tail = raw[name] ??= [];
 					tail.push(text);
 					if (tail.length > 400) tail.shift();
+					if (isFenceLine(text)) fences.push(text);
 				}
 			})).start();
 			opened.push(box);
@@ -7059,6 +7765,7 @@ const openKafkaLab = async (kafka) => {
 			netns: { client: clientNetns },
 			kernel: records.seen,
 			raw,
+			fences,
 			async sysctl(values) {
 				const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
 				if (settings.length > 0) await output(admin, [
@@ -7086,11 +7793,46 @@ const openKafkaLab = async (kafka) => {
 				}
 				throw new Error(`the brokers did not come up in ${within} ms: ${last}`);
 			},
-			async create(topic) {
+			async create(topic, partitions = 1) {
 				const ids = kafka.brokers.map((name) => nodeIdOf(kafka, name)).join(",");
-				const reply = await tool("create", topic, String(kafka.minInsync), ids);
+				const reply = await tool("create", topic, String(kafka.minInsync), ids, String(partitions));
 				if (!reply.includes("\"ok\":true")) throw new Error(`the create of the topic failed: ${reply}`);
 			},
+			async fill(topic, partitions, count) {
+				const reply = await tool("fill", topic, String(partitions), String(count));
+				if (!reply.includes("\"ok\":true")) throw new Error(`the fill of the topic failed: ${reply}`);
+			},
+			async consumer(args) {
+				const events = lines();
+				const container = await new GenericContainer(clientImage).withNetworkMode(`container:${holder.getId()}`).withCommand([
+					"sh",
+					"-c",
+					"java \"$@\"; :",
+					"consumer",
+					CLIENT_HEAP,
+					"-cp",
+					CLASS_PATH,
+					"ConsumerKafka",
+					bootstrap,
+					...args
+				]).withLogConsumer(events.consume).start();
+				const signaller = await withPid(new GenericContainer(image).withNetworkMode(`container:${holder.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW"), `container:${container.getId()}`).start();
+				return {
+					events: events.seen,
+					until: (kinds, within) => events.until(kinds, within, "client"),
+					signal: (signal, at) => stamped(signaller, [
+						"/opt/harness/inject",
+						"signal",
+						"java",
+						signal,
+						...after(at)
+					]),
+					stop: async () => {
+						await container.stop({ timeout: 2 }).catch(() => void 0);
+						await signaller.stop(STOP).catch(() => void 0);
+					}
+				};
+			},
 			async converge(topic, within = CONVERGE_MS) {
 				const begin = Date.now();
 				let calm = 0;
@@ -7342,6 +8084,105 @@ const kafkaOnce = async (lab, cell, environment, index) => {
 		}
 	};
 };
+/** This count is the partitions of the topic of a run of a paused consumer: one for each member. */
+const PAUSE_PARTITIONS = 2;
+/** This time bounds the wait for one partition for each member and for two commits, in ms. It has no source: it is far above two batches and one rebalance. */
+const PAUSE_READY_MS = 9e4;
+/** This time bounds the wait for the next batch of the first member, in ms. It has no source: it is far above one batch. */
+const PAUSE_BATCH_MS = 3e4;
+/**
+* This function does one run of a paused consumer:
+* - The topic of the run has two partitions, each with records that a run does not finish.
+* - Two members join one group, and the run waits until each member holds one partition and the
+*   first member has two commits on its partition.
+* - A SIGSTOP stops the first member while it holds its next batch, and a SIGCONT ends the stop
+*   after the hold of the cell. The run then waits for the tail.
+*
+* The fault is null when the members did not get one partition each. The measure then voids the run.
+* The law reads no kernel record, thus a lost record does not void this run.
+*/
+const kafkaPauseOnce = async (lab, cell, environment, index) => {
+	const pause = cell.kafka?.pause;
+	if (pause === void 0) throw new Error("a run of a paused consumer needs `Kafka.pause`");
+	if (cell.fault !== "freeze-process") throw new Error(`a paused consumer has no fault ${cell.fault}`);
+	if (cell.hold === void 0) throw new Error("a paused consumer needs a hold: its law reads it");
+	const hold = valueIn(cell.hold, "s");
+	const topic = topicOf(environment, index);
+	const group = `pause-e${environment}-r${index + 1}`;
+	await lab.brokersUp();
+	await lab.create(topic, PAUSE_PARTITIONS);
+	const leader = await lab.converge(topic);
+	await lab.fill(topic, PAUSE_PARTITIONS, pause.fill);
+	await lab.forget();
+	const from = lab.kernel.length;
+	const lostBefore = droppedSoFar$1(lab.kernel);
+	const common = [
+		"--topic",
+		topic,
+		"--group",
+		group,
+		"--protocol",
+		pause.protocol,
+		"--max-s",
+		String(Math.ceil(hold + 180)),
+		...Object.entries(cell.driverOptions ?? {}).flatMap(([name, value]) => ["--set", `${name}=${String(value)}`])
+	];
+	const members = [];
+	let fault = null;
+	let heal = null;
+	let otherEvents = [];
+	let firstEvents = [];
+	try {
+		const first = await lab.consumer([
+			...common,
+			"--process-ms",
+			String(valueIn(pause.process, "ms")),
+			"--after-pause",
+			pause.afterPause
+		]);
+		members.push(first);
+		const other = await lab.consumer([
+			...common,
+			"--process-ms",
+			"0",
+			"--after-pause",
+			"commit"
+		]);
+		members.push(other);
+		firstEvents = first.events;
+		otherEvents = other.events;
+		const begun = Date.now();
+		while (Date.now() - begun < PAUSE_READY_MS && !pauseReady(first.events, other.events)) await sleep$1(100);
+		if (pauseReady(first.events, other.events)) {
+			const seen = first.events.at(-1)?.t ?? 0;
+			const batch = await waitFor(first, (event) => event.ev === "processing" && event.t > seen, PAUSE_BATCH_MS);
+			if (batch !== void 0) {
+				fault = await first.signal("STOP", batch.t + valueIn(pause.stopAfter, "s") * 1e9);
+				heal = await first.signal("CONT", fault.t0 + hold * 1e9);
+				await sleep$1(KAFKA_TAIL_MS);
+			}
+		}
+	} finally {
+		for (const member of members) await member.stop();
+	}
+	const settled = await lab.settle();
+	return {
+		environment,
+		index,
+		events: [...firstEvents],
+		fault,
+		...heal === null ? {} : { heal },
+		kernel: lab.kernel.slice(from),
+		netns: lab.netns.client,
+		targetNetns: lab.members[leader].netns,
+		lost: settled === null ? null : settled - lostBefore,
+		pause: {
+			other: [...otherEvents],
+			fences: lab.fences.filter((line) => namesGroup(line, group)),
+			group
+		}
+	};
+};
 //#endregion
 //#region src/run.ts
 /**
@@ -7433,6 +8274,8 @@ const probeArgs = (cell, lab) => {
 	}
 	if (probe.keepalive) args.push("--keepalive");
 	if (probe.userTimeout !== void 0) args.push("--user-timeout-ms", String(valueIn(probe.userTimeout, "ms")));
+	if (probe.plan !== void 0) args.push("--schedule", SCHEDULE_PATH);
+	if (probe.streams !== void 0) args.push("--plan", LOAD_PATH);
 	args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
 	return args;
 };
@@ -7615,7 +8458,16 @@ const once = async (lab, cell, environment, index, dry = false) => {
 	const middleboxFrom = lab.middlebox.length;
 	const serverFrom = lab.server.length;
 	const lostBefore = droppedSoFar(lab.kernel);
-	const probe = await lab.probe(probeArgs(cell, lab));
+	const plan = cell.probe.plan === void 0 ? void 0 : schedule(cell.probe.plan.profile, cell.probe.plan.options);
+	const load = cell.probe.streams === void 0 ? void 0 : loadPlanOf(cell.probe.streams);
+	const files = [...plan === void 0 ? [] : [{
+		content: JSON.stringify(plan),
+		target: SCHEDULE_PATH
+	}], ...load === void 0 ? [] : [{
+		content: load.content,
+		target: LOAD_PATH
+	}]];
+	const probe = await lab.probe(probeArgs(cell, lab), files.length === 0 ? void 0 : files);
 	const server = cell.observer === "server";
 	const peer = server ? {
 		freeze: () => probe.freeze(),
@@ -7628,7 +8480,11 @@ const once = async (lab, cell, environment, index, dry = false) => {
 		const last = probe.events.slice(-3).map((event) => JSON.stringify(event));
 		throw new Error(`${error.message}; the client's last lines: ${last.join(" ")}`);
 	});
-	const placed = lab.topology === null ? {} : { topology: lab.topology };
+	const placed = {
+		...lab.topology === null ? {} : { topology: lab.topology },
+		...plan === void 0 ? {} : { schedule: plan },
+		...load === void 0 ? {} : { streams: load.streams }
+	};
 	const fabricOf = () => {
 		if (lab.fabric === void 0) throw new Error(`${cell.name} silences a router of a lab with no fabric`);
 		return lab.fabric;
@@ -7773,7 +8629,7 @@ const once = async (lab, cell, environment, index, dry = false) => {
 			fault = await apply();
 			try {
 				if (cell.idleBefore !== void 0 && cell.probe.mode !== "idle") await sleep(valueIn(cell.idleBefore, "ms"));
-				if ((cell.probe.mode === "inflight" || cell.probe.mode === "stream") && !replyInFlight) await go();
+				if ((cell.probe.mode === "inflight" || cell.probe.mode === "stream" || cell.probe.mode === "schedule") && !replyInFlight) await go();
 				const { prediction } = cell;
 				const http = cell.probe.http !== void 0;
 				if (prediction.detects !== null && prediction.driver !== void 0) {
@@ -7838,7 +8694,7 @@ const once = async (lab, cell, environment, index, dry = false) => {
 				await lift();
 				if (toxicFirst) await lab.toxic(null);
 			}
-			if (cell.fault === "freeze-process" && !server && cell.probe.mode !== "idle") await probe.until(["data"], 5e3).catch(() => void 0);
+			if (cell.fault === "freeze-process" && !server && cell.probe.mode !== "idle" && cell.probe.mode !== "schedule") await probe.until(["data"], 5e3).catch(() => void 0);
 			const expected = cell.prediction.detects !== false ? cell.prediction.reconnects : void 0;
 			if (expected !== void 0) {
 				const waits = expected.waits.reduce((sum, wait) => sum + valueIn(wait.high, "s"), 0);
@@ -7918,9 +8774,11 @@ const inEnvironment = async (cell, environment, count, say) => {
 		if (latency !== null) await lab.netem(latency, "target");
 		await lab.warm();
 		const manifest = await lab.manifest(await lab.sysctl(cell.sysctls), settings, targetSysctls);
+		const missing = await lab.namespaceSetup(cell.namespaceSetup ?? []).then(() => lab.hostSysctl(cell.hostSysctls ?? [])).then(() => void 0, (error) => error instanceof Error ? error.message : String(error));
 		const timed = (run) => ({
 			...run,
-			jitter: jitterOf(cell, run)
+			jitter: jitterOf(cell, run),
+			...missing === void 0 ? {} : { missing }
 		});
 		const warmup = {
 			...timed(await withSteal(() => once(lab, cell, environment, -1, endsEnvironment(cell)))),
@@ -7933,8 +8791,12 @@ const inEnvironment = async (cell, environment, count, say) => {
 			runs.push(run);
 			say(line(cell, run, measure(cell, run)));
 		}
+		const host = await lab.restoreHost();
 		return {
-			manifest,
+			manifest: host.length === 0 ? manifest : {
+				...manifest,
+				host
+			},
 			runs,
 			warmup
 		};
@@ -8064,16 +8926,17 @@ const inClusterEnvironment = async (cell, environment, count, say) => {
 */
 const inKafkaEnvironment = async (cell, environment, count, say) => {
 	const lab = await openKafkaLab(cell.kafka);
+	const once = cell.kafka.pause === void 0 ? kafkaOnce : kafkaPauseOnce;
 	try {
 		const manifest = await lab.manifest(await lab.sysctl(cell.sysctls));
 		const warmup = {
-			...await kafkaOnce(lab, cell, environment, -1),
+			...await once(lab, cell, environment, -1),
 			warmup: true
 		};
 		say(line(cell, warmup, measure(cell, warmup)));
 		const runs = [];
 		for (let index = 0; index < count; index += 1) {
-			const run = await kafkaOnce(lab, cell, environment, index);
+			const run = await once(lab, cell, environment, index);
 			runs.push(run);
 			say(line(cell, run, measure(cell, run)));
 		}
@@ -8140,6 +9003,6 @@ const runCell = async (cell, say = () => {}) => {
 	}
 };
 //#endregion
-export { AWS_FROM, CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, DAEMONS, DEBUG, DIRTY, DROP_HANDLING, FABRIC_CLIENT, FABRIC_DEVICE, FABRIC_TARGET, FRR_IMAGE, GIVES_AWAY, HASH_SCRIPT, INJECT, INSERT, KAFKA_TAIL_MS, KEPT, LINGER_MS, LISTING, LOCK_NAMES, LOSS_MARGIN, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, RESTART, ROUTERS, SEED_DOCUMENT, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STANDBY_READY, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, basesOf, bootstrapOf, brokerEnv, buildOf, buildRecordOf, callOf, cancelsOf, cargoVersions, clockOffsetOf, closeOrExit, cloudOf, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, commitProblems, comparable, controllerEnv, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, drillOnce, drillSpan, dropFinding, dropOf, endpointAfter, envOf, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, frrConf, goHttpImage, goVersions, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, harnessRecord, harnessVersion, hashOf, highestHeld, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isHopPlace, jarVersions, jitterOf, jitterOfLines, judge, kafkaArgs, kafkaClientImage, kafkaCutOf, kafkaOnce, kafkaViewOf, lagsOf, laterMove, latestEndOf, leafLacks, leftAfter, locksOf, logNameOf, manifestProblems, matches, measure, measureDriver, measureGroup, measurePorts, measureQueue, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicAt, monotonicOf, nodeClientImage, nodeCommand, nodeHttpImage, nodeIdOf, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, npmVersions, numbersOf, onMonotonic, openClusterLab, openFabric, openGroupLab, openKafkaLab, openLab, openReplicaSetLab, openStandby, pairArgs, pairOnce, pairingOf, pairingsOf, peersOf, pgClientImage, pingsOf, plainOf, planOf, programLines, programOf, quantileOf, readCpu, readHost, readOplog, reconnectFinding, reconnectOf, recordProblems, recoveryFindings, recoveryOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routeLine, routerImage, routesOf, runCell, seedScript, sentinelConfig, sessionsOf, shippedClientRecord, shippedSource, sliceQuery, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, standaloneCommand, standbyCommand, statusViewOf, stealShare, stopScript, subjectClient, subjectProblems, subscriptionsOf, substrateHere, substrateKey, substrateOf, substrateOn, targetImageOf, targetSourceOf, templated, toolJsonOf, topicOf, topologyOf, toxicBody, unlockedManifests, unpinnedBases, unpoolable, verdictOf, versionsOf, viewOf, writesOf };
+export { AWS_FROM, CLIENT_CAPABILITIES, CLIENT_NAMES, CLOSED_TOLERANCE, CLUSTER_STAGES, CONVERGE_MS, DAEMONS, DEBUG, DIRTY, DROP_HANDLING, FABRIC_CLIENT, FABRIC_DEVICE, FABRIC_TARGET, FRR_IMAGE, GIVES_AWAY, HASH_SCRIPT, INJECT, INSERT, KAFKA_TAIL_MS, KEPT, LINGER_MS, LISTING, LOAD_FORMAT, LOAD_PATH, LOCK_NAMES, LOSS_MARGIN, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, RESTART, ROUTERS, SCHEDULE_PATH, SEED_DOCUMENT, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STANDBY_READY, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, basesOf, bootstrapOf, brokerEnv, buildOf, buildRecordOf, callOf, cancelsOf, cargoVersions, clockOffsetOf, closeOrExit, cloudOf, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, commitProblems, comparable, controllerEnv, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, drillOnce, drillSpan, dropFinding, dropOf, endpointAfter, envOf, errorsOf, estimateOf, estimateVerdict, exitOf, fieldOf, fieldsOf, firstUnanswered, frrConf, goHttpImage, goVersions, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, harnessRecord, harnessVersion, hashOf, highestHeld, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isFenceLine, isHopPlace, jarVersions, jitterOf, jitterOfLines, judge, kafkaArgs, kafkaClientImage, kafkaCutOf, kafkaOnce, kafkaPauseOnce, kafkaViewOf, lagsOf, laterMove, latestEndOf, leafLacks, leftAfter, loadOf, loadPlanOf, loadVerdict, locksOf, logNameOf, logTimeOf, manifestProblems, matches, measure, measureConntrack, measureDriver, measureGroup, measurePause, measurePorts, measureQueue, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicAt, monotonicOf, namesGroup, nodeClientImage, nodeCommand, nodeHttpImage, nodeIdOf, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, npmVersions, numbersOf, onMonotonic, openClusterLab, openFabric, openGroupLab, openKafkaLab, openLab, openReplicaSetLab, openStandby, pairArgs, pairOnce, pairingOf, pairingsOf, pauseReady, peersOf, pgClientImage, pingsOf, plainOf, planOf, poolOf, programLines, programOf, quantileOf, readCpu, readHost, readOplog, reconnectFinding, reconnectOf, recordProblems, recoveryFindings, recoveryOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routeLine, routerImage, routesOf, runCell, seedScript, sentinelConfig, sessionsOf, shippedClientRecord, shippedSource, sliceQuery, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, standaloneCommand, standbyCommand, statusViewOf, stealShare, stopScript, streamsLoadOf, subjectClient, subjectProblems, subscriptionsOf, substrateHere, substrateKey, substrateOf, substrateOn, targetImageOf, targetSourceOf, templated, toolJsonOf, topicOf, topologyOf, toxicBody, unlockedManifests, unpinnedBases, unpoolable, verdictOf, versionsOf, viewOf, writesOf };
 
 //# sourceMappingURL=index.js.map