@mark1russell7/harness 0.0.0-stage → 0.2.0

package/build/index.js

@@ -0,0 +1,1126 @@
+import { quantity, valueIn } from "@mark1russell7/quantity";
+import { createHash } from "node:crypto";
+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, statSync } from "node:fs";
+import { createRequire } from "node:module";
+import { tmpdir } from "node:os";
+import { dirname, join, relative } from "node:path";
+import { fileURLToPath } from "node:url";
+//#region src/observe.ts
+/** These are the errno numbers that the kernel reports to a socket, by the names that predictions use. */
+const ERRNO = {
+	32: "EPIPE",
+	101: "ENETUNREACH",
+	103: "ECONNABORTED",
+	104: "ECONNRESET",
+	110: "ETIMEDOUT",
+	111: "ECONNREFUSED",
+	113: "EHOSTUNREACH"
+};
+/** This function gives the interval between two instants on the one clock, in seconds to the microsecond. */
+const between = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
+/** These functions give the states of a state record: before the change, and after it. TCP_ESTABLISHED is 1, TCP_CLOSE_WAIT 8. */
+const oldState = (record) => record.old;
+const newState = (record) => record.new;
+const count = (event, field) => event === void 0 ? 0 : Number(event[field] ?? 0);
+/** This function gives the view of the client on the same interval: TCP_INFO, with a sample each 2 ms. */
+const socketVantage = (cell, events, ready, request) => {
+	if (!cell.prediction.detects || cell.prediction.from === "peer-closed") return null;
+	const ended = events.find((event) => event.ev === "error" || event.ev === "eof");
+	const reference = cell.prediction.from === "last-received" ? ready : cell.prediction.from === "request-sent" ? request : events.find((event) => event.ev === "kernel" && event.t >= request.t && count(event, "retransmits") > 0);
+	return ended === void 0 || reference === void 0 ? null : between(reference.t, ended.t);
+};
+/**
+* This function reads one run against its cell, from the records of the kernel first.
+* - The socket of the run is the socket that the client connected from, in its namespace.
+* - The fault must hold before the event that it must meet. That is before the request goes into
+*   it, and before the first keepalive probe. A live peer answers that probe, and the answer
+*   starts the idle time again. A run where the fault was late is void. A run whose records the
+*   prober lost is also void.
+*/
+const measure = (cell, run) => {
+	const { events, kernel } = run;
+	const ready = events.find((event) => event.ev === "ready");
+	const local = events.find((event) => event.ev === "connected")?.["local"];
+	const inflight = cell.probe.mode !== "idle";
+	const request = inflight ? events.filter((event) => event.ev === "sent")[1] : ready;
+	const server = cell.observer === "server";
+	const netns = server ? run.targetNetns : run.netns;
+	const cookie = kernel.find((record) => record.netns === netns && (server ? record.dst : record.src) === local)?.cookie;
+	const socket = kernel.filter((record) => record.netns === netns && record.cookie === cookie);
+	const peerNetns = server ? run.netns : run.targetNetns;
+	const peerCookie = kernel.find((record) => record.netns === peerNetns && (server ? record.src : record.dst) === local)?.cookie;
+	const keepalives = socket.filter((record) => record.ev === "probe" && record.what === "keepalive");
+	const retransmitted = socket.filter((record) => record.ev === "retrans");
+	/**
+	* This function examines the timer that drives the measurement: keepalive or retransmit. It
+	* finds the first firing in an interval that was two jiffies or more behind its wheel slot. The
+	* wheel does a slot in the tick when it is due, thus such a delay comes from the CPU: the host
+	* stopped a vCPU. The prediction is about the timing of the kernel, and that firing measured
+	* the timing of the host. One jiffy is a tick that occurred while the wheel worked, and δ_abs
+	* includes it. The function ignores the other timer: the retransmit timer fires one time, idle,
+	* after a reply, and it does not move a keepalive.
+	*/
+	const delayed = (from, to, timer) => {
+		const firings = socket.filter((record) => record.ev === "timer" && record.what === timer && record.mono >= from && record.mono <= to);
+		const late = firings.find((record) => (record.behind?.value ?? 0) >= 2);
+		if (late !== void 0) return `its ${timer} timer ran ${late.behind.value} jiffies behind its wheel slot: the VM delayed it, not the kernel`;
+		return firings.find((record) => timer === "keepalive" && record.owned === true) === void 0 ? null : "its keepalive timer found the socket held by its owner and put itself off by HZ/20: the run measured the owner, not the timer";
+	};
+	const acks = request === void 0 ? 0 : socket.filter((record) => record.ev === "rx" && record.mono > request.t).length;
+	const base = {
+		environment: run.environment,
+		index: run.index,
+		probes: keepalives.length,
+		retransmissions: retransmitted.length,
+		acks
+	};
+	const nothing = {
+		outcome: { kind: "silent" },
+		elapsed: null,
+		socket: null,
+		noticed: null,
+		steps: [],
+		finding: null
+	};
+	if (ready === void 0) return {
+		...base,
+		...nothing,
+		void: "the client never had its reply"
+	};
+	if (request === void 0) return {
+		...base,
+		...nothing,
+		void: "the request was never sent into the fault"
+	};
+	if (run.lost === null) return {
+		...base,
+		...nothing,
+		void: "the prober wrote no clock line after the run, so nothing says what it dropped"
+	};
+	if (run.lost > 0) return {
+		...base,
+		...nothing,
+		void: `the prober dropped ${run.lost} records`
+	};
+	if (cookie === void 0) return {
+		...base,
+		...nothing,
+		void: "the prober saw nothing of this connection"
+	};
+	let voided = null;
+	if (run.fault !== null) {
+		const replyFirst = server && inflight && cell.fault !== "mtu-blackhole";
+		const replied = events.some((event) => event.ev === "data" && event.t > request.t && event.t < run.fault.t1);
+		if (inflight && !replyFirst && run.fault.t1 > request.t) voided = "the request went out before the fault held";
+		else if (replyFirst && run.fault.t1 < request.t) voided = "the request went out into the fault, so the server never had it";
+		else if (replyFirst && replied) voided = "the server's reply arrived before the fault held";
+		else if (keepalives[0] !== void 0 && keepalives[0].mono < run.fault.t1) voided = "a keepalive probe went out before the fault held";
+	}
+	const reported = socket.find((record) => record.ev === "error" && (record.err ?? 0) !== 0);
+	const seen = server ? void 0 : events.find((event) => event.ev === "error" || event.ev === "eof");
+	const closeWait = socket.find((record) => record.ev === "state" && newState(record) === 8);
+	const outcome = reported !== void 0 ? {
+		kind: "error",
+		errno: ERRNO[reported.err] ?? `E${reported.err}`
+	} : closeWait !== void 0 || seen?.ev === "eof" ? { kind: "eof" } : seen !== void 0 ? {
+		kind: "error",
+		errno: String(seen["errno"])
+	} : { kind: "silent" };
+	const ended = reported?.mono ?? closeWait?.mono ?? seen?.t;
+	const noticed = reported !== void 0 && seen !== void 0 ? between(reported.mono, seen.t) : null;
+	const { prediction } = cell;
+	if (prediction.detects === null) {
+		const sends = events.filter((event) => event.ev === "sent").slice(1);
+		const samples = sends.flatMap((send, n) => {
+			const next = sends[n + 1]?.t ?? Infinity;
+			const reply = events.find((event) => event.ev === "data" && event.t > send.t && event.t < next);
+			return reply === void 0 ? [] : [between(send.t, reply.t)];
+		});
+		return {
+			...base,
+			...nothing,
+			outcome: ended !== void 0 ? outcome : { kind: "answered" },
+			steps: retransmitted.filter((record) => record.mono > request.t).map((record) => between(request.t, record.mono)),
+			samples,
+			void: voided
+		};
+	}
+	if (!prediction.detects && prediction.recovers !== void 0) {
+		const { recovers } = prediction;
+		const heal = run.heal ?? null;
+		if (voided === null && heal === null) voided = "the stall never ended";
+		else if (voided === null && heal !== null) {
+			const from = between(request.t, heal.t0);
+			const to = between(request.t, heal.t1);
+			if (valueIn(from, "s") <= valueIn(recovers.heal.low, "s") || valueIn(to, "s") >= valueIn(recovers.heal.high, "s")) voided = `the stall ended ${from.value}–${to.value} s after the send, outside the ${recovers.heal.low.value}–${recovers.heal.high.value} s its prediction assumed`;
+		}
+		const steps = retransmitted.filter((record) => record.mono > request.t).map((record) => between(request.t, record.mono));
+		if (ended !== void 0) return {
+			...base,
+			...nothing,
+			outcome,
+			steps,
+			void: voided,
+			finding: `detected after ${between(request.t, ended).value} s, where it should recover`
+		};
+		const answer = heal === null ? void 0 : socket.find((record) => record.ev === "rx" && record.mono > heal.t0);
+		if (answer === void 0) return {
+			...base,
+			...nothing,
+			steps,
+			void: voided,
+			finding: "it never moved again: nothing came back after the stall ended"
+		};
+		const reply = events.find((event) => event.ev === "data" && event.t > request.t);
+		voided ??= delayed(request.t, answer.mono, "retransmit");
+		return {
+			...base,
+			outcome: { kind: "answered" },
+			elapsed: between(request.t, answer.mono),
+			socket: reply === void 0 ? null : between(request.t, reply.t),
+			noticed: reply === void 0 ? null : between(answer.mono, reply.t),
+			steps,
+			void: voided,
+			finding: null
+		};
+	}
+	if (!prediction.detects) {
+		const window = valueIn(prediction.window, "s") * 1e9;
+		if (ended === void 0 || ended - request.t > window) return {
+			...base,
+			...nothing,
+			void: voided
+		};
+		return {
+			...base,
+			...nothing,
+			outcome,
+			elapsed: between(request.t, ended),
+			void: voided,
+			finding: `detected after ${between(request.t, ended).value} s, where nothing should`
+		};
+	}
+	if (ended === void 0) return {
+		...base,
+		...nothing,
+		void: voided,
+		finding: `nothing detected it: ${prediction.timer} never fired`
+	};
+	const before = run.fault?.t1 ?? ready.t;
+	const kernelReference = prediction.from === "last-received" ? socket.filter((record) => record.ev === "rx" && record.mono <= before).at(-1) : prediction.from === "first-retransmission" ? retransmitted.find((record) => record.why === "rto") : prediction.from === "peer-closed" ? kernel.find((record) => record.netns === peerNetns && record.cookie === peerCookie && record.ev === "state" && oldState(record) === 1) : void 0;
+	const reference = prediction.from === "request-sent" ? request.t : kernelReference?.mono;
+	if (reference === void 0) return {
+		...base,
+		...nothing,
+		outcome,
+		noticed,
+		void: voided,
+		finding: `it ended without its reference event (${prediction.from})`
+	};
+	const steps = (prediction.from === "last-received" ? keepalives : prediction.from === "first-retransmission" ? retransmitted.filter((record) => record.why === "rto" && record.mono > reference) : []).map((record) => between(reference, record.mono));
+	const finding = (prediction.errno === "EOF" ? outcome.kind === "eof" : outcome.kind === "error" && outcome.errno === prediction.errno) ? null : `it ended with ${outcome.kind === "error" ? outcome.errno : "end of file"}, not ${prediction.errno}`;
+	if (prediction.from !== "request-sent" && prediction.from !== "peer-closed") voided ??= delayed(reference, ended, prediction.from === "last-received" ? "keepalive" : "retransmit");
+	return {
+		...base,
+		outcome,
+		elapsed: between(reference, ended),
+		socket: server ? null : socketVantage(cell, events, ready, request),
+		noticed,
+		steps,
+		void: voided,
+		finding
+	};
+};
+//#endregion
+//#region src/verdict.ts
+/**
+* This amount is the noise of the pipeline itself, δ_abs, as the calibration measured it. It is
+* the 99.9th percentile of the whole latency of a reset, at the concurrency cap or less. Its
+* minimum is one jiffy. On 2026-10-12 the worst of 60 resets at 1 and 4 environments took
+* 1.141 ms, thus the minimum sets it to 4 ms. The calibration occurs again after each change to
+* the harness, the kernel or Docker.
+*/
+const NOISE = quantity(4, "ms");
+/** This fraction, δ_rel, is how far the median of a cell can be outside the prediction and stay equivalent. */
+const RELATIVE = .05;
+/** These are the run counts of a cell: 11, then 22 and 44 while its median crosses the edge of the band. */
+const STEPS = [
+	11,
+	22,
+	44
+];
+const s = (value) => quantity(Math.round(value * 1e6) / 1e6, "s");
+const name = (run) => `env ${run.environment} run ${run.index}`;
+/** This function gives P(X ≤ j) for X ~ Binomial(n, 1/2), exactly. */
+const below = (n, j) => {
+	let term = 1;
+	let sum = 0;
+	for (let i = 0; i <= j; i += 1) {
+		sum += term;
+		term = term * (n - i) / (i + 1);
+	}
+	return sum / 2 ** n;
+};
+/**
+* This function gives the distribution-free interval of a median: [x_(k), x_(n+1−k)], for the
+* largest k whose two tails each hold 2.5 % or less. For eleven runs, it gives [x_(2), x_(10)],
+* at 98.8 %.
+*/
+const medianInterval = (sorted) => {
+	const n = sorted.length;
+	let k = 0;
+	while (k + 1 <= n / 2 && below(n, k) <= .025) k += 1;
+	return k === 0 ? [sorted[0], sorted[n - 1]] : [sorted[k - 1], sorted[n - k]];
+};
+const middle = (sorted) => {
+	const half = Math.floor(sorted.length / 2);
+	return sorted.length % 2 === 1 ? sorted[half] : (sorted[half - 1] + sorted[half]) / 2;
+};
+/**
+* This function gives the verdict of a cell, from two tests:
+* - Containment: each run lands in [T_lo − δ_abs, T_hi + δ_abs]. A run outside is a finding.
+* - Median equivalence: the interval of the median is in the band, made wider by δ_rel. An
+*   interval outside the band fails. An interval across its edge extends the cell to 22 runs, then
+*   44, and then the verdict is inconclusive.
+*
+* A silent prediction holds only if no run detected the fault in its window. After a stall, the
+* return of the reply gets the same test as a detection, by its interval, and a detection is a
+* finding. The distribution test (a DKW band) needs a predicted distribution from a simulator,
+* which does not exist at this time.
+*/
+const verdictOf = (prediction, runs) => {
+	const counted = runs.filter((run) => run.void === null);
+	const voided = runs.length - counted.length;
+	const outside = counted.filter((run) => run.finding !== null).map((run) => ({
+		run: name(run),
+		why: run.finding
+	}));
+	const empty = {
+		counted: counted.length,
+		voided,
+		median: null,
+		medianInterval: null
+	};
+	if (counted.length === 0) return {
+		...empty,
+		decision: "inconclusive",
+		band: null,
+		outside,
+		why: "no run counted"
+	};
+	if (prediction.detects === null) {
+		const samples = counted.flatMap((run) => (run.samples ?? []).map((sample) => valueIn(sample, "s"))).sort((a, b) => a - b);
+		const retransmissions = counted.reduce((sum, run) => sum + run.retransmissions, 0);
+		const ended = counted.filter((run) => run.outcome.kind === "error" || run.outcome.kind === "eof");
+		const why = samples.length === 0 ? `${counted.length} runs measured, with no reply: ${prediction.why}` : `${counted.length} runs, ${samples.length} replies: median ${s(middle(samples)).value} s, slowest ${s(samples.at(-1)).value} s, ${retransmissions} retransmissions, ${ended.length} connections ended. ${prediction.why}`;
+		return {
+			...empty,
+			decision: "measured",
+			band: null,
+			outside: [],
+			median: samples.length === 0 ? null : s(middle(samples)),
+			why
+		};
+	}
+	if (!prediction.detects && prediction.recovers === void 0) {
+		const decision = outside.length === 0 ? "holds" : "fails";
+		const why = decision === "holds" ? `nothing detected it in ${counted.length} runs of ${valueIn(prediction.window, "s")} s` : `${outside.length} of ${counted.length} runs detected it`;
+		return {
+			...empty,
+			decision,
+			band: null,
+			outside,
+			why
+		};
+	}
+	const timed = prediction.detects ? prediction : prediction.recovers;
+	const noise = valueIn(NOISE, "s");
+	const low = valueIn(timed.low, "s");
+	const high = valueIn(timed.high, "s");
+	const band = {
+		low: s(low - noise),
+		high: s(high + noise)
+	};
+	const values = [];
+	for (const run of counted) {
+		if (run.elapsed === null) continue;
+		const value = valueIn(run.elapsed, "s");
+		values.push(value);
+		if (value < low - noise || value > high + noise) outside.push({
+			run: name(run),
+			why: `${value} s is outside [${band.low.value}, ${band.high.value}] s`
+		});
+	}
+	values.sort((a, b) => a - b);
+	if (values.length === 0) return {
+		...empty,
+		decision: "fails",
+		band,
+		outside,
+		why: "no run measured its time"
+	};
+	const [from, to] = medianInterval(values);
+	const median = s(middle(values));
+	const interval = {
+		low: s(from),
+		high: s(to)
+	};
+	const wide = {
+		low: low * .95,
+		high: high * 1.05
+	};
+	const inside = from >= wide.low && to <= wide.high;
+	const disjoint = to < wide.low || from > wide.high;
+	const counts = `${counted.length - outside.length} of ${counted.length} runs inside [${band.low.value}, ${band.high.value}] s`;
+	const centre = `median ${median.value} s, its interval [${interval.low.value}, ${interval.high.value}] s`;
+	const result = {
+		counted: counted.length,
+		voided,
+		band,
+		outside,
+		median,
+		medianInterval: interval
+	};
+	if (outside.length > 0 || disjoint) return {
+		...result,
+		decision: "fails",
+		why: `${counts}; ${centre}`
+	};
+	if (inside) return {
+		...result,
+		decision: "holds",
+		why: `${counts}; ${centre}`
+	};
+	const decision = counted.length < STEPS[2] ? "extend" : "inconclusive";
+	return {
+		...result,
+		decision,
+		why: `${counts}; ${centre} straddles the band`
+	};
+};
+//#endregion
+//#region src/lab.ts
+/**
+* These are the roots of the two packages that the images read: this package, and the prober
+* that Node resolves for it. In a workspace, they are two folders side by side. From npm, the
+* prober is a link, and a Docker build context does not follow a link. Thus each image builds
+* from a context that the harness copies from the two roots.
+*/
+const ROOTS = {
+	harness: fileURLToPath(new URL("..", import.meta.url)),
+	prober: dirname(realpathSync(createRequire(import.meta.url).resolve("@mark1russell7/prober/package.json")))
+};
+/** This function gives the path of a part of an image, which is `<package>/<path>` in the context. */
+const pathOf = (part) => {
+	const [root, ...rest] = part.split("/");
+	const base = ROOTS[root];
+	if (base === void 0) throw new Error(`${part} is in no package that the images read`);
+	return join(base, ...rest);
+};
+/** This function loads Testcontainers when a lab first opens. Code that only measures and judges does not load it. */
+const containers = () => import("testcontainers");
+/** This image holds the prober, the TCP client, the injector and the namespace admin. */
+const HARNESS = {
+	name: "loom-harness",
+	dockerfile: "harness/image/Dockerfile",
+	from: [
+		"harness/image/Dockerfile",
+		"harness/image/native",
+		"prober/bpf",
+		"prober/native",
+		"prober/fixtures"
+	]
+};
+/** This image holds the Node client: ioredis 6.0.0 on Node 24.21.0. */
+const NODE = {
+	name: "loom-harness-node",
+	dockerfile: "harness/image/node/Dockerfile",
+	from: ["harness/image/node"]
+};
+/** This function gives each file under a path, in a stable sequence, but not the output of a local build. */
+const sources = (path) => {
+	if (statSync(path).isFile()) return [path];
+	return readdirSync(path, { withFileTypes: true }).filter((entry) => entry.name !== "target" && entry.name !== "node_modules").flatMap((entry) => sources(join(path, entry.name))).sort();
+};
+/** This function gives each file of an image, with its path in the build context. */
+const filesOf = (image) => image.from.flatMap((part) => {
+	const base = pathOf(part);
+	return sources(base).map((file) => ({
+		file,
+		at: join(part, relative(base, file)).replaceAll("\\", "/")
+	}));
+});
+/** This function gives the tag of an image from its content. Thus a changed probe or client cannot operate as an old one. */
+const imageTag = (image) => {
+	const hash = createHash("sha256");
+	for (const { file, at } of filesOf(image)) {
+		hash.update(at);
+		hash.update(readFileSync(file));
+	}
+	return `${image.name}:${hash.digest("hex").slice(0, 12)}`;
+};
+/** This function copies the files of an image into a new build context, and gives its path. */
+const stage = (image) => {
+	const context = mkdtempSync(join(tmpdir(), `${image.name}-`));
+	for (const { file, at } of filesOf(image)) {
+		mkdirSync(dirname(join(context, at)), { recursive: true });
+		copyFileSync(file, join(context, at));
+	}
+	return context;
+};
+const built = /* @__PURE__ */ new Map();
+/** This function builds an image one time for each process. Docker keeps its layers between runs. */
+const build = (image) => {
+	let tag = built.get(image.name);
+	if (tag === void 0) {
+		tag = (async () => {
+			const name = imageTag(image);
+			const { GenericContainer } = await containers();
+			const context = stage(image);
+			try {
+				await GenericContainer.fromDockerfile(context, image.dockerfile).build(name, { deleteOnExit: false });
+			} finally {
+				rmSync(context, {
+					recursive: true,
+					force: true
+				});
+			}
+			return name;
+		})();
+		built.set(image.name, tag);
+	}
+	return tag;
+};
+/** This function gives the image of the harness. */
+const harnessImage = () => build(HARNESS);
+/** This function gives the image of the Node client. */
+const nodeClientImage = () => build(NODE);
+/** This function gives the arguments of the injector for the options of a fault. */
+const optionArgs = (options) => [
+	...options.direction === void 0 ? [] : ["--direction", options.direction],
+	...options.code === void 0 ? [] : ["--code", options.code],
+	...options.above === void 0 ? [] : ["--above", String(options.above)]
+];
+/** These are the actions of the injector that lift each network fault. */
+const LIFT = {
+	blackhole: "clear",
+	rst: "clear-rst",
+	reject: "clear-reject",
+	mtu: "clear-mtu"
+};
+const lines = () => {
+	const seen = [];
+	const other = [];
+	const waiting = [];
+	let partial = "";
+	const take = (text) => {
+		if (!text.startsWith("{")) {
+			if (text.length > 0) other.push(text);
+			return;
+		}
+		const line = JSON.parse(text);
+		seen.push(line);
+		for (const wait of waiting.splice(0)) if (wait.kinds.includes(line.ev)) wait.resolve(line);
+		else waiting.push(wait);
+	};
+	return {
+		seen,
+		other,
+		consume: (stream) => stream.on("data", (chunk) => {
+			partial += chunk.toString();
+			const split = partial.split("\n");
+			partial = split.pop() ?? "";
+			for (const text of split) take(text.trim());
+		}),
+		until(kinds, within, who) {
+			const found = seen.find((line) => kinds.includes(line.ev));
+			if (found !== void 0) return Promise.resolve(found);
+			return new Promise((resolve, reject) => {
+				const timer = setTimeout(() => {
+					const said = other.length === 0 ? "" : `; it wrote: ${other.slice(-3).join(" | ")}`;
+					reject(/* @__PURE__ */ new Error(`no ${kinds.join(" or ")} from the ${who} in ${within} ms${said}`));
+				}, within);
+				waiting.push({
+					kinds,
+					resolve: (line) => {
+						clearTimeout(timer);
+						resolve(line);
+					}
+				});
+			});
+		}
+	};
+};
+const STOP = { timeout: 0 };
+const output = async (container, argv) => {
+	const result = await container.exec(argv);
+	if (result.exitCode !== 0) throw new Error(`${argv.join(" ")} exited ${result.exitCode}: ${result.output.trim()}`);
+	return result.stdout.trim();
+};
+/** This function gives the inode of the network namespace of a container: net:[4026532583]. */
+const netnsOf = async (container) => {
+	const link = await output(container, ["readlink", "/proc/self/ns/net"]);
+	const inode = /^net:\[(\d+)\]$/.exec(link)?.[1];
+	if (inode === void 0) throw new Error(`not a network namespace: ${link}`);
+	return Number(inode);
+};
+/**
+* This function opens one environment:
+* - a new network and the stock target.
+* - a client that holds its own network namespace, with a privileged sidecar in it for its
+*   sysctls. Docker Desktop keeps /proc/sys read-only in a container that operates.
+* - an injector sidecar in the namespace of the target.
+* - the prober, a privileged container that loads the kernel program and watches the two
+*   namespaces.
+*
+* The client of each run is its own container in the namespace of the client. It is the TCP
+* client of the harness, or a driver from its own image (ioredis on Node). The two print the same
+* lines.
+*/
+const openLab = async (target, kind = "tcp") => {
+	const image = await harnessImage();
+	const clientImage = kind === "ioredis" ? await nodeClientImage() : image;
+	const command = kind === "ioredis" ? ["node", "/opt/harness/node/client-ioredis.mjs"] : ["/opt/harness/client"];
+	const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();
+	const opened = [];
+	let network;
+	const close = async () => {
+		for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
+		await network?.stop().catch(() => void 0);
+	};
+	try {
+		network = await new Network().start();
+		const stock = new GenericContainer(target.image).withNetwork(network).withWaitStrategy(Wait.forLogMessage(target.ready));
+		const server = await (target.command === void 0 ? stock : stock.withCommand([...target.command])).start();
+		opened.push(server);
+		const client = await new GenericContainer(image).withNetwork(network).start();
+		opened.push(client);
+		const admin = await new GenericContainer(image).withNetworkMode(`container:${client.getId()}`).withPrivilegedMode().start();
+		opened.push(admin);
+		const injector = await new GenericContainer(image).withNetworkMode(`container:${server.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW").start();
+		opened.push(injector);
+		const netns = {
+			client: await netnsOf(admin),
+			target: await netnsOf(injector)
+		};
+		const records = lines();
+		const prober = await new GenericContainer(image).withPrivilegedMode().withNetworkMode("none").withCommand([
+			"/opt/harness/prober",
+			String(netns.client),
+			String(netns.target)
+		]).withLogConsumer(records.consume).start();
+		opened.push(prober);
+		const begun = await records.until(["begin", "fail"], 3e4, "prober");
+		if (begun.ev === "fail") throw new Error(`the prober failed: ${String(begun["why"])}`);
+		const targetIp = server.getIpAddress(network.getName());
+		const clientIp = client.getIpAddress(network.getName());
+		const runtime = await getContainerRuntimeClient();
+		const paused = runtime.container.getById(server.getId());
+		let targetAdmin;
+		return {
+			targetIp,
+			clientIp,
+			netns,
+			kernel: records.seen,
+			async sysctl(values) {
+				const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
+				if (settings.length > 0) await output(admin, [
+					"sysctl",
+					"-q",
+					"-w",
+					...settings
+				]);
+				const read = {};
+				for (const key of Object.keys(values)) read[key] = await output(admin, [
+					"sysctl",
+					"-n",
+					key
+				]);
+				return read;
+			},
+			async targetSysctl(values) {
+				const read = {};
+				if (Object.keys(values).length === 0) return read;
+				targetAdmin ??= await new GenericContainer(image).withNetworkMode(`container:${server.getId()}`).withPrivilegedMode().start();
+				opened.push(targetAdmin);
+				const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
+				await output(targetAdmin, [
+					"sysctl",
+					"-q",
+					"-w",
+					...settings
+				]);
+				for (const key of Object.keys(values)) read[key] = await output(targetAdmin, [
+					"sysctl",
+					"-n",
+					key
+				]);
+				return read;
+			},
+			async configure(values) {
+				const read = {};
+				if (Object.keys(values).length === 0) return read;
+				const how = target.settings;
+				if (how === void 0) throw new Error(`${target.image} names no way to set its settings`);
+				for (const [name, value] of Object.entries(values)) {
+					await output(server, [
+						...how.set,
+						name,
+						value
+					]);
+					read[name] = (await output(server, [...how.get, name])).split("\n").at(-1) ?? "";
+				}
+				return read;
+			},
+			async fault(kind, at = "target", options = {}) {
+				const [where, peer] = at === "client" ? [admin, targetIp] : [injector, clientIp];
+				const args = [
+					"/opt/harness/inject",
+					kind,
+					peer,
+					...optionArgs(options)
+				];
+				const stamp = JSON.parse(await output(where, args));
+				return {
+					t0: stamp.t0,
+					t1: stamp.t1
+				};
+			},
+			async lift(kind, at = "target", options = {}) {
+				const [where, peer] = at === "client" ? [admin, targetIp] : [injector, clientIp];
+				await output(where, [
+					"/opt/harness/inject",
+					LIFT[kind],
+					peer,
+					...optionArgs(options)
+				]);
+			},
+			async netem(spec, at = "target") {
+				const where = at === "client" ? admin : injector;
+				const args = spec === null ? [
+					"/opt/harness/inject",
+					"clear-netem",
+					"eth0"
+				] : [
+					"/opt/harness/inject",
+					"netem",
+					"eth0",
+					...spec.split(/\s+/)
+				];
+				const stamp = JSON.parse(await output(where, args));
+				return {
+					t0: stamp.t0,
+					t1: stamp.t1
+				};
+			},
+			async offload(on, at = "target") {
+				await output(at === "client" ? admin : injector, [
+					"/opt/harness/inject",
+					"offload",
+					"eth0",
+					on ? "on" : "off"
+				]);
+			},
+			async kill() {
+				if (target.kill === void 0) throw new Error(`${target.image} names no way to kill its server`);
+				await output(server, [...target.kill]);
+			},
+			async warm() {
+				await output(admin, [
+					"ping",
+					"-c",
+					"2",
+					"-W",
+					"2",
+					targetIp
+				]);
+				await output(injector, [
+					"ping",
+					"-c",
+					"2",
+					"-W",
+					"2",
+					clientIp
+				]);
+			},
+			async setup(commands) {
+				for (const argv of commands) await output(server, [...argv]);
+			},
+			async heal(at, when) {
+				const [where, peer] = at === "client" ? [admin, targetIp] : [injector, clientIp];
+				const stamp = JSON.parse(await output(where, [
+					"/opt/harness/inject",
+					"clear",
+					peer,
+					"--at",
+					String(Math.round(when))
+				]));
+				return {
+					t0: stamp.t0,
+					t1: stamp.t1
+				};
+			},
+			async forget() {
+				await output(admin, [
+					"ip",
+					"tcp_metrics",
+					"flush",
+					"all"
+				]);
+				await output(injector, [
+					"ip",
+					"tcp_metrics",
+					"flush",
+					"all"
+				]);
+			},
+			freeze: () => paused.pause().then(() => void 0),
+			thaw: () => paused.unpause().then(() => void 0),
+			async probe(args) {
+				const events = lines();
+				const container = await new GenericContainer(clientImage).withNetworkMode(`container:${client.getId()}`).withCommand([...command, ...args]).withLogConsumer(events.consume).start();
+				return {
+					events: events.seen,
+					until: (kinds, within) => events.until(kinds, within, "client"),
+					go: () => output(container, ["touch", "/tmp/go"]).then(() => void 0),
+					freeze: () => runtime.container.getById(container.getId()).pause().then(() => void 0),
+					thaw: () => runtime.container.getById(container.getId()).unpause().then(() => void 0),
+					kill: () => runtime.container.getById(container.getId()).kill({ signal: "SIGKILL" }).then(() => void 0),
+					stop: () => container.stop(STOP).then(() => void 0).catch(() => void 0)
+				};
+			},
+			async settle() {
+				const after = records.seen.length;
+				const clock = await new Promise((resolve) => {
+					const started = Date.now();
+					const look = () => {
+						const line = records.seen.slice(after).find((record) => record.ev === "clock");
+						if (line !== void 0) resolve(line);
+						else if (Date.now() - started > 5e3) resolve(null);
+						else setTimeout(look, 50);
+					};
+					look();
+				});
+				return clock === null ? null : Number(clock["lost"] ?? 0);
+			},
+			async manifest(sysctls, settings = {}, targetSysctls = {}) {
+				const config = await output(admin, ["zcat", "/proc/config.gz"]).catch(() => "");
+				const inspect = async (container) => runtime.container.inspect(runtime.container.getById(container.getId()));
+				return {
+					kernel: await output(client, ["uname", "-r"]),
+					architecture: await output(client, ["uname", "-m"]),
+					hz: /^CONFIG_HZ=(\d+)$/m.exec(config)?.[1] ?? null,
+					boot_id: await output(client, ["cat", "/proc/sys/kernel/random/boot_id"]),
+					docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
+					target: {
+						image: target.image,
+						id: (await inspect(server)).Image,
+						settings,
+						sysctls: targetSysctls
+					},
+					harness: {
+						image,
+						id: (await inspect(client)).Image,
+						packages: (await output(client, [
+							"apk",
+							"info",
+							"-v"
+						])).split("\n").sort()
+					},
+					client: {
+						kind,
+						image: clientImage
+					},
+					netns,
+					sysctls
+				};
+			},
+			close
+		};
+	} catch (error) {
+		await close();
+		throw error;
+	}
+};
+//#endregion
+//#region src/run.ts
+/**
+* This function measures and judges the raw records of a cell, with the code of the harness at
+* this time. `runCell` uses it during a cell. A holder of the raw records of a cell can use it
+* again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.
+*/
+const judge = (cell, runs) => {
+	const measured = runs.map((run) => measure(cell, run));
+	return {
+		measured,
+		verdict: verdictOf(cell.prediction, measured)
+	};
+};
+/** This function gives the count of lost records in the latest clock line of the prober, since it began. */
+const droppedSoFar = (records) => {
+	for (let at = records.length - 1; at >= 0; at -= 1) if (records[at].ev === "clock") return Number(records[at]["lost"] ?? 0);
+	return 0;
+};
+/** This function gives the argv of the client for a cell, with a deadline far after all that it waits for. */
+const probeArgs = (cell, lab) => {
+	const { prediction, probe } = cell;
+	const longest = prediction.detects ? valueIn(prediction.high, "s") : valueIn(prediction.window, "s");
+	const args = [
+		lab.targetIp,
+		String(cell.target.port),
+		"--mode",
+		probe.mode
+	];
+	if (probe.request !== void 0) args.push("--request", probe.request);
+	if (probe.count !== void 0) args.push("--count", String(probe.count));
+	if (probe.interval !== void 0) args.push("--interval-ms", String(valueIn(probe.interval, "ms")));
+	if (probe.driver !== void 0) {
+		args.push("--keepalive-ms", String(valueIn(probe.driver.keepAlive, "ms")));
+		args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
+		return args;
+	}
+	if (probe.keepalive) args.push("--keepalive");
+	if (probe.userTimeout !== void 0) args.push("--user-timeout-ms", String(valueIn(probe.userTimeout, "ms")));
+	args.push("--max-s", String(Math.ceil(longest * 2 + 10)));
+	return args;
+};
+/**
+* This function waits for an error on the socket of the target for the connection of the
+* client, or for its end of file (CLOSE_WAIT, state 8). The prober records both in the namespace
+* of the target. A cell on the server side waits for that. The promise resolves after `within` ms
+* at the latest.
+*/
+const serverError = (lab, local, within) => new Promise((resolve) => {
+	const started = Date.now();
+	const look = () => {
+		if (lab.kernel.some((record) => record.netns === lab.netns.target && record.dst === local && (record.ev === "error" && (record.err ?? 0) !== 0 || record.ev === "state" && record.new === 8)) || Date.now() - started > within) resolve();
+		else setTimeout(look, 50);
+	};
+	look();
+});
+/**
+* This function waits for the request that an in-flight client sends on the signal: its second
+* send, after the PING that made it ready. Its stamp is when the request first met the fault.
+*/
+const requestSent = (probe, within) => new Promise((resolve, reject) => {
+	const started = Date.now();
+	const look = () => {
+		const sent = probe.events.filter((event) => event.ev === "sent")[1];
+		if (sent !== void 0) resolve(sent);
+		else if (Date.now() - started > within) reject(/* @__PURE__ */ new Error(`the client sent no request within ${within} ms of being told to`));
+		else setTimeout(look, 1);
+	};
+	look();
+});
+/**
+* This function does one run. The client connects and gets its reply, the fault occurs, and an
+* in-flight client sends into it. The harness then waits for the observer to see the fault, or for
+* the silent window to end. The observer is the client, or the socket of the target for a cell on
+* the server side.
+*
+* A stall stops by itself, at its length after the send of the request, and the
+* run waits for the reply. Then the harness removes the fault, thus the environment can serve the
+* next run. The run keeps the records of the prober from its start to its end. The frozen process
+* is the peer of the observer: the target, or the client for the server.
+*/
+/** This function gives the netem specification of a cell: its latency, then the netem of its fault when it has one. */
+const netemSpec = (cell, fault) => {
+	const parts = [...cell.latency === void 0 ? [] : [`delay ${valueIn(cell.latency, "ms")}ms`], ...fault === void 0 ? [] : [fault]];
+	return parts.length === 0 ? null : parts.join(" ");
+};
+const once = async (lab, cell, environment, index) => {
+	await lab.forget();
+	const from = lab.kernel.length;
+	const lostBefore = droppedSoFar(lab.kernel);
+	const probe = await lab.probe(probeArgs(cell, lab));
+	const server = cell.observer === "server";
+	const peer = server ? {
+		freeze: () => probe.freeze(),
+		thaw: () => probe.thaw()
+	} : {
+		freeze: () => lab.freeze(),
+		thaw: () => lab.thaw()
+	};
+	const go = () => probe.go().catch((error) => {
+		const last = probe.events.slice(-3).map((event) => JSON.stringify(event));
+		throw new Error(`${error.message}; the client's last lines: ${last.join(" ")}`);
+	});
+	let fault;
+	let heal = null;
+	try {
+		await probe.until(["ready"], 15e3);
+		const replyInFlight = cell.probe.mode === "inflight" && (server && cell.fault !== "mtu-blackhole" || cell.fault === "fin");
+		if (replyInFlight) {
+			await go();
+			await new Promise((resolve) => setTimeout(resolve, 200));
+		}
+		const faultAt = server ? "client" : "target";
+		const direction = cell.drops === void 0 ? void 0 : cell.drops === "to-server" === (faultAt === "target") ? "in" : "out";
+		const options = {
+			...direction === void 0 ? {} : { direction },
+			...cell.icmp === void 0 ? {} : { code: cell.icmp },
+			...cell.mtu === void 0 ? {} : { above: cell.mtu }
+		};
+		const apply = async () => {
+			switch (cell.fault) {
+				case "freeze-process":
+					await peer.freeze();
+					return null;
+				case "fin":
+					await (server ? probe.kill() : lab.kill());
+					return null;
+				case "unreachable": return lab.fault("reject", faultAt, { code: cell.icmp ?? "host" });
+				case "simplex": return lab.fault("blackhole", faultAt, options);
+				case "mtu-blackhole": return lab.fault("mtu", "client", { above: cell.mtu ?? 1200 });
+				case "netem": return lab.netem(netemSpec(cell, cell.netem), "target");
+				case "rst": return lab.fault("rst", faultAt);
+				default: return lab.fault("blackhole", faultAt);
+			}
+		};
+		const lift = async () => {
+			switch (cell.fault) {
+				case "freeze-process": return peer.thaw();
+				case "fin": return;
+				case "unreachable": return lab.lift("reject", faultAt, { code: cell.icmp ?? "host" });
+				case "simplex": return lab.lift("blackhole", faultAt, options);
+				case "mtu-blackhole": return lab.lift("mtu", "client", { above: cell.mtu ?? 1200 });
+				case "netem":
+					await lab.netem(netemSpec(cell, void 0), "target");
+					return;
+				case "rst": return lab.lift("rst", faultAt);
+				default: return heal === null ? lab.lift("blackhole", faultAt) : void 0;
+			}
+		};
+		fault = await apply();
+		try {
+			if (cell.probe.mode !== "idle" && !replyInFlight) await go();
+			const { prediction } = cell;
+			if (cell.fault === "stall") {
+				if (cell.stall === void 0) throw new Error(`${cell.name} says no stall's length`);
+				const sent = await requestSent(probe, 5e3);
+				heal = await lab.heal(faultAt, sent.t + valueIn(cell.stall, "s") * 1e9);
+				const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
+				await probe.until([
+					"data",
+					"error",
+					"eof"
+				], window * 1e3).catch(() => void 0);
+			} else if (prediction.detects === null) {
+				const window = valueIn(prediction.window, "s") * 1e3;
+				await probe.until([
+					"done",
+					"error",
+					"eof",
+					"deadline"
+				], window).catch(() => void 0);
+			} else if (prediction.detects) {
+				const within = valueIn(prediction.high, "s") * 2e3 + 1e4;
+				if (server) {
+					const local = probe.events.find((event) => event.ev === "connected")?.["local"];
+					await serverError(lab, local, within);
+				} else await probe.until([
+					"error",
+					"eof",
+					"deadline"
+				], within).catch(() => void 0);
+			} else await new Promise((resolve) => setTimeout(resolve, valueIn(prediction.window, "s") * 1e3));
+		} finally {
+			await lift();
+		}
+		if (cell.fault === "freeze-process" && !server) await probe.until(["data"], 5e3).catch(() => void 0);
+	} finally {
+		await probe.stop();
+	}
+	const settled = await lab.settle();
+	const lost = settled === null ? null : settled - lostBefore;
+	return {
+		environment,
+		index,
+		events: [...probe.events],
+		fault,
+		...cell.fault === "stall" ? { heal } : {},
+		kernel: lab.kernel.slice(from),
+		netns: lab.netns.client,
+		targetNetns: lab.netns.target,
+		lost
+	};
+};
+/** This function divides a count of runs across environments, as equally as possible. */
+const spread = (runs, environments) => Array.from({ length: environments }, (_, at) => Math.floor(runs / environments) + (at < runs % environments ? 1 : 0)).filter((count) => count > 0);
+/** This function does the part of the runs of a cell in one environment, and records its manifest. */
+const inEnvironment = async (cell, environment, count, say) => {
+	const lab = await openLab(cell.target, cell.probe.driver?.name ?? "tcp");
+	try {
+		const settings = await lab.configure(cell.targetSettings ?? {});
+		const targetSysctls = await lab.targetSysctl(cell.targetSysctls ?? {});
+		if (cell.targetSetup !== void 0) await lab.setup(cell.targetSetup);
+		if (cell.fault === "mtu-blackhole" || cell.fault === "netem") {
+			await lab.offload(false, "target");
+			await lab.offload(false, "client");
+		}
+		const latency = netemSpec(cell, void 0);
+		if (latency !== null) await lab.netem(latency, "target");
+		await lab.warm();
+		const manifest = await lab.manifest(await lab.sysctl(cell.sysctls), settings, targetSysctls);
+		const runs = [];
+		for (let index = 0; index < count; index += 1) {
+			const run = await once(lab, cell, environment, index);
+			runs.push(run);
+			const read = measure(cell, run);
+			say(`${cell.name}: env ${environment} run ${index} ${read.outcome.kind}${read.elapsed === null ? "" : ` ${read.elapsed.value} s`}${read.void === null ? "" : ` (void: ${read.void})`}`);
+		}
+		return {
+			manifest,
+			runs
+		};
+	} finally {
+		await lab.close();
+	}
+};
+/**
+* This function does the runs of a cell. They go across new environments, and each environment records
+* its manifest. Then the verdict judges the runs. A median across the edge of the band extends
+* the cell from 11 runs to 22, and then to 44, each extension in new environments. With a
+* concurrency of more than one, that many environments operate at the same time, each with its
+* own prober. The runs then go across at least as many environments.
+*/
+const runCell = async (cell, say = () => {}) => {
+	const started = (/* @__PURE__ */ new Date()).toISOString();
+	const manifests = [];
+	const runs = [];
+	const concurrency = Math.max(1, cell.concurrency ?? 1);
+	let target = cell.runs;
+	let next = 0;
+	for (;;) {
+		const counts = spread(target - runs.length, Math.max(cell.environments, concurrency));
+		for (let at = 0; at < counts.length; at += concurrency) {
+			const batch = counts.slice(at, at + concurrency).map((count) => ({
+				count,
+				env: next++
+			}));
+			const done = await Promise.all(batch.map(({ count, env }) => inEnvironment(cell, env, count, say)));
+			for (const each of done) {
+				manifests.push(each.manifest);
+				runs.push(...each.runs);
+			}
+		}
+		const { measured, verdict } = judge(cell, runs);
+		const larger = STEPS.find((step) => step > runs.length);
+		if (verdict.decision !== "extend" || larger === void 0) return {
+			cell,
+			manifests,
+			runs,
+			measured,
+			verdict,
+			at: {
+				started,
+				ended: (/* @__PURE__ */ new Date()).toISOString()
+			}
+		};
+		target = larger;
+	}
+};
+//#endregion
+export { NOISE, RELATIVE, STEPS, harnessImage, judge, measure, medianInterval, nodeClientImage, openLab, runCell, spread, verdictOf };
+
+//# sourceMappingURL=index.js.map