@mark1russell7/harness 0.3.0 → 0.4.0

package/build/index.js

@@ -24,6 +24,55 @@ const STEPS = [
 ];
 const s = (value) => quantity(Math.round(value * 1e6) / 1e6, "s");
 const name = (run) => `env ${run.environment} run ${run.index}`;
+/** This function gives the value at quantile p of sorted values, by nearest rank: x_(⌈p·n⌉). */
+const quantileOf = (sorted, p) => sorted[Math.min(sorted.length, Math.max(1, Math.ceil(p * sorted.length))) - 1];
+/** This function gives the spread of a set of times, or null for none. */
+const spreadOf = (times) => {
+	if (times.length === 0) return null;
+	const sorted = times.map((time) => valueIn(time, "s")).sort((a, b) => a - b);
+	return {
+		count: sorted.length,
+		median: s(middle(sorted)),
+		p99: s(quantileOf(sorted, .99)),
+		p999: s(quantileOf(sorted, .999)),
+		max: s(sorted.at(-1))
+	};
+};
+/** This function adds the reordering records of runs: the counts add, and each largest value is the largest of all. */
+const reorderingTotal = (runs) => {
+	const each = runs.flatMap((run) => run.reordering === void 0 ? [] : [run.reordering]);
+	if (each.length === 0) return void 0;
+	const sum = (key) => each.reduce((total, one) => total + one[key], 0);
+	const add = (key) => each.some((one) => one[key] !== null) ? each.reduce((total, one) => total + (one[key] ?? 0), 0) : null;
+	const top = (key) => each.some((one) => one[key] !== null) ? Math.max(...each.map((one) => one[key] ?? 0)) : null;
+	return {
+		outOfOrder: sum("outOfOrder"),
+		duplicates: sum("duplicates"),
+		ooo: add("ooo"),
+		sacked: top("sacked"),
+		lost: top("lost"),
+		dsacks: add("dsacks"),
+		fast: sum("fast"),
+		degree: top("degree"),
+		seen: add("seen")
+	};
+};
+/** This function says a reordering record in words. */
+const reorderingWords = (total) => [
+	`${total.outOfOrder} arrivals out of order and ${total.duplicates} duplicate arrivals at the client`,
+	...total.ooo === null ? [] : [`its kernel counted ${total.ooo} segments out of order`],
+	...total.sacked === null ? [] : [`the server held up to ${total.sacked} segments SACKed`],
+	...total.dsacks === null ? [] : [`got ${total.dsacks} DSACKs`],
+	`sent ${total.fast} fast retransmissions`,
+	...total.seen === null ? [] : [`saw ${total.seen} reorderings`],
+	...total.degree === null ? [] : [`and its reordering degree went to ${total.degree}`]
+].join(", ");
+/** This function says a spread in words, in milliseconds. */
+const spreadWords = (spread) => {
+	if (spread === null) return "no time";
+	const ms = (time) => Math.round(valueIn(time, "s") * 1e6) / 1e3;
+	return `median ${ms(spread.median)} ms, p99 ${ms(spread.p99)} ms, p99.9 ${ms(spread.p999)} ms, max ${ms(spread.max)} ms of ${spread.count}`;
+};
 /** This function gives P(X ≤ j) for X ~ Binomial(n, 1/2), exactly. */
 const below = (n, j) => {
 	let term = 1;
@@ -82,6 +131,25 @@ const verdictOf = (prediction, runs) => {
 		why: "no run counted"
 	};
 	if (prediction.detects === null) {
+		const crossed = counted.flatMap((run) => run.crossings === void 0 ? [] : [run.crossings]);
+		if (crossed.length > 0) {
+			const hop = crossed[0].hop;
+			const up = spreadOf(crossed.flatMap((each) => each.up));
+			const down = spreadOf(crossed.flatMap((each) => each.down));
+			return {
+				...empty,
+				decision: "measured",
+				band: null,
+				outside: [],
+				median: up?.median ?? null,
+				proxy: {
+					hop,
+					up,
+					down
+				},
+				why: `${counted.length} runs measured ${hop}: toward the target ${spreadWords(up)}; toward the client ${spreadWords(down)}. ${prediction.why}`
+			};
+		}
 		if (prediction.from !== void 0) {
 			const times = counted.flatMap((run) => run.elapsed === null ? [] : [valueIn(run.elapsed, "s")]).sort((a, b) => a - b);
 			const kinds = /* @__PURE__ */ new Map();
@@ -103,13 +171,24 @@ const verdictOf = (prediction, runs) => {
 		const samples = counted.flatMap((run) => (run.samples ?? []).map((sample) => valueIn(sample, "s"))).sort((a, b) => a - b);
 		const retransmissions = counted.reduce((sum, run) => sum + run.retransmissions, 0);
 		const ended = counted.filter((run) => run.outcome.kind === "error" || run.outcome.kind === "eof");
-		const why = samples.length === 0 ? `${counted.length} runs measured, with no reply: ${prediction.why}` : `${counted.length} runs, ${samples.length} replies: median ${s(middle(samples)).value} s, slowest ${s(samples.at(-1)).value} s, ${retransmissions} retransmissions, ${ended.length} connections ended. ${prediction.why}`;
+		const reordering = reorderingTotal(counted);
+		const reordered = reordering !== void 0 && [
+			reordering.outOfOrder,
+			reordering.duplicates,
+			reordering.ooo,
+			reordering.sacked,
+			reordering.dsacks,
+			reordering.fast,
+			reordering.seen
+		].some((count) => (count ?? 0) > 0) ? ` Reordering: ${reorderingWords(reordering)}.` : "";
+		const why = samples.length === 0 ? `${counted.length} runs measured, with no reply: ${prediction.why}` : `${counted.length} runs, ${samples.length} replies: median ${s(middle(samples)).value} s, p99 ${s(quantileOf(samples, .99)).value} s, slowest ${s(samples.at(-1)).value} s, ${retransmissions} retransmissions, ${ended.length} connections ended.${reordered} ${prediction.why}`;
 		return {
 			...empty,
 			decision: "measured",
 			band: null,
 			outside: [],
 			median: samples.length === 0 ? null : s(middle(samples)),
+			...reordering === void 0 ? {} : { reordering },
 			why
 		};
 	}
@@ -277,6 +356,65 @@ const peerOf = (cell, run) => {
 		cookie
 	};
 };
+/** This function tells if sequence number a is after b, in the 32-bit space of TCP (`after` in include/net/tcp.h). */
+const after32 = (a, b) => (a - b | 0) > 0;
+/** This function gives the largest value of a count of the kernel in records, or null when no record has it. */
+const most = (records, count) => records.reduce((top, record) => {
+	const value = count(record);
+	return value === void 0 ? top : Math.max(top ?? 0, value);
+}, null);
+/** This function gives what reordering did in a run, from the records of the receiver and of the sender. */
+const reorderingOf = (receiver, sender) => {
+	const arrivals = receiver.filter((record) => record.ev === "rx" && (record.seg?.len.value ?? 0) > 0 && record.tcp?.rcv_nxt !== void 0);
+	return {
+		outOfOrder: arrivals.filter((record) => after32(record.seg.seq, record.tcp.rcv_nxt)).length,
+		duplicates: arrivals.filter((record) => !after32(record.seg.seq + record.seg.len.value, record.tcp.rcv_nxt)).length,
+		ooo: most(receiver, (record) => record.tcp?.rcv_ooopack),
+		sacked: most(sender, (record) => record.tcp?.sacked),
+		lost: most(sender, (record) => record.tcp?.lost),
+		dsacks: most(sender, (record) => record.tcp?.dsack_dups),
+		fast: sender.filter((record) => record.ev === "retrans" && record.why === "fast").length,
+		degree: most(sender, (record) => record.tcp?.reordering),
+		seen: most(sender, (record) => record.tcp?.reord_seen)
+	};
+};
+/**
+* This function gives the times of a proxy in a run, for each request after the first. The time
+* of a request starts at its arrival on the socket of the proxy toward the client. It stops at the
+* next write on the socket toward the target. The time of its reply starts at its arrival on the
+* socket toward the target, and it stops at the next write toward the client. A request without
+* each record gives no time.
+*/
+const crossingsOf = (run, hop, toClient, toTarget) => {
+	const up = [];
+	const down = [];
+	if (toClient === void 0 || toTarget === void 0) return {
+		hop,
+		up,
+		down
+	};
+	const of = (place) => run.kernel.filter((record) => record.netns === place.netns && record.cookie === place.cookie);
+	const near = of(toClient);
+	const far = of(toTarget);
+	const arrival = (records, from, to) => records.find((record) => record.ev === "rx" && (record.seg?.len.value ?? 0) > 0 && record.mono > from && record.mono < to);
+	const write = (records, from, to) => records.find((record) => record.ev === "tx" && record.mono >= from && record.mono < to);
+	const sends = run.events.filter((event) => event.ev === "sent").slice(1);
+	sends.forEach((send, n) => {
+		const next = sends[n + 1]?.t ?? Infinity;
+		const request = arrival(near, send.t, next);
+		const relayed = request === void 0 ? void 0 : write(far, request.mono, next);
+		if (request === void 0 || relayed === void 0) return;
+		up.push(between(request.mono, relayed.mono));
+		const reply = arrival(far, relayed.mono, next);
+		const answered = reply === void 0 ? void 0 : write(near, reply.mono, next);
+		if (reply !== void 0 && answered !== void 0) down.push(between(reply.mono, answered.mono));
+	});
+	return {
+		hop,
+		up,
+		down
+	};
+};
 /**
 * This function reads one run against its cell, from the records of the kernel first.
 * - The socket of the run is the socket of its observer. It is the socket that the client connected
@@ -321,12 +459,15 @@ const measure = (cell, run) => {
 		return firings.find((record) => timer === "keepalive" && record.owned === true) === void 0 ? null : "its keepalive timer found the socket held by its owner and put itself off by HZ/20: the run measured the owner, not the timer";
 	};
 	const acks = request === void 0 ? 0 : socket.filter((record) => record.ev === "rx" && record.mono > request.t).length;
+	const srtt = socket.find((record) => record.tcp?.srtt !== void 0)?.tcp?.srtt;
+	const rtt = srtt === void 0 ? null : quantity(valueIn(srtt, "us") / 1e3, "ms");
 	const base = {
 		environment: run.environment,
 		index: run.index,
 		probes: keepalives.length,
 		retransmissions: retransmitted.length,
-		acks
+		acks,
+		rtt
 	};
 	const nothing = {
 		outcome: { kind: "silent" },
@@ -370,6 +511,8 @@ const measure = (cell, run) => {
 		else if (replyFirst && replied) voided = "the server's reply arrived before the fault held";
 		else if (keepalives[0] !== void 0 && keepalives[0].mono < run.fault.t1) voided = "a keepalive probe went out before the fault held";
 	}
+	const assumed = cell.prediction.detects === null ? void 0 : cell.prediction.rtt;
+	if (voided === null && assumed !== void 0 && rtt !== null && valueIn(rtt, "ms") > valueIn(assumed.high, "ms")) voided = `rtt-over-assumption: its first round trip was ${rtt.value} ms, above the ${valueIn(assumed.high, "ms")} ms that the prediction assumes`;
 	const remote = cell.observer !== void 0 && cell.observer !== "client";
 	const reported = socket.find((record) => record.ev === "error" && (record.err ?? 0) !== 0);
 	const seen = remote ? void 0 : events.find((event) => event.ev === "error" || event.ev === "eof");
@@ -397,13 +540,29 @@ const measure = (cell, run) => {
 	};
 	if (prediction.detects === null) {
 		const sends = events.filter((event) => event.ev === "sent").slice(1);
+		const answers = events.some((event) => event.ev === "replied") ? "replied" : "data";
 		const samples = sends.flatMap((send, n) => {
 			const next = sends[n + 1]?.t ?? Infinity;
-			const reply = events.find((event) => event.ev === "data" && event.t > send.t && event.t < next);
+			const reply = events.find((event) => event.ev === answers && event.t > send.t && event.t < next);
 			return reply === void 0 ? [] : [between(send.t, reply.t)];
 		});
 		const from = prediction.from === void 0 ? void 0 : referenceAt(prediction.from);
 		const until = ended ?? after.at(-1)?.t;
+		const peerRecords = peerCookie === void 0 ? [] : kernel.filter((record) => record.netns === peerNetns && record.cookie === peerCookie);
+		const proxy = cell.calibrates === void 0 ? void 0 : run.topology?.hops[cell.calibrates];
+		const crossings = cell.calibrates === void 0 ? {} : { crossings: crossingsOf(run, cell.calibrates, proxy === void 0 ? void 0 : socketOf({
+			...cell,
+			observer: {
+				hop: cell.calibrates,
+				toward: "client"
+			}
+		}, run), proxy === void 0 ? void 0 : socketOf({
+			...cell,
+			observer: {
+				hop: cell.calibrates,
+				toward: "target"
+			}
+		}, run)) };
 		return {
 			...base,
 			...nothing,
@@ -413,6 +572,8 @@ const measure = (cell, run) => {
 			noticed: prediction.from === void 0 ? null : noticed,
 			steps: retransmitted.filter((record) => record.mono > request.t).map((record) => between(request.t, record.mono)),
 			samples,
+			reordering: reorderingOf(socket.filter((record) => record.mono > request.t), peerRecords.filter((record) => record.mono > request.t)),
+			...crossings,
 			void: voided
 		};
 	}
@@ -794,6 +955,12 @@ const clientOf = async (kind, image) => {
 		clientKind: kind
 	};
 };
+/** This function gives the device of a route from its line in `ip -o route get`: "10.0.0.2 dev eth0 src …". */
+const deviceOf = (route) => {
+	const device = /\bdev (\S+)/.exec(route)?.[1];
+	if (device === void 0) throw new Error(`no device in the route: ${route}`);
+	return device;
+};
 /**
 * This function opens one environment:
 * - a chain of new networks, and the stock target on the last one. A direct pair has one network.
@@ -871,10 +1038,12 @@ const openLab = async (target, kind = "tcp", topology) => {
 				await join(box, targetSide);
 				const ready = await seen.until(["mbx-ready"], 3e4, `router ${hop.name}`);
 				await seen.until(hop.form === "reset-both" ? ["resetter"] : ["ct-watch"], 3e4, `router ${hop.name}`);
+				const injector = await new GenericContainer(image).withNetworkMode(`container:${box.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW").start();
+				opened.push(injector);
 				inChain[at] = {
 					hop,
 					box,
-					injector: box,
+					injector,
 					ip: {
 						client: await ipOn(box, clientSide),
 						target: await ipOn(box, targetSide)
@@ -883,6 +1052,10 @@ const openLab = async (target, kind = "tcp", topology) => {
 						client: "",
 						target: endpointOf(at + 1)
 					},
+					devices: {
+						client: "",
+						target: ""
+					},
 					netns: await netnsOf(box),
 					sysctls: { "net.netfilter.nf_conntrack_tcp_timeout_established": String(ready["idle"]) }
 				};
@@ -913,6 +1086,10 @@ const openLab = async (target, kind = "tcp", topology) => {
 					client: "",
 					target: next
 				},
+				devices: {
+					client: "",
+					target: ""
+				},
 				netns: await netnsOf(injector),
 				sysctls: {}
 			};
@@ -961,7 +1138,21 @@ const openLab = async (target, kind = "tcp", topology) => {
 		}
 		for (let at = 0; at < inChain.length; at += 1) {
 			const hop = inChain[at];
-			if (hop.hop.kind !== "proxy" || hop.hop.sysctls === void 0) continue;
+			const toward = async (address) => deviceOf(await output(hop.injector, [
+				"ip",
+				"-o",
+				"route",
+				"get",
+				address.split(":")[0]
+			]));
+			inChain[at] = {
+				...hop,
+				devices: {
+					client: await toward(hop.peer.client),
+					target: await toward(hop.peer.target)
+				}
+			};
+			if (hop.hop.sysctls === void 0) continue;
 			const where = await adminOf(at + 1);
 			const settings = Object.entries(hop.hop.sysctls).map(([key, value]) => `${key}=${value}`);
 			await output(where, [
@@ -1010,13 +1201,38 @@ const openLab = async (target, kind = "tcp", topology) => {
 			if (found === void 0) throw new Error(`the topology has no hop ${name}`);
 			return found;
 		};
-		/** This function gives the namespace and the peer of the rules of a fault at a place. */
+		/**
+		* This function gives the namespace and the peer of the rules of a fault at a place, and the
+		* other arguments of the injector. At a proxy, the rules are INPUT and OUTPUT rules, which act
+		* on its sockets. A router forwards past INPUT and OUTPUT, thus its rules are FORWARD rules on
+		* its device toward the side (`--via`).
+		*/
 		const placeOf = (at) => {
-			if (at === "client") return [admin, targetIp];
-			if (at === "target") return [injector, targetPeer];
+			if (at === "client") return [
+				admin,
+				targetIp,
+				[]
+			];
+			if (at === "target") return [
+				injector,
+				targetPeer,
+				[]
+			];
 			const hop = hopNamed(at.hop);
-			if (hop.hop.kind !== "proxy") throw new Error(`a fault at ${at.hop} needs a proxy: a router forwards past INPUT and OUTPUT`);
-			return [hop.injector, at.toward === "client" ? hop.peer.client : hop.peer.target.split(":")[0]];
+			const peer = at.toward === "client" ? hop.peer.client : hop.peer.target.split(":")[0];
+			const via = hop.hop.kind === "router" ? ["--via", hop.devices[at.toward]] : [];
+			return [
+				hop.injector,
+				peer,
+				via
+			];
+		};
+		/** This function gives the namespace and the device of netem at a place: an end, or the device of a hop toward a side. */
+		const deviceAt = (at) => {
+			if (at === "client") return [admin, "eth0"];
+			if (at === "target") return [injector, "eth0"];
+			const hop = hopNamed(at.hop);
+			return [hop.injector, hop.devices[at.toward]];
 		};
 		const toxiproxy = () => {
 			const found = inChain.find((hop) => hop.hop.kind === "proxy" && hop.hop.api !== void 0);
@@ -1090,42 +1306,46 @@ const openLab = async (target, kind = "tcp", topology) => {
 				return read;
 			},
 			async fault(kind, at = "target", options = {}) {
-				const [where, peer] = placeOf(at);
+				const [where, peer, via] = placeOf(at);
 				return stamped(where, [
 					"/opt/harness/inject",
 					kind,
 					peer,
-					...optionArgs(options)
+					...optionArgs(options),
+					...via
 				]);
 			},
 			async lift(kind, at = "target", options = {}) {
-				const [where, peer] = placeOf(at);
+				const [where, peer, via] = placeOf(at);
 				await output(where, [
 					"/opt/harness/inject",
 					LIFT[kind],
 					peer,
-					...optionArgs(options)
+					...optionArgs(options),
+					...via
 				]);
 			},
 			async netem(spec, at = "target") {
-				const where = at === "client" ? admin : injector;
+				const [where, device] = deviceAt(at);
 				const args = spec === null ? [
 					"/opt/harness/inject",
 					"clear-netem",
-					"eth0"
+					device
 				] : [
 					"/opt/harness/inject",
 					"netem",
-					"eth0",
+					device,
 					...spec.split(/\s+/)
 				];
 				return stamped(where, args);
 			},
 			async offload(on, at = "target") {
-				await output(at === "client" ? admin : injector, [
+				const devices = typeof at === "object" ? Object.values(hopNamed(at.hop).devices) : [deviceAt(at)[1]];
+				const [where] = deviceAt(at);
+				for (const device of devices) await output(where, [
 					"/opt/harness/inject",
 					"offload",
-					"eth0",
+					device,
 					on ? "on" : "off"
 				]);
 			},
@@ -1199,13 +1419,14 @@ const openLab = async (target, kind = "tcp", topology) => {
 				for (const argv of commands) await output(server, [...argv]);
 			},
 			async heal(at, when) {
-				const [where, peer] = placeOf(at);
+				const [where, peer, via] = placeOf(at);
 				return stamped(where, [
 					"/opt/harness/inject",
 					"clear",
 					peer,
 					"--at",
-					String(Math.round(when))
+					String(Math.round(when)),
+					...via
 				]);
 			},
 			async forget() {
@@ -1469,6 +1690,8 @@ const once = async (lab, cell, environment, index, dry = false) => {
 			};
 			const toxicFirst = cell.toxic !== void 0 && cell.fault !== "toxic";
 			if (toxicFirst) await lab.toxic(cell.toxic);
+			const netemAt = cell.faultAt ?? "target";
+			const netemOn = (fault) => netemAt === "target" ? lab.netem(netemSpec(cell, fault), "target") : lab.netem(fault ?? null, netemAt);
 			const apply = async () => {
 				switch (cell.fault) {
 					case "freeze-process":
@@ -1480,10 +1703,11 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					case "unreachable": return lab.fault("reject", faultAt, { code: cell.icmp ?? "host" });
 					case "simplex": return lab.fault("blackhole", faultAt, options);
 					case "mtu-blackhole": return lab.fault("mtu", "client", { above: cell.mtu ?? 1200 });
-					case "netem": return lab.netem(netemSpec(cell, cell.netem), "target");
+					case "netem": return netemOn(cell.netem);
 					case "rst": return lab.fault("rst", faultAt);
 					case "middlebox-forget":
-					case "proxy-idle": return null;
+					case "proxy-idle":
+					case "none": return null;
 					case "toxic":
 						if (cell.toxic === void 0) throw new Error(`${cell.name} names no toxic`);
 						return lab.toxic(cell.toxic);
@@ -1499,12 +1723,13 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					case "fin":
 					case "middlebox-forget":
 					case "proxy-idle":
-					case "server-close": return;
+					case "server-close":
+					case "none": return;
 					case "unreachable": return lab.lift("reject", faultAt, { code: cell.icmp ?? "host" });
 					case "simplex": return lab.lift("blackhole", faultAt, options);
 					case "mtu-blackhole": return lab.lift("mtu", "client", { above: cell.mtu ?? 1200 });
 					case "netem":
-						await lab.netem(netemSpec(cell, void 0), "target");
+						await netemOn(void 0);
 						return;
 					case "rst": return lab.lift("rst", faultAt);
 					case "toxic":
@@ -1591,6 +1816,7 @@ const inEnvironment = async (cell, environment, count, say) => {
 		if (cell.fault === "mtu-blackhole" || cell.fault === "netem") {
 			await lab.offload(false, "target");
 			await lab.offload(false, "client");
+			if (typeof cell.faultAt === "object") await lab.offload(false, cell.faultAt);
 		}
 		const latency = netemSpec(cell, void 0);
 		if (latency !== null) await lab.netem(latency, "target");
@@ -1665,6 +1891,6 @@ const runCell = async (cell, say = () => {}) => {
 	}
 };
 //#endregion
-export { LISTING, NOISE, PUMP, RELATIVE, STEPS, endpointAfter, harnessImage, judge, measure, medianInterval, nodeClientImage, openLab, routerImage, routesOf, runCell, socketOf, spread, templated, toxicBody, verdictOf };
+export { LISTING, NOISE, PUMP, RELATIVE, STEPS, crossingsOf, deviceOf, endpointAfter, harnessImage, judge, measure, medianInterval, nodeClientImage, openLab, quantileOf, reorderingOf, routerImage, routesOf, runCell, socketOf, spread, spreadOf, templated, toxicBody, verdictOf };
 
 //# sourceMappingURL=index.js.map