@mark1russell7/harness 0.5.0 → 0.6.0

package/build/index.js

@@ -10,6 +10,8 @@ import { fileURLToPath } from "node:url";
 const isHopPlace = (place) => typeof place === "object" && "hop" in place;
 //#endregion
 //#region src/verdict.ts
+/** This function gives the words of the outcome of a second request: its connection, a retry, and its end. */
+const requestWords = (outcome) => `${outcome.connection}${outcome.retried ? ", a retry" : ""}, ${outcome.pending === true ? "pending" : outcome.error ?? "a reply"}`;
 /**
 * This amount is the noise of the pipeline itself, δ_abs, as the calibration measured it. It is
 * the 99.9th percentile of the whole latency of a reset, at the concurrency cap or less. Its
@@ -115,14 +117,32 @@ const middle = (sorted) => {
 * which does not exist at this time.
 */
 const verdictOf = (prediction, runs) => {
-	const verdict = verdictOfBand(prediction, runs);
+	const band = verdictOfBand(prediction, runs);
 	const counted = runs.filter((run) => run.void === null);
+	const verdict = counted.some((run) => run.request !== void 0) ? withRequests(band, counted) : band;
 	if (prediction.group === void 0 && !counted.some((run) => run.group !== void 0)) return verdict;
 	return {
 		...verdict,
 		group: groupVerdictOf(prediction, counted)
 	};
 };
+/**
+* This function adds the outcomes of the second request of an HTTP client to a verdict. A cell
+* whose outcome is a distribution gets them in its reason too.
+*/
+const withRequests = (verdict, counted) => {
+	const requests = {};
+	for (const run of counted) {
+		const words = run.request === null || run.request === void 0 ? "no second request" : requestWords(run.request);
+		requests[words] = (requests[words] ?? 0) + 1;
+	}
+	const listed = Object.entries(requests).map(([words, n]) => `${n} ${words}`).join("; ");
+	return {
+		...verdict,
+		requests,
+		why: verdict.decision === "measured" ? `${verdict.why} Second requests: ${listed}.` : verdict.why
+	};
+};
 /** This function gives the median of some counts, or null for none. */
 const medianCount = (values) => values.length === 0 ? null : middle([...values].sort((a, b) => a - b));
 /** This function sums what the group of a group cell did in its counted runs, and tests each stage that has a band. */
@@ -761,8 +781,60 @@ const crossingsOf = (run, hop, toClient, toTarget) => {
 *   it, and before the first keepalive probe. A live peer answers that probe, and the answer
 *   starts the idle time again. A run where the fault was late is void. A run whose records the
 *   prober lost is also void.
+* - For an HTTP client, it also reads what the second request met (`requestOf`). An outcome other
+*   than the one that the prediction gives is a finding.
 */
 const measure = (cell, run) => {
+	const read = measureSocket(cell, run);
+	const expected = cell.prediction.request;
+	if (cell.probe.http === void 0 && expected === void 0) return read;
+	const request = requestOf(run.events);
+	if (expected === void 0 || read.void !== null) return {
+		...read,
+		request
+	};
+	const differs = requestFinding(expected, request);
+	const finding = read.finding === null ? differs : differs === null ? read.finding : `${read.finding}. ${differs}`;
+	return {
+		...read,
+		request,
+		finding
+	};
+};
+/** This function tells how a request reading differs from its prediction, or gives null. */
+const requestFinding = (expected, read) => {
+	if (read === null) return `the second request never went out, where it should meet: ${requestWords(expected)}`;
+	return read.connection === expected.connection && read.retried === expected.retried && read.error === expected.error && read.pending === (expected.pending === true) ? null : `the second request met: ${requestWords(read)}, not ${requestWords(expected)}`;
+};
+/**
+* This function reads the second request of an HTTP client from its lines. Its first try is its
+* first `request` line, and each `retry` line is a new try. Its end is its `error` line, or its
+* `replied` line. A request with neither is pending. It gives null when the client wrote no try.
+*
+* A new try that gets no connection writes no `retry` line. Its dial writes a `dial-error` line.
+* Thus a `dial-error` line after the first try and before the end is also a new try.
+*/
+const requestOf = (events) => {
+	const of = (event) => Number(event["i"]) === 2;
+	const tries = events.filter((event) => (event.ev === "request" || event.ev === "retry") && of(event));
+	const first = tries[0];
+	if (first === void 0) return null;
+	const failed = events.find((event) => event.ev === "error" && of(event));
+	const replied = events.some((event) => event.ev === "replied" && of(event));
+	const ready = events.find((event) => event.ev === "ready");
+	const sent = events.find((event) => event.ev === "sent" && of(event));
+	const end = failed ?? events.find((event) => event.ev === "done" && of(event));
+	const dialed = events.some((event) => event.ev === "dial-error" && event.t > first.t && (end === void 0 || event.t <= end.t));
+	return {
+		connection: first["reused"] === true ? "reused" : "new",
+		retried: dialed || tries.some((each) => each.ev === "retry"),
+		error: failed === void 0 ? null : String(failed["errno"]),
+		pending: failed === void 0 && !replied,
+		gap: ready === void 0 || sent === void 0 ? null : between(ready.t, sent.t)
+	};
+};
+/** This function reads the socket of one run against its cell, as `measure` does without the second request of an HTTP client. */
+const measureSocket = (cell, run) => {
 	if (cell.group !== void 0) {
 		const read = measureGroup(cell, run);
 		return {
@@ -784,7 +856,7 @@ const measure = (cell, run) => {
 	const { events, kernel } = run;
 	const ready = events.find((event) => event.ev === "ready");
 	const local = events.find((event) => event.ev === "connected")?.["local"];
-	const inflight = cell.probe.mode !== "idle";
+	const inflight = cell.probe.mode !== "idle" && cell.probe.mode !== "gap";
 	const request = inflight ? events.filter((event) => event.ev === "sent")[1] : ready;
 	const server = cell.observer === "server";
 	const place = socketOf(cell, run);
@@ -873,14 +945,15 @@ const measure = (cell, run) => {
 	const reported = socket.find((record) => record.ev === "error" && (record.err ?? 0) !== 0);
 	const seen = remote ? void 0 : events.find((event) => event.ev === "error" || event.ev === "eof");
 	const closeWait = socket.find((record) => record.ev === "state" && newState(record) === 8);
+	const own = cell.prediction.detects === true && cell.prediction.errno === "CLOSED" ? socket.find((record) => record.ev === "state" && oldState(record) === 1 && newState(record) === 4) : void 0;
 	const outcome = reported !== void 0 ? {
 		kind: "error",
 		errno: ERRNO[reported.err] ?? `E${reported.err}`
-	} : closeWait !== void 0 || seen?.ev === "eof" ? { kind: "eof" } : seen !== void 0 ? {
+	} : closeWait !== void 0 || seen?.ev === "eof" ? { kind: "eof" } : own !== void 0 ? { kind: "closed" } : seen !== void 0 ? {
 		kind: "error",
 		errno: String(seen["errno"])
 	} : { kind: "silent" };
-	const ended = reported?.mono ?? closeWait?.mono ?? seen?.t;
+	const ended = reported?.mono ?? closeWait?.mono ?? own?.mono ?? seen?.t;
 	const noticed = reported !== void 0 && seen !== void 0 ? between(reported.mono, seen.t) : null;
 	const { prediction } = cell;
 	const before = run.fault?.t1 ?? ready.t;
@@ -1056,8 +1129,9 @@ const measure = (cell, run) => {
 		finding: `it ended without its reference event (${prediction.from})`
 	};
 	const steps = (prediction.from === "last-received" ? keepalives : prediction.from === "first-retransmission" ? retransmitted.filter((record) => record.why === "rto" && record.mono > reference) : []).map((record) => between(reference, record.mono));
-	const finding = (prediction.errno === "EOF" ? outcome.kind === "eof" : outcome.kind === "error" && outcome.errno === prediction.errno) ? null : `it ended with ${outcome.kind === "error" ? outcome.errno : "end of file"}, not ${prediction.errno}`;
-	if (prediction.from === "last-received" || prediction.from === "first-retransmission") voided ??= delayed(reference, ended, prediction.from === "last-received" ? "keepalive" : "retransmit");
+	const finding = (prediction.errno === "EOF" ? outcome.kind === "eof" : prediction.errno === "CLOSED" ? outcome.kind === "closed" : outcome.kind === "error" && outcome.errno === prediction.errno) ? null : `it ended with ${outcome.kind === "error" ? outcome.errno : outcome.kind === "closed" ? "its own close" : "end of file"}, not ${prediction.errno}`;
+	const programTimer = cell.probe.http !== void 0 && prediction.from === "last-received" && prediction.timer !== "keepalive";
+	if (prediction.from === "last-received" && !programTimer || prediction.from === "first-retransmission") voided ??= delayed(reference, ended, prediction.from === "last-received" ? "keepalive" : "retransmit");
 	const lag = (valueIn(prediction.high, "s") - valueIn(prediction.low, "s")) * 1e9;
 	const slow = prediction.timer.startsWith("expire_") ? (run.middlebox ?? []).find((line) => line.ev === "pump-slow" && line.t <= ended && line.t + Number(line["took"]) * 1e9 >= ended - lag) : void 0;
 	if (slow !== void 0) voided ??= `a listing of the router took ${Math.round(Number(slow["took"]) * 1e4) / 10} ms, longer than the bound of its prediction`;
@@ -1114,6 +1188,18 @@ const NODE = {
 	dockerfile: "harness/image/node/Dockerfile",
 	from: ["harness/image/node"]
 };
+/** This image holds the Go HTTP client: the http.Transport of Go 1.27.2. */
+const GO_HTTP = {
+	name: "loom-harness-go-http",
+	dockerfile: "harness/image/go-http/Dockerfile",
+	from: ["harness/image/go-http"]
+};
+/** This image holds the Node HTTP clients: the http.Agent of Node 24.21.0, and undici 8.12.2. */
+const NODE_HTTP = {
+	name: "loom-harness-node-http",
+	dockerfile: "harness/image/node-http/Dockerfile",
+	from: ["harness/image/node-http"]
+};
 /** This image is the router of a middlebox chain: NAT and conntrack, with a form for a forgotten flow. */
 const ROUTER = {
 	name: "loom-harness-router",
@@ -1180,6 +1266,10 @@ const harnessImage = () => build(HARNESS);
 const nodeClientImage = () => build(NODE);
 /** This function gives the image of the router. */
 const routerImage = () => build(ROUTER);
+/** This function gives the image of the Go HTTP client. */
+const goHttpImage = () => build(GO_HTTP);
+/** This function gives the image of the Node HTTP clients. */
+const nodeHttpImage = () => build(NODE_HTTP);
 /**
 * This time is the interval of the listing of a router, when its hop does not set one. A listing
 * evicts each expired flow that it walks past, and the DESTROY event of that flow follows.
@@ -1322,6 +1412,16 @@ const clientOf = async (kind, image) => {
 		command: ["node", "/opt/harness/node/client-ioredis.mjs"],
 		clientKind: kind
 	};
+	if (kind === "go-http") return {
+		clientImage: await goHttpImage(),
+		command: ["/opt/harness/client-go"],
+		clientKind: kind
+	};
+	if (kind === "http-agent" || kind === "undici") return {
+		clientImage: await nodeHttpImage(),
+		command: ["node", `/opt/harness/node-http/client-${kind === "undici" ? "undici" : "agent"}.mjs`],
+		clientKind: kind
+	};
 	return {
 		clientImage: image,
 		command: ["/opt/harness/client"],
@@ -1362,6 +1462,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 	const runtime = await getContainerRuntimeClient();
 	const opened = [];
 	const networks = [];
+	const serverLines = lines();
 	const close = async () => {
 		for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
 		for (const network of networks.reverse()) await network.stop().catch(() => void 0);
@@ -1381,7 +1482,12 @@ const openLab = async (target, kind = "tcp", topology) => {
 	try {
 		for (let at = 0; at <= hops.length; at += 1) networks.push(await new Network().start());
 		const last = networks[hops.length];
-		const stock = new GenericContainer(target.image).withNetwork(last).withWaitStrategy(Wait.forLogMessage(target.ready));
+		let stock = new GenericContainer(target.image).withNetwork(last).withWaitStrategy(Wait.forLogMessage(target.ready));
+		if (target.files !== void 0) stock = stock.withCopyContentToContainer(Object.entries(target.files).map(([path, content]) => ({
+			content,
+			target: path
+		})));
+		if (target.lines === true) stock = stock.withLogConsumer(serverLines.consume);
 		const server = await (target.command === void 0 ? stock : stock.withCommand([...target.command])).start();
 		opened.push(server);
 		const serverIp = await ipOn(server, last);
@@ -1633,6 +1739,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 			netns,
 			topology: placed,
 			middlebox,
+			server: serverLines.seen,
 			kernel: records.seen,
 			async sysctl(values) {
 				const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
@@ -1898,7 +2005,9 @@ const openLab = async (target, kind = "tcp", topology) => {
 						image: target.image,
 						id: (await inspect(server)).Image,
 						settings,
-						sysctls: targetSysctls
+						sysctls: targetSysctls,
+						...target.files === void 0 ? {} : { files: Object.fromEntries(Object.entries(target.files).map(([path, content]) => [path, createHash("sha256").update(content).digest("hex")])) },
+						...serverLines.seen[0] === void 0 ? {} : { first: serverLines.seen[0] }
 					},
 					harness: {
 						image,
@@ -2621,6 +2730,18 @@ const probeArgs = (cell, lab) => {
 		"--mode",
 		probe.mode
 	];
+	if (probe.http !== void 0) {
+		const { http } = probe;
+		const gap = http.gap === void 0 ? 0 : valueIn(http.gap, "s");
+		args.push("--method", http.method);
+		if (http.gap !== void 0) args.push("--gap-ms", String(valueIn(http.gap, "ms")));
+		if (http.hold !== void 0) args.push("--hold-ms", String(valueIn(http.hold, "ms")));
+		if (http.bytes !== void 0) args.push("--bytes", String(http.bytes));
+		for (const [name, value] of Object.entries(http.options ?? {})) args.push(`--${name}`, String(value));
+		args.push("--linger-ms", String(LINGER_MS));
+		args.push("--max-s", String(Math.ceil(longest * 2 + gap + 10)));
+		return args;
+	}
 	if (probe.request !== void 0) args.push("--request", probe.request);
 	if (probe.count !== void 0) args.push("--count", String(probe.count));
 	if (probe.interval !== void 0) args.push("--interval-ms", String(valueIn(probe.interval, "ms")));
@@ -2651,6 +2772,22 @@ const observerError = (cell, partial, within) => new Promise((resolve) => {
 });
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /**
+* This time is the pause between the client's first reply and the fault. It is more than Linux's
+* TCP_DELACK_MAX of 200 ms. Thus the delayed ACK of the reply is out before the fault.
+*/
+const QUIET_MS = 250;
+/**
+* This time is how long an HTTP client waits after its second request, before it exits. The exit
+* closes its sockets, thus that close comes after the window of the cell.
+*/
+const LINGER_MS = 1e3;
+/** These are the lines that end the workload of an HTTP client: the end of its second request, its deadline, or its failure. */
+const HTTP_DONE = [
+	"done",
+	"deadline",
+	"fail"
+];
+/**
 * This function waits for the request that an in-flight client sends on the signal: its second
 * send, after the PING that made it ready. Its stamp is when the request first met the fault.
 */
@@ -2686,6 +2823,7 @@ const once = async (lab, cell, environment, index, dry = false) => {
 	await lab.forget();
 	const from = lab.kernel.length;
 	const middleboxFrom = lab.middlebox.length;
+	const serverFrom = lab.server.length;
 	const lostBefore = droppedSoFar(lab.kernel);
 	const probe = await lab.probe(probeArgs(cell, lab));
 	const server = cell.observer === "server";
@@ -2748,11 +2886,12 @@ const once = async (lab, cell, environment, index, dry = false) => {
 						return null;
 					case "unreachable": return lab.fault("reject", faultAt, { code: cell.icmp ?? "host" });
 					case "simplex": return lab.fault("blackhole", faultAt, options);
-					case "mtu-blackhole": return lab.fault("mtu", "client", { above: cell.mtu ?? 1200 });
+					case "mtu-blackhole": return lab.fault("mtu", cell.faultAt ?? "client", { above: cell.mtu ?? 1200 });
 					case "netem": return netemOn(cell.netem);
 					case "rst": return lab.fault("rst", faultAt);
 					case "middlebox-forget":
 					case "proxy-idle":
+					case "app-stall":
 					case "none": return null;
 					case "toxic":
 						if (cell.toxic === void 0) throw new Error(`${cell.name} names no toxic`);
@@ -2770,10 +2909,11 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					case "middlebox-forget":
 					case "proxy-idle":
 					case "server-close":
+					case "app-stall":
 					case "none": return;
 					case "unreachable": return lab.lift("reject", faultAt, { code: cell.icmp ?? "host" });
 					case "simplex": return lab.lift("blackhole", faultAt, options);
-					case "mtu-blackhole": return lab.lift("mtu", "client", { above: cell.mtu ?? 1200 });
+					case "mtu-blackhole": return lab.lift("mtu", cell.faultAt ?? "client", { above: cell.mtu ?? 1200 });
 					case "netem":
 						await netemOn(void 0);
 						return;
@@ -2787,9 +2927,14 @@ const once = async (lab, cell, environment, index, dry = false) => {
 			fault = await apply();
 			try {
 				if (cell.idleBefore !== void 0 && cell.probe.mode !== "idle") await sleep(valueIn(cell.idleBefore, "ms"));
-				if (cell.probe.mode !== "idle" && !replyInFlight) await go();
+				if ((cell.probe.mode === "inflight" || cell.probe.mode === "stream") && !replyInFlight) await go();
 				const { prediction } = cell;
-				if (cell.fault === "stall") {
+				const http = cell.probe.http !== void 0;
+				if (cell.probe.mode === "gap") {
+					const gap = cell.probe.http?.gap === void 0 ? 0 : valueIn(cell.probe.http.gap, "s");
+					const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
+					await probe.until(HTTP_DONE, (gap + window) * 2e3 + 1e4).catch(() => void 0);
+				} else if (cell.fault === "stall") {
 					if (cell.stall === void 0) throw new Error(`${cell.name} says no stall's length`);
 					const sent = await requestSent(probe, 5e3);
 					heal = await lab.heal(faultAt, sent.t + valueIn(cell.stall, "s") * 1e9);
@@ -2810,10 +2955,12 @@ const once = async (lab, cell, environment, index, dry = false) => {
 				} else if (prediction.detects) {
 					const within = valueIn(prediction.high, "s") * 2e3 + 1e4;
 					if (cell.observer !== void 0 && cell.observer !== "client") await observerError(cell, partial, within);
+					else if (http && cell.probe.mode !== "idle") await probe.until(HTTP_DONE, within).catch(() => void 0);
 					else await probe.until([
 						"error",
 						"eof",
-						"deadline"
+						"deadline",
+						...http ? ["closed"] : []
 					], within).catch(() => void 0);
 				} else await sleep(valueIn(prediction.window, "s") * 1e3);
 			} finally {
@@ -2838,7 +2985,8 @@ const once = async (lab, cell, environment, index, dry = false) => {
 		targetNetns: lab.netns.target,
 		lost,
 		...placed,
-		...lab.topology === null ? {} : { middlebox: lab.middlebox.slice(middleboxFrom) }
+		...lab.topology === null ? {} : { middlebox: lab.middlebox.slice(middleboxFrom) },
+		...cell.target.lines === true ? { server: lab.server.slice(serverFrom) } : {}
 	};
 };
 /** This function divides a count of runs across environments, as equally as possible. */
@@ -2967,6 +3115,6 @@ const runCell = async (cell, say = () => {}) => {
 	}
 };
 //#endregion
-export { CONVERGE_MS, LISTING, NOISE, PUMP, RELATIVE, STAGES, STEPS, crossingsOf, deadlineOf, deviceOf, endpointAfter, errorsOf, fieldsOf, groupOnce, groupVerdictOf, harnessImage, infoOf, isHopPlace, judge, lagsOf, measure, measureGroup, medianInterval, memberConfig, nodeClientImage, openGroupLab, openLab, peersOf, quantileOf, reorderingOf, routerImage, routesOf, runCell, sentinelConfig, socketOf, spread, spreadOf, stagesOf, subscriptionsOf, templated, toxicBody, verdictOf, viewOf, writesOf };
+export { CONVERGE_MS, LINGER_MS, LISTING, NOISE, PUMP, QUIET_MS, RELATIVE, STAGES, STEPS, crossingsOf, deadlineOf, deviceOf, endpointAfter, errorsOf, fieldsOf, goHttpImage, groupOnce, groupVerdictOf, harnessImage, infoOf, isHopPlace, judge, lagsOf, measure, measureGroup, medianInterval, memberConfig, nodeClientImage, nodeHttpImage, openGroupLab, openLab, peersOf, quantileOf, reorderingOf, requestOf, requestWords, routerImage, routesOf, runCell, sentinelConfig, socketOf, spread, spreadOf, stagesOf, subscriptionsOf, templated, toxicBody, verdictOf, viewOf, writesOf };
 
 //# sourceMappingURL=index.js.map