@mark1russell7/harness 0.7.0 → 0.8.0

package/build/index.js

@@ -119,7 +119,8 @@ const middle = (sorted) => {
 const verdictOf = (prediction, runs) => {
 	const band = verdictOfBand(prediction, runs);
 	const counted = runs.filter((run) => run.void === null);
-	const verdict = counted.some((run) => run.request !== void 0) ? withRequests(band, counted) : band;
+	const asked = counted.some((run) => run.request !== void 0) ? withRequests(band, counted) : band;
+	const verdict = withProgram(asked, counted);
 	if (prediction.group === void 0 && !counted.some((run) => run.group !== void 0)) return verdict;
 	return {
 		...verdict,
@@ -127,6 +128,43 @@ const verdictOf = (prediction, runs) => {
 	};
 };
 /**
+* This function adds the reconnect of the client and the outcomes of its call in flight to a
+* verdict. Its reason gets a sentence for each. A verdict whose runs have neither stays as it is.
+*/
+const withProgram = (verdict, counted) => {
+	const reconnects = counted.flatMap((run) => run.reconnect === void 0 ? [] : [run.reconnect]);
+	const calls = counted.flatMap((run) => run.call === void 0 ? [] : [run.call]);
+	const lefts = counted.flatMap((run) => run.left === void 0 ? [] : [run.left]);
+	if (reconnects.length === 0 && calls.length === 0 && lefts.length === 0) return verdict;
+	const ms = (spread) => spread === null ? "none" : `${Math.round(valueIn(spread.median, "s") * 1e6) / 1e3} ms`;
+	const tries = Math.max(0, ...reconnects.map((each) => each.waits.length));
+	const reconnect = reconnects.length === 0 ? void 0 : {
+		waits: Array.from({ length: tries }, (_, at) => spreadOf(reconnects.flatMap((each) => each.waits[at] === void 0 ? [] : [each.waits[at]]))),
+		syn: spreadOf(reconnects.flatMap((each) => each.syn === null ? [] : [each.syn])),
+		established: spreadOf(reconnects.flatMap((each) => each.established === null ? [] : [each.established])),
+		connect: spreadOf(reconnects.flatMap((each) => each.connects))
+	};
+	const outcomes = {};
+	for (const call of calls) outcomes[call.outcome] = (outcomes[call.outcome] ?? 0) + 1;
+	const gone = lefts.flatMap((each) => each === null ? [] : [each]);
+	const left = lefts.length === 0 ? void 0 : {
+		spread: spreadOf(gone),
+		stayed: lefts.length - gone.length
+	};
+	const said = [
+		...left === void 0 ? [] : [`After the drop, the program closed its socket in ${gone.length} runs, median ${ms(left.spread)} after its drop line${left.stayed === 0 ? "" : `, and it did not close it in ${left.stayed}`}.`],
+		...reconnect === void 0 ? [] : [`Reconnect: the waits before each try, median ${reconnect.waits.map(ms).join(", ")}; the first SYN median ${ms(reconnect.syn)} and ESTABLISHED median ${ms(reconnect.established)} after the detection.`],
+		...calls.length === 0 ? [] : [`Calls in flight: ${Object.entries(outcomes).map(([outcome, n]) => `${n} ${outcome}`).join("; ")}.`]
+	];
+	return {
+		...verdict,
+		...reconnect === void 0 ? {} : { reconnect },
+		...calls.length === 0 ? {} : { calls: outcomes },
+		...left === void 0 ? {} : { left },
+		why: `${verdict.why.endsWith(".") ? verdict.why : `${verdict.why}.`} ${said.join(" ")}`
+	};
+};
+/**
 * This function adds the outcomes of the second request of an HTTP client to a verdict. A cell
 * whose outcome is a distribution gets them in its reason too.
 */
@@ -383,6 +421,150 @@ const verdictOfBand = (prediction, runs) => {
 	};
 };
 //#endregion
+//#region src/application.ts
+/**
+* This module reads the acts of the program of the observer: its heartbeat, its drop of the
+* connection, its reconnect and its call in flight. The kernel has no report of a detection by
+* a program. Thus the program writes a line before each of its own acts, on CLOCK_MONOTONIC:
+* - `ping` before each ping of its heartbeat, and `drop` before it closes the connection by itself.
+* - `reconnecting` when it starts its reconnect timer, and `dial` before each new connection.
+*
+* The kernel then stamps each write of the program on the socket (`tx`) and each change of the
+* state of the socket. A line of the client names its socket by its local address (`local`), and
+* a line of a server names it by its peer (`peer`).
+*/
+/** This function gives the interval between two instants on the one clock, in seconds to the microsecond. */
+const between$3 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
+/** This function gives the state of a state record after the change, and before it. TCP_ESTABLISHED is 1, TCP_SYN_SENT 2. */
+const newState$2 = (record) => record.new;
+const oldState$2 = (record) => record.old;
+/** This function gives the lines of the program of the observer: the client, or the target for the server. A proxy has no lines. */
+const programLines = (cell, run) => {
+	const observer = cell.observer ?? "client";
+	if (observer === "client") return run.events;
+	if (observer === "server") return run.server ?? [];
+	return [];
+};
+/**
+* This function tells if a line of the program is about the connection of the run. The client
+* names it by its local address, and a server by its peer. A line with neither is about the one
+* connection of its program. A line whose address is null is about a socket that did not open:
+* a try that failed.
+*/
+const onConnection = (line, local) => {
+	if (!("local" in line) && !("peer" in line)) return true;
+	return (line["local"] ?? line["peer"]) === local;
+};
+/** This function gives the first record of a socket that matches, at the stamp of a line or in δ_abs after it. */
+const kernelAt = (socket, line, matches) => {
+	const noise = valueIn(NOISE, "s") * 1e9;
+	return socket.find((record) => record.mono >= line.t && record.mono <= line.t + noise && matches(record));
+};
+const eventOf = (socket, line, matches) => {
+	const record = kernelAt(socket, line, matches);
+	return {
+		line,
+		at: record?.mono ?? line.t,
+		kernel: record !== void 0,
+		record: record === void 0 ? null : record.ev === "tx" ? "write" : "state"
+	};
+};
+/**
+* This function gives the time from a line of the program to the close of the socket by its own
+* side. That close is its change from ESTABLISHED to FIN_WAIT1, or to CLOSE (an abort). A FIN of
+* the peer (CLOSE_WAIT) is not such a close. It is null when the socket did not close itself.
+*/
+const leftAfter = (socket, line) => {
+	const left = socket.find((record) => record.ev === "state" && oldState$2(record) === 1 && (newState$2(record) === 4 || newState$2(record) === 7) && record.mono >= line.t);
+	return left === void 0 ? null : between$3(line.t, left.mono);
+};
+/**
+* This function tells how a drop differs from the way that its prediction gives, or gives null.
+* A close closes the socket in δ_abs. A close frame is a write, and the program does not close
+* the socket at the drop. A peer that answers the close frame can close it, and that is no finding.
+*/
+const dropFinding = (expected, drop, left) => {
+	const noise = valueIn(NOISE, "s");
+	const soon = left !== null && valueIn(left, "s") <= noise;
+	if (expected === "close") {
+		if (left === null) return "the program did not close its socket after the drop, where a close is at once";
+		return soon ? null : `the program closed its socket ${left.value} s after the drop, not at once`;
+	}
+	if (drop.record !== "write") return "the drop wrote no close frame on its socket";
+	return soon ? "the program closed its socket at the drop, where a close frame leaves it open" : null;
+};
+const isWrite = (record) => record.ev === "tx";
+/** This function gives each ping of the program on the connection of the run, with the kernel's stamp of its write. */
+const pingsOf = (lines, socket, local) => lines.filter((line) => line.ev === "ping" && onConnection(line, local)).map((line) => eventOf(socket, line, isWrite));
+/**
+* This function gives the first ping that got no answer: no segment with data reached the socket
+* after its write, up to `until`. Its write starts the wait of the heartbeat that the fault met.
+*/
+const firstUnanswered = (pings, socket, until = Number.POSITIVE_INFINITY) => pings.filter((ping) => ping.at <= until).find((ping) => !socket.some((record) => record.ev === "rx" && (record.seg?.len.value ?? 0) > 0 && record.mono > ping.at && record.mono <= until));
+/**
+* This function gives the first drop of the connection by its program. The kernel stamps it as the
+* write of a close frame. Or it stamps the change of the socket out of ESTABLISHED: a close or an
+* abort.
+*/
+const dropOf = (lines, socket, local) => {
+	const line = lines.find((each) => each.ev === "drop" && onConnection(each, local));
+	return line === void 0 ? void 0 : eventOf(socket, line, (record) => isWrite(record) || record.ev === "state" && oldState$2(record) === 1);
+};
+/**
+* This function reads the reconnect of the client after the detection at `detected`. The socket of
+* the run has the cookie `cookie`. The lines of the program count from `from`: the drop line of a
+* drop, which comes before the kernel's stamp of it.
+*/
+const reconnectOf = (run, detected, cookie, from = detected) => {
+	const after = run.events.filter((line) => line.t > from);
+	const starts = after.filter((line) => line.ev === "reconnecting");
+	const dials = after.filter((line) => line.ev === "dial");
+	const waits = starts.flatMap((start, at) => {
+		const next = starts[at + 1]?.t ?? Number.POSITIVE_INFINITY;
+		const dial = dials.find((each) => each.t >= start.t && each.t < next);
+		return dial === void 0 ? [] : [between$3(start.t, dial.t)];
+	});
+	const fresh = run.kernel.filter((record) => record.netns === run.netns && record.cookie !== cookie && record.ev === "state" && record.mono > detected);
+	const syns = fresh.filter((record) => newState$2(record) === 2);
+	const established = fresh.find((record) => newState$2(record) === 1);
+	const connects = dials.flatMap((dial) => {
+		const syn = syns.find((record) => record.mono >= dial.t);
+		return syn === void 0 ? [] : [between$3(dial.t, syn.mono)];
+	});
+	return {
+		waits,
+		syn: syns[0] === void 0 ? null : between$3(detected, syns[0].mono),
+		established: established === void 0 ? null : between$3(detected, established.mono),
+		connects
+	};
+};
+/** This function tells how the waits of a reconnect differ from their bands, each widened by δ_abs, or gives null. */
+const reconnectFinding = (expected, read) => {
+	const noise = valueIn(NOISE, "s");
+	const round = (value) => Math.round(value * 1e6) / 1e6;
+	const found = expected.waits.flatMap((band, at) => {
+		const wait = read.waits[at];
+		if (wait === void 0) return [`the client made no try ${at + 1} after the detection`];
+		const value = valueIn(wait, "s");
+		const low = round(valueIn(band.low, "s") - noise);
+		const high = round(valueIn(band.high, "s") + noise);
+		return value < low || value > high ? [`its wait before try ${at + 1} was ${value} s, outside [${low}, ${high}] s`] : [];
+	});
+	return found.length === 0 ? null : found.join(". ");
+};
+/** This function reads the call that the client sent into the fault: its `call-error` or `data` line after the request. */
+const callOf = (events, request, detected) => {
+	const end = events.find((line) => (line.ev === "call-error" || line.ev === "data") && line.t > request.t);
+	if (end === void 0) return {
+		outcome: "pending",
+		after: null
+	};
+	return {
+		outcome: end.ev === "data" ? "a reply" : String(end["message"] ?? end["code"] ?? "an error"),
+		after: detected === void 0 ? null : between$3(detected, end.t)
+	};
+};
+//#endregion
 //#region src/driver.ts
 /** These are the errno numbers that the kernel reports to a socket, by the names that predictions use. */
 const ERRNO$1 = {
@@ -1118,9 +1300,10 @@ const newState = (record) => record.new;
 const count = (event, field) => event === void 0 ? 0 : Number(event[field] ?? 0);
 /** This function gives the view of the client on the same interval: TCP_INFO, with a sample each 2 ms. */
 const socketVantage = (cell, events, ready, request) => {
-	if (!cell.prediction.detects || cell.prediction.from === "peer-closed" || cell.prediction.from === "fault") return null;
+	const { from } = cell.prediction.detects === true ? cell.prediction : { from: void 0 };
+	if (from === void 0 || from === "peer-closed" || from === "fault" || from === "ping-sent") return null;
 	const ended = events.find((event) => event.ev === "error" || event.ev === "eof");
-	const reference = cell.prediction.from === "last-received" ? ready : cell.prediction.from === "request-sent" ? request : events.find((event) => event.ev === "kernel" && event.t >= request.t && count(event, "retransmits") > 0);
+	const reference = from === "last-received" ? ready : from === "request-sent" ? request : events.find((event) => event.ev === "kernel" && event.t >= request.t && count(event, "retransmits") > 0);
 	return ended === void 0 || reference === void 0 ? null : between(reference.t, ended.t);
 };
 const ipOf = (address) => address.split(":")[0];
@@ -1418,21 +1601,28 @@ const measureSocket = (cell, run) => {
 	const seen = remote ? void 0 : events.find((event) => event.ev === "error" || event.ev === "eof");
 	const closeWait = socket.find((record) => record.ev === "state" && newState(record) === 8);
 	const own = cell.prediction.detects === true && cell.prediction.errno === "CLOSED" ? socket.find((record) => record.ev === "state" && oldState(record) === 1 && newState(record) === 4) : void 0;
-	const outcome = reported !== void 0 ? {
+	const lines = programLines(cell, run);
+	const kernelEnded = reported?.mono ?? closeWait?.mono ?? own?.mono ?? seen?.t;
+	const first = dropOf(lines, socket, local);
+	const drop = first !== void 0 && (kernelEnded === void 0 || first.at <= kernelEnded) ? first : void 0;
+	const dropped = drop !== void 0;
+	const outcome = dropped ? { kind: "dropped" } : reported !== void 0 ? {
 		kind: "error",
 		errno: ERRNO[reported.err] ?? `E${reported.err}`
 	} : closeWait !== void 0 || seen?.ev === "eof" ? { kind: "eof" } : own !== void 0 ? { kind: "closed" } : seen !== void 0 ? {
 		kind: "error",
 		errno: String(seen["errno"])
 	} : { kind: "silent" };
-	const ended = reported?.mono ?? closeWait?.mono ?? own?.mono ?? seen?.t;
-	const noticed = reported !== void 0 && seen !== void 0 ? between(reported.mono, seen.t) : null;
+	const ended = drop?.at ?? kernelEnded;
+	const noticed = reported !== void 0 && seen !== void 0 && !dropped ? between(reported.mono, seen.t) : null;
 	const { prediction } = cell;
+	const pings = pingsOf(lines, socket, local);
 	const before = run.fault?.t1 ?? ready.t;
 	const referenceAt = (from) => {
 		if (from === "request-sent") return request.t;
 		if (from === "fault") return run.fault?.t0;
 		if (from === "client-start") return (events.find((event) => event.ev === "connecting") ?? events[0])?.t;
+		if (from === "ping-sent") return firstUnanswered(pings, socket, ended)?.at;
 		return (from === "last-received" ? socket.filter((each) => each.ev === "rx" && each.mono <= before).at(-1) : from === "first-retransmission" ? retransmitted.find((each) => each.why === "rto") : kernel.find((each) => each.netns === peerNetns && each.cookie === peerCookie && each.ev === "state" && oldState(each) === 1))?.mono;
 	};
 	const lastRx = kernel.filter((each) => each.netns === run.netns && each.src === local && each.ev === "rx" && each.mono <= before).at(-1)?.mono ?? ready.t;
@@ -1481,21 +1671,22 @@ const measureSocket = (cell, run) => {
 	}
 	if (!prediction.detects && prediction.recovers !== void 0) {
 		const { recovers } = prediction;
+		const start = cell.probe.mode === "idle" ? lines.find((line) => line.ev === "ping" && line.t >= (run.fault?.t1 ?? 0)) ?? request : request;
 		const heal = run.heal ?? null;
 		if (voided === null && heal === null) voided = "the stall never ended";
 		else if (voided === null && heal !== null) {
-			const from = between(request.t, heal.t0);
-			const to = between(request.t, heal.t1);
+			const from = between(start.t, heal.t0);
+			const to = between(start.t, heal.t1);
 			if (valueIn(from, "s") <= valueIn(recovers.heal.low, "s") || valueIn(to, "s") >= valueIn(recovers.heal.high, "s")) voided = `the stall ended ${from.value}–${to.value} s after the send, outside the ${recovers.heal.low.value}–${recovers.heal.high.value} s its prediction assumed`;
 		}
-		const steps = retransmitted.filter((record) => record.mono > request.t).map((record) => between(request.t, record.mono));
+		const steps = retransmitted.filter((record) => record.mono > start.t).map((record) => between(start.t, record.mono));
 		if (ended !== void 0) return {
 			...base,
 			...nothing,
 			outcome,
 			steps,
 			void: voided,
-			finding: `detected after ${between(request.t, ended).value} s, where it should recover`
+			finding: `detected after ${between(start.t, ended).value} s, where it should recover`
 		};
 		const answer = heal === null ? void 0 : socket.find((record) => record.ev === "rx" && record.mono > heal.t0);
 		if (answer === void 0) return {
@@ -1505,13 +1696,13 @@ const measureSocket = (cell, run) => {
 			void: voided,
 			finding: "it never moved again: nothing came back after the stall ended"
 		};
-		const reply = events.find((event) => event.ev === "data" && event.t > request.t);
-		voided ??= delayed(request.t, answer.mono, "retransmit");
+		const reply = events.find((event) => event.ev === "data" && event.t > start.t);
+		voided ??= delayed(start.t, answer.mono, "retransmit");
 		return {
 			...base,
 			outcome: { kind: "answered" },
-			elapsed: between(request.t, answer.mono),
-			socket: reply === void 0 ? null : between(request.t, reply.t),
+			elapsed: between(start.t, answer.mono),
+			socket: reply === void 0 ? null : between(start.t, reply.t),
 			noticed: reply === void 0 ? null : between(answer.mono, reply.t),
 			steps,
 			void: voided,
@@ -1601,23 +1792,41 @@ const measureSocket = (cell, run) => {
 		void: voided,
 		finding: `it ended without its reference event (${prediction.from})`
 	};
-	const steps = (prediction.from === "last-received" ? keepalives : prediction.from === "first-retransmission" ? retransmitted.filter((record) => record.why === "rto" && record.mono > reference) : []).map((record) => between(reference, record.mono));
-	const finding = (prediction.errno === "EOF" ? outcome.kind === "eof" : prediction.errno === "CLOSED" ? outcome.kind === "closed" : outcome.kind === "error" && outcome.errno === prediction.errno) ? null : `it ended with ${outcome.kind === "error" ? outcome.errno : outcome.kind === "closed" ? "its own close" : "end of file"}, not ${prediction.errno}`;
+	const steps = prediction.from === "ping-sent" ? pings.filter((ping) => ping.at > reference && ping.at <= ended).map((ping) => between(reference, ping.at)) : (prediction.from === "last-received" ? keepalives : prediction.from === "first-retransmission" ? retransmitted.filter((record) => record.why === "rto" && record.mono > reference) : []).map((record) => between(reference, record.mono));
+	const matches = prediction.errno === "EOF" ? outcome.kind === "eof" : prediction.errno === "CLOSED" ? outcome.kind === "closed" : prediction.errno === "DROPPED" ? outcome.kind === "dropped" : outcome.kind === "error" && outcome.errno === prediction.errno;
+	const words = outcome.kind === "error" ? outcome.errno : outcome.kind === "closed" ? "its own close" : outcome.kind === "dropped" ? "a drop by its own program" : "end of file";
+	const reconnect = prediction.reconnects === void 0 ? void 0 : reconnectOf(run, ended, cookie, drop?.line.t ?? ended);
+	const late = reconnect === void 0 || prediction.reconnects === void 0 ? null : reconnectFinding(prediction.reconnects, reconnect);
+	const left = prediction.drop === void 0 || drop === void 0 ? void 0 : leftAfter(socket, drop.line);
+	const how = prediction.drop === void 0 || drop === void 0 || left === void 0 ? null : dropFinding(prediction.drop, drop, left);
+	const found = [
+		matches ? null : `it ended with ${words}, not ${prediction.errno}`,
+		late,
+		how
+	].filter((each) => each !== null);
+	const finding = found.length === 0 ? null : found.join(". ");
 	const programTimer = cell.probe.http !== void 0 && prediction.from === "last-received" && prediction.timer !== "keepalive";
 	if (prediction.from === "last-received" && !programTimer || prediction.from === "first-retransmission") voided ??= delayed(reference, ended, prediction.from === "last-received" ? "keepalive" : "retransmit");
+	if (prediction.from === "ping-sent") voided ??= delayed(reference, ended, "retransmit");
 	const lag = (valueIn(prediction.high, "s") - valueIn(prediction.low, "s")) * 1e9;
 	const slow = prediction.timer.startsWith("expire_") ? (run.middlebox ?? []).find((line) => line.ev === "pump-slow" && line.t <= ended && line.t + Number(line["took"]) * 1e9 >= ended - lag) : void 0;
 	if (slow !== void 0) voided ??= `a listing of the router took ${Math.round(Number(slow["took"]) * 1e4) / 10} ms, longer than the bound of its prediction`;
 	const commit = run.fault === null ? 0 : (run.fault.t1 - run.fault.t0) / 1e6;
 	if (prediction.from === "fault" && commit > valueIn(NOISE, "ms")) voided ??= `the commit of the fault took ${Math.round(commit * 10) / 10} ms, longer than the noise of the pipeline: its start is not known`;
+	const unanswered = prediction.from === "ping-sent" ? firstUnanswered(pings, socket, ended) : void 0;
+	const endLine = drop?.line ?? seen;
+	const view = unanswered !== void 0 ? endLine === void 0 ? null : between(unanswered.line.t, endLine.t) : remote ? null : socketVantage(cell, events, ready, request);
 	return {
 		...base,
 		...chain,
 		outcome,
 		elapsed: between(reference, ended),
-		socket: remote ? null : socketVantage(cell, events, ready, request),
+		socket: view,
 		noticed,
 		steps,
+		...reconnect === void 0 ? {} : { reconnect },
+		...left === void 0 ? {} : { left },
+		...cell.probe.ws !== void 0 && inflight ? { call: callOf(events, request, ended) } : {},
 		void: voided,
 		finding
 	};
@@ -1679,6 +1888,12 @@ const MONGO = {
 	dockerfile: "harness/image/mongo/Dockerfile",
 	from: ["harness/image/mongo"]
 };
+/** This image holds `@procedures/client` 0.3.0 on Node 24.21.0: its WebSocket client as a client of the harness, and the script of its server. */
+const PROCEDURES = {
+	name: "loom-harness-procedures",
+	dockerfile: "harness/image/procedures/Dockerfile",
+	from: ["harness/image/procedures"]
+};
 /** This image is the router of a middlebox chain: NAT and conntrack, with a form for a forgotten flow. */
 const ROUTER = {
 	name: "loom-harness-router",
@@ -1751,6 +1966,10 @@ const goHttpImage = () => build(GO_HTTP);
 const nodeHttpImage = () => build(NODE_HTTP);
 /** This function gives the image of the MongoDB client. */
 const mongoClientImage = () => build(MONGO);
+/** This function gives the image of `@procedures/client`: its WebSocket client and its server. */
+const proceduresImage = () => build(PROCEDURES);
+/** This function gives the image that a target operates: its stock image, or an image of the harness (`Target.harness`). */
+const targetImageOf = (target) => target.harness === "procedures" ? proceduresImage() : Promise.resolve(target.image);
 /**
 * This time is the interval of the listing of a router, when its hop does not set one. A listing
 * evicts each expired flow that it walks past, and the DESTROY event of that flow follows.
@@ -1908,6 +2127,11 @@ const clientOf = async (kind, image) => {
 		command: ["node", "/opt/harness/mongo/client-mongo.mjs"],
 		clientKind: kind
 	};
+	if (kind === "procedures-ws") return {
+		clientImage: await proceduresImage(),
+		command: ["node", "/opt/harness/procedures/client-ws.mjs"],
+		clientKind: kind
+	};
 	return {
 		clientImage: image,
 		command: ["/opt/harness/client"],
@@ -1968,7 +2192,8 @@ const openLab = async (target, kind = "tcp", topology) => {
 	try {
 		for (let at = 0; at <= hops.length; at += 1) networks.push(await new Network().start());
 		const last = networks[hops.length];
-		let stock = new GenericContainer(target.image).withNetwork(last).withWaitStrategy(Wait.forLogMessage(target.ready));
+		const targetImage = await targetImageOf(target);
+		let stock = new GenericContainer(targetImage).withNetwork(last).withWaitStrategy(Wait.forLogMessage(target.ready));
 		if (target.files !== void 0) stock = stock.withCopyContentToContainer(Object.entries(target.files).map(([path, content]) => ({
 			content,
 			target: path
@@ -2502,7 +2727,11 @@ const openLab = async (target, kind = "tcp", topology) => {
 					boot_id: await output(client, ["cat", "/proc/sys/kernel/random/boot_id"]),
 					docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
 					target: {
-						image: target.image,
+						image: targetImage,
+						...target.harness === void 0 ? {} : {
+							harness: target.harness,
+							base: target.image
+						},
 						id: (await inspect(server)).Image,
 						settings,
 						sysctls: targetSysctls,
@@ -3857,6 +4086,20 @@ const probeArgs = (cell, lab) => {
 		"--mode",
 		probe.mode
 	];
+	if (probe.ws !== void 0) {
+		const { ws } = probe;
+		const waits = (prediction.detects === true ? prediction.reconnects?.waits ?? [] : []).reduce((sum, wait) => sum + valueIn(wait.high, "s"), 0);
+		args.push("--websocket", ws.socket);
+		args.push("--heartbeat", ws.heartbeat === null ? "off" : "on");
+		if (ws.heartbeat !== null) args.push("--heartbeat-interval-ms", String(valueIn(ws.heartbeat.interval, "ms")), "--heartbeat-timeout-ms", String(valueIn(ws.heartbeat.timeout, "ms")), "--heartbeat-misses", String(ws.heartbeat.misses));
+		args.push("--reconnect", ws.reconnect === null ? "off" : "on");
+		if (ws.reconnect !== null) args.push("--reconnect-initial-ms", String(valueIn(ws.reconnect.initialDelay, "ms")), "--reconnect-multiplier", String(ws.reconnect.multiplier), "--reconnect-max-ms", String(valueIn(ws.reconnect.maxDelay, "ms")), "--reconnect-attempts", String(ws.reconnect.attempts), "--reconnect-jitter", String(ws.reconnect.jitter));
+		args.push("--request-timeout-ms", String(valueIn(ws.requestTimeout, "ms")));
+		args.push("--connection-timeout-ms", String(valueIn(ws.connectionTimeout, "ms")));
+		if (ws.hold !== void 0) args.push("--hold-ms", String(valueIn(ws.hold, "ms")));
+		args.push("--max-s", String(Math.ceil((longest + waits) * 2 + 10)));
+		return args;
+	}
 	if (probe.http !== void 0) {
 		const { http } = probe;
 		const gap = http.gap === void 0 ? 0 : valueIn(http.gap, "s");
@@ -3894,14 +4137,15 @@ const probeArgs = (cell, lab) => {
 /**
 * This function waits for an error on the socket of the observer, or for its end of file
 * (CLOSE_WAIT, state 8). The prober records both. A cell whose observer is the server, or a proxy
-* of a topology, waits for that. The promise resolves after `within` ms at the latest.
+* of a topology, waits for that. A target with its own detector can also drop the connection
+* itself (`dropped`). The promise resolves after `within` ms at the latest.
 */
-const observerError = (cell, partial, within) => new Promise((resolve) => {
+const observerError = (cell, partial, within, dropped = () => false) => new Promise((resolve) => {
 	const started = Date.now();
 	const look = () => {
 		const run = partial();
 		const at = socketOf(cell, run);
-		if (at !== void 0 && run.kernel.some((record) => record.netns === at.netns && record.cookie === at.cookie && (record.ev === "error" && (record.err ?? 0) !== 0 || record.ev === "state" && record.new === 8)) || Date.now() - started > within) resolve();
+		if (at !== void 0 && run.kernel.some((record) => record.netns === at.netns && record.cookie === at.cookie && (record.ev === "error" && (record.err ?? 0) !== 0 || record.ev === "state" && record.new === 8)) || dropped() || Date.now() - started > within) resolve();
 		else setTimeout(look, 50);
 	};
 	look();
@@ -4000,6 +4244,31 @@ const beforeOnce = async (lab, cell, environment, index) => {
 		}
 	};
 };
+/** This function waits for the first ping of the heartbeat of the client at the instant `after` or later: its first ping into a stall. */
+const pingSent = (probe, after, within) => new Promise((resolve, reject) => {
+	const started = Date.now();
+	const look = () => {
+		const ping = probe.events.find((event) => event.ev === "ping" && event.t >= after);
+		if (ping !== void 0) resolve(ping);
+		else if (Date.now() - started > within) reject(/* @__PURE__ */ new Error(`the client sent no ping into the stall within ${within} ms`));
+		else setTimeout(look, 1);
+	};
+	look();
+});
+/**
+* This function waits until the client wrote `count` lines `dial`, one before each connection
+* that it tries, or until its deadline. It resolves after `within` ms at the latest.
+*/
+const dials = (probe, count, within) => new Promise((resolve) => {
+	const started = Date.now();
+	const look = () => {
+		const tried = probe.events.filter((event) => event.ev === "dial").length;
+		const ended = probe.events.some((event) => event.ev === "deadline");
+		if (tried >= count || ended || Date.now() - started > within) resolve();
+		else setTimeout(look, 20);
+	};
+	look();
+});
 /** This function gives the netem specification of a cell: its latency, then the netem of its fault when it has one. */
 const netemSpec = (cell, fault) => {
 	const parts = [...cell.latency === void 0 ? [] : [`delay ${valueIn(cell.latency, "ms")}ms`], ...fault === void 0 ? [] : [fault]];
@@ -4162,13 +4431,14 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					await probe.until(HTTP_DONE, (gap + window) * 2e3 + 1e4).catch(() => void 0);
 				} else if (cell.fault === "stall") {
 					if (cell.stall === void 0) throw new Error(`${cell.name} says no stall's length`);
-					const sent = await requestSent(probe, 5e3);
-					heal = await lab.heal(faultAt, sent.t + valueIn(cell.stall, "s") * 1e9);
 					const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
+					const sent = cell.probe.mode === "idle" ? await pingSent(probe, fault?.t1 ?? 0, window * 1e3) : await requestSent(probe, 5e3);
+					heal = await lab.heal(faultAt, sent.t + valueIn(cell.stall, "s") * 1e9);
 					await probe.until([
 						"data",
 						"error",
-						"eof"
+						"eof",
+						"drop"
 					], window * 1e3).catch(() => void 0);
 				} else if (prediction.detects === null) {
 					const window = valueIn(prediction.window, "s") * 1e3;
@@ -4180,12 +4450,13 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					], window).catch(() => void 0);
 				} else if (prediction.detects) {
 					const within = valueIn(prediction.high, "s") * 2e3 + 1e4;
-					if (cell.observer !== void 0 && cell.observer !== "client") await observerError(cell, partial, within);
+					if (cell.observer !== void 0 && cell.observer !== "client") await observerError(cell, partial, within, () => lab.server.slice(serverFrom).some((line) => line.ev === "drop"));
 					else if (http && cell.probe.mode !== "idle") await probe.until(HTTP_DONE, within).catch(() => void 0);
 					else await probe.until([
 						"error",
 						"eof",
 						"deadline",
+						"drop",
 						...http ? ["closed"] : []
 					], within).catch(() => void 0);
 				} else await sleep(valueIn(prediction.window, "s") * 1e3);
@@ -4193,7 +4464,12 @@ const once = async (lab, cell, environment, index, dry = false) => {
 				await lift();
 				if (toxicFirst) await lab.toxic(null);
 			}
-			if (cell.fault === "freeze-process" && !server) await probe.until(["data"], 5e3).catch(() => void 0);
+			if (cell.fault === "freeze-process" && !server && cell.probe.mode !== "idle") await probe.until(["data"], 5e3).catch(() => void 0);
+			const expected = cell.prediction.detects === true ? cell.prediction.reconnects : void 0;
+			if (expected !== void 0) {
+				const waits = expected.waits.reduce((sum, wait) => sum + valueIn(wait.high, "s"), 0);
+				await dials(probe, 1 + expected.waits.length, waits * 2e3 + 5e3);
+			}
 		}
 	} finally {
 		await probe.stop();
@@ -4370,6 +4646,6 @@ const runCell = async (cell, say = () => {}) => {
 	}
 };
 //#endregion
-export { CONVERGE_MS, KEPT, LINGER_MS, LISTING, MEMBER_MARKS, NOISE, PUMP, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, STAGES, STATUS_SCRIPT, STEPS, STEP_DOWN, clockOffsetOf, crossingsOf, cutOf, deadlineOf, deviceOf, endpointAfter, errorsOf, fieldOf, fieldsOf, goHttpImage, groupOnce, groupVerdictOf, harnessImage, infoOf, initiateScript, isHopPlace, judge, lagsOf, matches, measure, measureDriver, measureGroup, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, nodeClientImage, nodeHttpImage, onMonotonic, openGroupLab, openLab, openReplicaSetLab, peersOf, quantileOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, routerImage, routesOf, runCell, sentinelConfig, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, statusViewOf, subscriptionsOf, templated, toxicBody, verdictOf, viewOf, writesOf };
+export { CONVERGE_MS, KEPT, LINGER_MS, LISTING, MEMBER_MARKS, NOISE, PUMP, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, STAGES, STATUS_SCRIPT, STEPS, STEP_DOWN, callOf, clockOffsetOf, crossingsOf, cutOf, deadlineOf, deviceOf, dropFinding, dropOf, endpointAfter, errorsOf, fieldOf, fieldsOf, firstUnanswered, goHttpImage, groupOnce, groupVerdictOf, harnessImage, infoOf, initiateScript, isHopPlace, judge, lagsOf, leftAfter, matches, measure, measureDriver, measureGroup, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, nodeClientImage, nodeHttpImage, onMonotonic, openGroupLab, openLab, openReplicaSetLab, peersOf, pingsOf, proceduresImage, programLines, quantileOf, reconnectFinding, reconnectOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, routerImage, routesOf, runCell, sentinelConfig, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, statusViewOf, subscriptionsOf, targetImageOf, templated, toxicBody, verdictOf, viewOf, writesOf };
 
 //# sourceMappingURL=index.js.map