@mark1russell7/harness 0.14.2 → 0.14.3

package/build/index.js

@@ -1,10 +1,12 @@
-import { C as targetImageOf, E as unpinnedBases, S as subjectProblems, T as unlockedManifests, _ as pgClientImage, a as SHIPPED_NAMES, b as routerImage, c as grpcClientImage, d as imageOf, f as imageProblems, g as nodeHttpImage, h as nodeClientImage, i as SHIPPED_CLIENTS, l as harnessImage, m as mongoClientImage, n as PINNED, o as amqpClientImage, p as inContext, r as REPOSITORY, s as goHttpImage, t as CLIENT_NAMES, u as hikariClientImage, v as planOf, w as targetSourceOf, x as subjectClient, y as programOf } from "./image-Lg8LL6C3.js";
+import { A as unpinnedBases, C as routerImage, D as targetImageOf, E as subjectProblems, F as locksOf, I as npmVersions, L as versionsOf, M as cargoVersions, N as goVersions, O as targetSourceOf, P as jarVersions, S as programOf, T as subjectClient, _ as mongoClientImage, a as SHIPPED_NAMES, b as pgClientImage, c as buildRecordOf, d as harnessImage, f as harnessVersion, g as inContext, h as imageProblems, i as SHIPPED_CLIENTS, j as LOCK_NAMES, k as unlockedManifests, l as goHttpImage, m as imageOf, n as PINNED, o as amqpClientImage, p as hikariClientImage, r as REPOSITORY, s as basesOf, t as CLIENT_NAMES, u as grpcClientImage, v as nodeClientImage, w as shippedSource, x as planOf, y as nodeHttpImage } from "./image-BMdV7yWD.js";
 import { quantity, valueIn } from "@mark1russell7/quantity";
 import { median, medianInterval as medianInterval$1, quantileSorted } from "@mark1russell7/rigor-statistics";
-import { createHash } from "node:crypto";
-import { existsSync, readFileSync } from "node:fs";
-import { release } from "node:os";
+import { createHash, randomBytes } from "node:crypto";
+import { execFile } from "node:child_process";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { release, tmpdir } from "node:os";
 import { join } from "node:path";
+import { existsSync, readFileSync } from "node:fs";
 //#region src/cell.ts
 /** This function tells if a fault place is a hop of a topology. */
 const isHopPlace = (place) => typeof place === "object" && "hop" in place;
@@ -404,6 +406,7 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
 			band: null,
 			outside: [],
 			median: samples.length === 0 ? null : s(median(samples)),
+			...samples.length === 0 ? {} : { slowest: s(samples.at(-1)) },
 			...reordering === void 0 ? {} : { reordering },
 			why
 		};
@@ -870,7 +873,7 @@ const ipOf$1 = (address) => address.slice(0, address.lastIndexOf(":"));
 /** This function gives the first line of the client that matches, after an instant. */
 const lineAfter = (events, match, after) => events.find((event) => event.t > after && matches(event, match));
 /** This function gives the name of the error of a line: the field that the match names, or `name`. */
-const nameOf = (line, match) => String(fieldOf(line, match.name ?? "name") ?? line.ev);
+const nameOf$1 = (line, match) => String(fieldOf(line, match.name ?? "name") ?? line.ev);
 /**
 * This function gives the report of the kernel on a socket after an instant: an error, or the end of
 * the file (CLOSE_WAIT). With neither, a close of the socket in SYN_SENT is the end. The program gave
@@ -1052,12 +1055,12 @@ const measureDriver = (cell, run) => {
 			findings.push(line === void 0 ? `${step.what}: ${wordsOf(step.ends)} never came` : `${step.what}: its reference (${step.from}) never came`);
 			return {
 				what: step.what,
-				name: line === void 0 ? null : nameOf(line, step.ends),
+				name: line === void 0 ? null : nameOf$1(line, step.ends),
 				elapsed: null
 			};
 		}
 		const elapsed = between$2(from, line.t);
-		const name = nameOf(line, step.ends);
+		const name = nameOf$1(line, step.ends);
 		if (step.name !== "OK" && name !== step.name) findings.push(`${step.what}: it ended with ${name}, not ${step.name}`);
 		const [low, high] = [round(valueIn(step.low, "s") - noise), round(valueIn(step.high, "s") + noise)];
 		if (elapsed.value < low || elapsed.value > high) findings.push(`${step.what}: ${elapsed.value} s is outside [${low}, ${high}] s`);
@@ -1107,7 +1110,7 @@ const measureDriver = (cell, run) => {
 	const ended = line?.t ?? report?.at;
 	const outcome = line !== void 0 && driver.ends !== void 0 ? {
 		kind: "error",
-		errno: nameOf(line, driver.ends)
+		errno: nameOf$1(line, driver.ends)
 	} : report?.outcome ?? { kind: "silent" };
 	if (ended === void 0) return {
 		...base,
@@ -1142,6 +1145,171 @@ const measureDriver = (cell, run) => {
 	};
 };
 //#endregion
+//#region src/recovery.ts
+/**
+* This module holds the pure parts of a recovery drill. These are the scripts of the shell, the reader
+* of the output of `bsondump`, and the set check of a recovery. It has no container, thus unit tests
+* run it. `drill.ts` does the drill on a lab.
+*
+* A dump with `--oplog` reads the oplog from the start of the dump to its end. A slice reads the
+* oplog after the last entry of the backup before it. Thus the backups together hold each write up
+* to the last entry of the last backup, and no write after it. One writer sends the writes one at a
+* time, thus the highest sequence number is also the latest write.
+*/
+/** This size is one document of generated data. A tenant holds as many documents as its size needs. */
+const SEED_DOCUMENT = 65536;
+/** This count is how far the lost writes can be from the writes after the last backup: the writes in flight at the fault. */
+const LOSS_MARGIN = 2;
+/** This time is how long a script waits for a server to stop, in tenths of a second. */
+const STOP_TENTHS = 600;
+/** This function gives the time of a drill from the start of the backups to the fault, in seconds. */
+const drillSpan = (drill) => drill.slices * valueIn(drill.sliceEvery, "s") + valueIn(drill.age, "s");
+/**
+* This script writes the generated data of each tenant into the collection `tenants`. Each write
+* waits for `copies` members, thus each member holds the data when the script ends.
+*/
+const seedScript = (drill, copies) => {
+	const documents = Math.ceil(valueIn(drill.perTenant, "B") / SEED_DOCUMENT);
+	const blob = `BinData(0, require("crypto").randomBytes(${SEED_DOCUMENT}).toString("base64"))`;
+	return `const c = db.getSiblingDB("cells"); c.tenants.drop({ writeConcern: { w: ${copies} } }); for (let t = 1; t <= ${drill.tenants}; t += 1) { const docs = []; for (let d = 1; d <= ${documents}; d += 1) docs.push({ _id: "t" + t + ".d" + d, tenant: t, blob: ${blob} }); c.tenants.insertMany(docs, { writeConcern: { w: ${copies}, wtimeout: 60000 } }); } print(JSON.stringify({ ok: true }))`;
+};
+/** This script prints the hash of the collection `tenants`. It takes locks, thus a run reads it with no writer. */
+const HASH_SCRIPT = "print(JSON.stringify({ hash: db.getSiblingDB(\"cells\").runCommand({ dbHash: 1, collections: [\"tenants\"] }).collections.tenants }))";
+/** This function gives the hash from the output of `HASH_SCRIPT`. */
+const hashOf = (text) => {
+	const line = text.split("\n").filter((each) => each.trim() !== "").at(-1) ?? "";
+	const { hash } = JSON.parse(line);
+	if (typeof hash !== "string" || hash === "") throw new Error(`no hash of the tenants in: ${line}`);
+	return hash;
+};
+/** This function gives the command of a server with no set, on its own port. A client of the set cannot reach that server, and a restore writes to it. */
+const standaloneCommand = (set, port) => `${set.programs.server} --bind_ip_all --port ${port} ${(set.args ?? []).join(" ")}`.trim();
+/**
+* This script waits for the end of the server of a box, and it can stop the server first. `kill -0`
+* also succeeds on a zombie, and the shell of the box can leave its child as a zombie for a long
+* time. Thus the script reads the state of the process, and a zombie has ended.
+*/
+const stopScript = (terminate) => "pid=$(cat /tmp/server.pid); " + (terminate ? "kill -TERM \"$pid\"; " : "") + `for i in $(seq 1 ${STOP_TENTHS}); do s=$(cut -d' ' -f3 /proc/$pid/stat 2>/dev/null); if [ -z "\$s" ] || [ "\$s" = Z ]; then exit 0; fi; sleep 0.1; done; exit 1`;
+/** This function gives a number from Extended JSON, in its relaxed form or in its canonical form, or null. */
+const numberOf = (value) => {
+	if (typeof value === "number") return value;
+	if (typeof value !== "object" || value === null) return null;
+	const wrapped = value;
+	const text = wrapped.$numberInt ?? wrapped.$numberLong ?? wrapped.$numberDouble;
+	return typeof text === "string" && text !== "" && Number.isFinite(Number(text)) ? Number(text) : null;
+};
+/** This function gives each document of the output of `bsondump`. A line that is not a document does not count. */
+const documentsOf = (text) => text.split("\n").flatMap((line) => {
+	if (!line.trim().startsWith("{")) return [];
+	try {
+		return [JSON.parse(line)];
+	} catch {
+		return [];
+	}
+});
+/** This function gives the `ts` of an oplog entry, or null. */
+const timeOf = (entry) => {
+	const stamp = entry["ts"]?.$timestamp;
+	const t = numberOf(stamp?.t);
+	const i = numberOf(stamp?.i);
+	return t === null || i === null ? null : {
+		t,
+		i
+	};
+};
+/** This function reads an oplog file from the output of `bsondump`. Only an insert into `cells.w` is a write of a client. */
+const readOplog = (text) => {
+	let last = null;
+	let highest = null;
+	for (const entry of documentsOf(text)) {
+		const time = timeOf(entry);
+		if (time !== null && (last === null || time.t > last.t || time.t === last.t && time.i > last.i)) last = time;
+		if (entry["op"] !== "i" || entry["ns"] !== "cells.w") continue;
+		const n = numberOf(entry["o"]?.n);
+		if (n !== null && (highest === null || n > highest)) highest = n;
+	}
+	return {
+		last,
+		highest
+	};
+};
+/** This function gives the highest sequence number of the file of the writes in a dump, from the output of `bsondump`, or null. */
+const highestHeld = (text) => {
+	const held = documentsOf(text).flatMap((document) => {
+		const n = numberOf(document["n"]);
+		return n === null ? [] : [n];
+	});
+	return held.length === 0 ? null : Math.max(...held);
+};
+/** This function gives the query of a slice of the oplog: each entry after an oplog time, in Extended JSON. */
+const sliceQuery = (after) => JSON.stringify({ ts: { $gt: { $timestamp: {
+	t: after.t,
+	i: after.i
+} } } });
+const rounded = (value) => Math.round(value * 1e6) / 1e6;
+/** This function gives the acknowledgement time of each accepted write of the first client of a run. */
+const firstAcks = (events, second) => {
+	const acks = /* @__PURE__ */ new Map();
+	for (const event of events) {
+		const n = Number(event["n"]);
+		if (event.ev === "ack" && (second === null || n < second)) acks.set(n, event.t);
+	}
+	return acks;
+};
+/** This function gives the measurement of a drill, from its run. `t0` is the fault. */
+const recoveryOf = (drill, run, t0) => {
+	const record = run.group?.drill;
+	const sameData = record === void 0 || record.after === null ? null : record.after === record.before;
+	if (drill.recovery === "election" || record === void 0) return {
+		rpo: null,
+		excess: null,
+		holes: [],
+		afterBackup: null,
+		sameData
+	};
+	const acks = firstAcks(run.events, record.second);
+	const accepted = [...acks.keys()].sort((a, b) => a - b);
+	const cutoff = record.backups.at(-1)?.last?.n ?? 0;
+	const afterBackup = accepted.filter((n) => n > cutoff).length;
+	const holds = run.group?.holds ?? null;
+	if (holds === null) return {
+		rpo: null,
+		excess: null,
+		holes: [],
+		afterBackup,
+		sameData
+	};
+	const kept = new Set(holds);
+	const ownKept = holds.filter((n) => record.second === null || n < record.second);
+	const highest = ownKept.length === 0 ? 0 : Math.max(...ownKept);
+	const holes = accepted.filter((n) => n < highest && !kept.has(n));
+	const lastKept = accepted.filter((n) => kept.has(n)).at(-1);
+	const rpo = lastKept === void 0 ? null : rounded((t0 - acks.get(lastKept)) / 1e9);
+	return {
+		rpo: rpo === null ? null : {
+			value: rpo,
+			unit: "s"
+		},
+		excess: rpo === null || drill.quiet === true ? null : {
+			value: rounded(rpo - valueIn(drill.age, "s")),
+			unit: "s"
+		},
+		holes,
+		afterBackup,
+		sameData
+	};
+};
+/** This function gives the findings of a drill: each rule of a recovery that its run broke. */
+const recoveryFindings = (measured, writes) => {
+	const findings = [];
+	if (measured.sameData === null) findings.push("the set did not converge, thus nothing compared the data of the tenants");
+	else if (!measured.sameData) findings.push("the data of the tenants changed in the recovery");
+	if (measured.holes.length > 0) findings.push(`the restore lost ${measured.holes.length} accepted writes below a write that it kept, the first at ${measured.holes[0]}`);
+	const lost = writes.lost?.length ?? null;
+	if (measured.afterBackup !== null && lost !== null && Math.abs(lost - measured.afterBackup) > 2) findings.push(`the restore lost ${lost} accepted writes, and ${measured.afterBackup} came after the last backup`);
+	return findings;
+};
+//#endregion
 //#region src/timeline.ts
 /**
 * This module measures a run of a group cell. It gives the first write that a primary accepted
@@ -1487,7 +1655,8 @@ const measureGroup = (cell, run) => {
 		stages,
 		writes: writesOf(run, t0, cluster ? "" : primary),
 		errors: errorsOf(events, t0),
-		...cell.calibrates === "sentinels" ? { lags: lagsOf(run, kernel) } : {}
+		...cell.calibrates === "sentinels" ? { lags: lagsOf(run, kernel) } : {},
+		...cell.drill === void 0 ? {} : { recovery: recoveryOf(cell.drill, run, t0) }
 	};
 	let voided = null;
 	const tilt = timeline.find((line) => line.ev === "sentinel" && line.event === "+tilt" && line.t >= t0);
@@ -1541,6 +1710,7 @@ const measureGroup = (cell, run) => {
 		if (lost === null) findings.push("the group did not converge, thus the count of its lost writes is not known");
 		else if (lost.length < low || lost.length > high) findings.push(`the group lost ${lost.length} accepted writes, outside [${low}, ${high}]`);
 	}
+	if (group.recovery !== void 0) findings.push(...recoveryFindings(group.recovery, group.writes));
 	if (accepted === void 0) {
 		if (prediction.detects !== null) findings.push(lands === "new" ? "the client never wrote to a new primary in the window" : "the client never wrote again in the window");
 		return {
@@ -2169,6 +2339,440 @@ const measureSocket = (cell, run) => {
 	};
 };
 //#endregion
+//#region src/fabric.ts
+/**
+* A fabric is a routed network between the client and the target, on layer E3 (docs/S7.md). Four
+* routers of FRRouting, two leaves and two spines, run real BGP over veth pairs that containerlab
+* makes. The client hangs off leaf1 and the target off leaf2. The path between them goes through
+* spine1 by a weight, and spine2 holds the other path. A cell silences spine1, and the routes of
+* the leaves move to spine2 when a detector of the routers fires.
+*
+* This file holds the plan of the fabric as pure functions, and `openFabric` that starts it. A lab
+* opens a fabric with `Topology.fabric`. Thus the client, the target, the prober and the verdict
+* stay the same as on layer E1. The fabric needs a Linux host with containerlab and sudo.
+*/
+/** This image is FRRouting 10.7.1, pinned by digest. */
+const FRR_IMAGE = "quay.io/frrouting/frr:10.7.1@sha256:e995beaa50fdc9edb35eadcfefa29b7f062cc06f2b812613789b68fa541554d2";
+/** This device is the side of the fabric in the client and in the target. Docker keeps eth0. */
+const FABRIC_DEVICE = "eth1";
+/** This address is the client on its network behind leaf1. */
+const FABRIC_CLIENT = "10.1.0.2";
+/** This address is the target on its network behind leaf2. */
+const FABRIC_TARGET = "10.2.0.2";
+/** This device of a leaf goes to spine1, thus a route on it is on the preferred path. */
+const PREFERRED_DEVICE = "eth1";
+/** This limit is the time that the sessions and the routes get to come back before a run. */
+const READY_MS = 9e4;
+const READY_POLL_MS = 250;
+const CLIENT_NET = "10.1.0.0/24";
+const TARGET_NET = "10.2.0.0/24";
+/** These are the routers. Each leaf prefers spine1 by a weight of 100, thus the path of each run is known. */
+const ROUTERS = [
+	{
+		name: "leaf1",
+		as: 65001,
+		id: "10.255.0.1",
+		interfaces: {
+			eth1: "10.0.1.0/31",
+			eth2: "10.0.2.0/31",
+			eth3: "10.1.0.1/24"
+		},
+		neighbors: [{
+			address: "10.0.1.1",
+			as: 65101,
+			weight: 100
+		}, {
+			address: "10.0.2.1",
+			as: 65102
+		}],
+		network: CLIENT_NET,
+		remote: TARGET_NET
+	},
+	{
+		name: "leaf2",
+		as: 65002,
+		id: "10.255.0.2",
+		interfaces: {
+			eth1: "10.0.3.0/31",
+			eth2: "10.0.4.0/31",
+			eth3: "10.2.0.1/24"
+		},
+		neighbors: [{
+			address: "10.0.3.1",
+			as: 65101,
+			weight: 100
+		}, {
+			address: "10.0.4.1",
+			as: 65102
+		}],
+		network: TARGET_NET,
+		remote: CLIENT_NET
+	},
+	{
+		name: "spine1",
+		as: 65101,
+		id: "10.255.1.1",
+		interfaces: {
+			eth1: "10.0.1.1/31",
+			eth2: "10.0.3.1/31"
+		},
+		neighbors: [{
+			address: "10.0.1.0",
+			as: 65001
+		}, {
+			address: "10.0.3.0",
+			as: 65002
+		}]
+	},
+	{
+		name: "spine2",
+		as: 65102,
+		id: "10.255.1.2",
+		interfaces: {
+			eth1: "10.0.2.1/31",
+			eth2: "10.0.4.1/31"
+		},
+		neighbors: [{
+			address: "10.0.2.0",
+			as: 65001
+		}, {
+			address: "10.0.4.0",
+			as: 65002
+		}]
+	}
+];
+const LEAVES = ROUTERS.filter((router) => router.remote !== void 0);
+/** These are the links between the routers. The links to the client and the target come from `topologyOf`. */
+const LINKS = [
+	["leaf1:eth1", "spine1:eth1"],
+	["leaf1:eth2", "spine2:eth1"],
+	["leaf2:eth1", "spine1:eth2"],
+	["leaf2:eth2", "spine2:eth2"]
+];
+/** This text is the `daemons` file of each router: BGP and BFD, on top of what FRRouting always starts. */
+const DAEMONS = [
+	"bgpd=yes",
+	"bfdd=yes",
+	"vtysh_enable=yes",
+	"zebra_options=\"  -A 127.0.0.1 -s 90000000\"",
+	"mgmtd_options=\"  -A 127.0.0.1\"",
+	"bgpd_options=\"   -A 127.0.0.1\"",
+	"bfdd_options=\"   -A 127.0.0.1\"",
+	"staticd_options=\"-A 127.0.0.1\"",
+	""
+].join("\n");
+/**
+* This function gives the configuration of a router. Graceful restart is off: after a hold expiry,
+* a helper can keep the stale routes, and the move that a cell measures does not come.
+* `maximum-paths 1` keeps one path, thus a move is one change of one route.
+*/
+const frrConf = (router, fabric) => {
+	const keepalive = valueIn(fabric.keepalive, "s");
+	const hold = valueIn(fabric.hold, "s");
+	const out = [
+		"frr defaults traditional",
+		`hostname ${router.name}`,
+		"ip forwarding",
+		"!"
+	];
+	for (const [name, address] of Object.entries(router.interfaces)) out.push(`interface ${name}`, ` ip address ${address}`, "!");
+	if (fabric.bfd !== null) {
+		const interval = valueIn(fabric.bfd.interval, "ms");
+		out.push("bfd", " profile fast", `  receive-interval ${interval}`, `  transmit-interval ${interval}`, `  detect-multiplier ${fabric.bfd.multiplier}`, " exit", "exit", "!");
+	}
+	out.push(`router bgp ${router.as}`, ` bgp router-id ${router.id}`, " no bgp ebgp-requires-policy", " no bgp network import-check", " bgp graceful-restart-disable");
+	for (const neighbor of router.neighbors) {
+		out.push(` neighbor ${neighbor.address} remote-as ${neighbor.as}`, ` neighbor ${neighbor.address} timers ${keepalive} ${hold}`, ` neighbor ${neighbor.address} timers connect 1`, ` neighbor ${neighbor.address} advertisement-interval 0`);
+		if (fabric.bfd !== null) out.push(` neighbor ${neighbor.address} bfd profile fast`);
+	}
+	out.push(" !", " address-family ipv4 unicast");
+	if (router.network !== void 0) out.push(`  network ${router.network}`);
+	out.push("  maximum-paths 1");
+	for (const neighbor of router.neighbors) if (neighbor.weight !== void 0) out.push(`  neighbor ${neighbor.address} weight ${neighbor.weight}`);
+	out.push(" exit-address-family", "exit", "");
+	return out.join("\n");
+};
+/**
+* This function gives the topology file of containerlab. The client and the target are containers
+* of the lab, and the fabric adds a device to each. Each link has the MTU of a Docker network, thus
+* a segment is as long as on layer E1. The default of containerlab is 9500. The management network
+* belongs to the lab, and Docker chooses its subnet.
+*/
+const topologyOf = (lab, dir, client, target) => {
+	const nodes = ROUTERS.flatMap((router) => [
+		`    ${router.name}:`,
+		"      kind: linux",
+		`      image: ${FRR_IMAGE}`,
+		"      binds:",
+		`        - ${dir}/${router.name}.conf:/etc/frr/frr.conf`,
+		`        - ${dir}/daemons:/etc/frr/daemons`,
+		"      sysctls:",
+		"        net.ipv4.ip_forward: 1",
+		"        net.ipv4.conf.all.rp_filter: 0",
+		"        net.ipv4.conf.default.rp_filter: 0"
+	]);
+	const links = [
+		...LINKS,
+		["leaf1:eth3", `${client}:${FABRIC_DEVICE}`],
+		["leaf2:eth3", `${target}:${FABRIC_DEVICE}`]
+	];
+	return [
+		`name: ${lab}`,
+		"mgmt:",
+		`  network: clab-${lab}`,
+		"  ipv4-subnet: auto",
+		"topology:",
+		"  nodes:",
+		...nodes,
+		`    ${client}:`,
+		"      kind: ext-container",
+		`    ${target}:`,
+		"      kind: ext-container",
+		"  links:",
+		...links.flatMap(([a, b]) => [`    - endpoints: ["${a}", "${b}"]`, "      mtu: 1500"]),
+		""
+	].join("\n");
+};
+/** This function reads a line of `ip -ts -o monitor route`, or it gives null for another line. The clock of a container is UTC. */
+const routeLine = (text) => {
+	const line = text.trim();
+	const found = /^\[(\d{4})-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)\.(\d{6})\] /.exec(line);
+	if (found === null) return null;
+	const [stamp, year, month, day, hour, minute, second, micro] = found;
+	const words = line.slice(stamp.length).split(" ").filter((word) => word !== "");
+	const change = words[0] === "Deleted" || words[0] === "Replaced" ? words[0] : null;
+	const prefix = words[change === null ? 0 : 1];
+	if (prefix === void 0) return null;
+	const after = (name) => {
+		const at = words.indexOf(name);
+		return at === -1 ? null : words[at + 1] ?? null;
+	};
+	return {
+		wall: Date.UTC(Number(year), Number(month) - 1, Number(day), Number(hour), Number(minute), Number(second)) + Number(micro) / 1e3,
+		deleted: change === "Deleted",
+		prefix,
+		via: after("via"),
+		dev: after("dev")
+	};
+};
+/**
+* This function takes a time on the wall clock to the monotonic clock. The harness runs on the host
+* of the containers, thus `process.hrtime` reads the CLOCK_MONOTONIC of the injector and the
+* prober. The error of the pair is approximately 1 ms, the step of `Date.now`.
+*/
+const monotonicAt = (wall, pair) => pair.mono + (wall - pair.wall) * 1e6;
+/** This function gives the state of the BGP sessions from `show bgp neighbors json`. */
+const sessionsOf = (json) => {
+	const views = Object.values(JSON.parse(json));
+	return {
+		up: views.filter((view) => view.bgpState === "Established").length,
+		established: views.reduce((sum, view) => sum + (view.connectionsEstablished ?? 0), 0)
+	};
+};
+/**
+* This function tells what a leaf still lacks before a run, or it gives null. The two sessions must
+* be up and the route must be on spine1. Spine2 must hold the other path. Without these, the fault
+* of a run acts on another path, or nothing can take over.
+*/
+const leafLacks = (views) => {
+	const { up } = sessionsOf(views.neighbors);
+	if (up < 2) return `${up} of 2 BGP sessions are up`;
+	if (!views.route.includes(` dev ${PREFERRED_DEVICE} `)) return `the route is not on spine1: ${views.route || "no route"}`;
+	const paths = JSON.parse(views.paths).paths?.length ?? 0;
+	if (paths < 2) return `BGP holds ${paths} of 2 paths`;
+	if (views.bfd !== null) {
+		const live = JSON.parse(views.bfd).filter((peer) => peer.status === "up").length;
+		if (live < 2) return `${live} of 2 BFD peers are up`;
+	}
+	return null;
+};
+/**
+* This function gives the later move of the leaves after an instant: for each leaf, the first route
+* to its far network that leaves spine1. It gives null when a leaf has no move. The later move ends
+* the outage in the two directions.
+*/
+const laterMove = (routes, after) => {
+	const firsts = LEAVES.map((leaf) => routes.find((route) => route.leaf === leaf.name && route.prefix === leaf.remote && route.t > after && !route.deleted && route.dev !== PREFERRED_DEVICE)?.t);
+	return firsts.every((t) => t !== void 0) ? Math.max(...firsts) : null;
+};
+/** This function runs a program on the host and gives its output. Its error holds the stderr of the program. */
+const host = (program, args) => new Promise((resolve, reject) => {
+	execFile(program, [...args], { maxBuffer: 16777216 }, (error, stdout, stderr) => {
+		if (error === null) resolve(stdout);
+		else reject(/* @__PURE__ */ new Error(`${program} ${args.join(" ")}: ${error.message}\n${stderr}`));
+	});
+});
+const nameOf = (container) => container.getName().replace(/^\//, "");
+/** This function gives a clock pair at this instant. */
+const clockPair = () => ({
+	mono: Number(process.hrtime.bigint()),
+	wall: Date.now()
+});
+/**
+* This function deploys the routers of a lab and joins the client and the target to them. It also
+* starts the sidecars: the injector of spine1 and a route monitor on each leaf. The caller stops the
+* sidecars with the other containers of the lab, and it calls `close` first.
+*/
+const openFabric = async (parts) => {
+	const { fabric, image, admin, injector, opened, output } = parts;
+	const { GenericContainer } = await import("testcontainers");
+	const lab = `e3-${randomBytes(4).toString("hex")}`;
+	const dir = await mkdtemp(join(tmpdir(), "fabric-"));
+	const file = join(dir, "lab.clab.yml");
+	await writeFile(join(dir, "daemons"), DAEMONS);
+	for (const router of ROUTERS) await writeFile(join(dir, `${router.name}.conf`), frrConf(router, fabric));
+	await writeFile(file, topologyOf(lab, dir, nameOf(parts.client), nameOf(parts.server)));
+	const close = async () => {
+		await host("sudo", [
+			"containerlab",
+			"destroy",
+			"-t",
+			file,
+			"--cleanup"
+		]).catch(() => void 0);
+		await rm(dir, {
+			recursive: true,
+			force: true
+		}).catch(() => void 0);
+	};
+	const node = (name) => `clab-${lab}-${name}`;
+	try {
+		await host("sudo", [
+			"containerlab",
+			"deploy",
+			"-t",
+			file
+		]);
+		const version = /version:\s*(\S+)/.exec(await host("containerlab", ["version"]))?.[1] ?? "unknown";
+		await output(admin, [
+			"ip",
+			"addr",
+			"add",
+			`${FABRIC_CLIENT}/24`,
+			"dev",
+			FABRIC_DEVICE
+		]);
+		await output(admin, [
+			"ip",
+			"route",
+			"replace",
+			TARGET_NET,
+			"via",
+			"10.1.0.1",
+			"dev",
+			FABRIC_DEVICE
+		]);
+		await output(injector, [
+			"ip",
+			"addr",
+			"add",
+			`${FABRIC_TARGET}/24`,
+			"dev",
+			FABRIC_DEVICE
+		]);
+		await output(injector, [
+			"ip",
+			"route",
+			"replace",
+			CLIENT_NET,
+			"via",
+			"10.2.0.1",
+			"dev",
+			FABRIC_DEVICE
+		]);
+		const silencer = await new GenericContainer(image).withNetworkMode(`container:${node("spine1")}`).withAddedCapabilities("NET_ADMIN", "NET_RAW").start();
+		opened.push(silencer);
+		const routes = [];
+		const monitors = /* @__PURE__ */ new Map();
+		for (const leaf of LEAVES) {
+			const monitor = await new GenericContainer(image).withNetworkMode(`container:${node(leaf.name)}`).withCommand([
+				"ip",
+				"-ts",
+				"-o",
+				"monitor",
+				"route"
+			]).withLogConsumer((stream) => {
+				let rest = "";
+				stream.on("data", (chunk) => {
+					const text = (rest + chunk.toString()).split("\n");
+					rest = text.pop() ?? "";
+					for (const each of text) {
+						const line = routeLine(each);
+						if (line === null) continue;
+						const { wall, ...route } = line;
+						routes.push({
+							...route,
+							leaf: leaf.name,
+							t: monotonicAt(wall, clockPair())
+						});
+					}
+				});
+			}).start();
+			opened.push(monitor);
+			monitors.set(leaf.name, monitor);
+		}
+		const stamp = async (action) => {
+			const last = (await output(silencer, ["/opt/harness/inject", action])).split("\n").at(-1);
+			const { t0, t1 } = JSON.parse(last);
+			return {
+				t0,
+				t1
+			};
+		};
+		const vtysh = (leaf, view) => host("docker", [
+			"exec",
+			node(leaf),
+			"vtysh",
+			"-c",
+			view
+		]);
+		return {
+			routes,
+			silence: () => stamp("silence"),
+			clear: () => stamp("clear-silence"),
+			async ready() {
+				const deadline = Date.now() + READY_MS;
+				for (;;) {
+					const lacks = [];
+					for (const leaf of LEAVES) {
+						const prefix = leaf.remote;
+						const lacking = await (async () => leafLacks({
+							neighbors: await vtysh(leaf.name, "show bgp neighbors json"),
+							route: `${await output(monitors.get(leaf.name), [
+								"ip",
+								"-o",
+								"route",
+								"show",
+								prefix
+							])} `,
+							paths: await vtysh(leaf.name, `show bgp ipv4 unicast ${prefix} json`),
+							bfd: fabric.bfd === null ? null : await vtysh(leaf.name, "show bfd peers json")
+						}))().catch((error) => String(error));
+						if (lacking !== null) lacks.push(`${leaf.name}: ${lacking}`);
+					}
+					if (lacks.length === 0) return;
+					if (Date.now() > deadline) throw new Error(`the fabric is not ready: ${lacks.join("; ")}`);
+					await new Promise((resolve) => setTimeout(resolve, READY_POLL_MS));
+				}
+			},
+			async sessions() {
+				const out = {};
+				for (const leaf of LEAVES) out[leaf.name] = sessionsOf(await vtysh(leaf.name, "show bgp neighbors json")).established;
+				return out;
+			},
+			manifest: () => ({
+				...fabric,
+				image: FRR_IMAGE,
+				containerlab: version
+			}),
+			close
+		};
+	} catch (error) {
+		await close();
+		throw error;
+	}
+};
+//#endregion
 //#region src/substrate.ts
 /** GitHub announced Linux jobs on AWS on this date. Each earlier hosted job ran on Azure. */
 const AWS_FROM = "2026-10-28";
@@ -2207,6 +2811,41 @@ const substrateOn = (layer, host = readHost()) => ({
 	...host
 });
 /**
+* This function gives the substrate of a lab at this place. A guest cannot read its own runner: its
+* DMI vendor is QEMU, and `ImageOS` is not set. Thus the runner writes its host to `LAB_RUNNER`, and
+* the guest keeps only its own kernel and HZ. A value of that variable makes the layer E4.
+*/
+const substrateHere = (layer, host = readHost(), env = process.env) => {
+	const runner = env["LAB_RUNNER"];
+	if (runner === void 0 || runner === "") return substrateOn(layer, host);
+	return substrateOn("E4", {
+		...JSON.parse(runner),
+		kernel: host.kernel,
+		hz: host.hz
+	});
+};
+/** This function reads the CPU time of this machine, or gives null without `/proc/stat`. `root` is for tests. */
+const readCpu = (root = "/") => {
+	const line = textOf(join(root, "proc", "stat"))?.split("\n")[0];
+	if (line === void 0 || !line.startsWith("cpu ")) return null;
+	const ticks = line.trim().split(/\s+/).slice(1, 9).map(Number);
+	const steal = ticks[7];
+	if (steal === void 0 || ticks.some(Number.isNaN)) return null;
+	return {
+		steal,
+		total: ticks.reduce((sum, tick) => sum + tick, 0)
+	};
+};
+/**
+* This function gives the share of the CPU time between two readings that the host took. The share
+* goes from zero to one. It is null if no tick passed. A guest sees only the host above it. On a hosted
+* runner that host is itself a guest, and this number does not show the contention of the layer below it.
+*/
+const stealShare = (before, after) => {
+	const total = after.total - before.total;
+	return total <= 0 ? null : (after.steal - before.steal) / total;
+};
+/**
 * This function reads the substrate of a record, the upgrade of a record from before this block.
 * Nothing rewrites such a record. It reads as the given layer. Its cloud is Azure if the record is
 * from before `AWS_FROM`, and unknown after that date, because its runner did not record the cloud.
@@ -2442,6 +3081,25 @@ const netnsOf = async (container) => {
 	if (inode === void 0) throw new Error(`not a network namespace: ${link}`);
 	return Number(inode);
 };
+/** This function gives the build record of a source as a part of a manifest: `{ build }`, or nothing for an image that the harness pulls. */
+const buildOf = (source) => {
+	const build = buildRecordOf(source);
+	return build === void 0 ? {} : { build };
+};
+/** This function gives the version of the harness and the build of its image, as a part of the harness record of a manifest. */
+const harnessRecord = () => ({
+	version: harnessVersion(),
+	...buildOf(shippedSource("harness"))
+});
+/**
+* This function gives the record of the client of a run that the harness ships, for a manifest:
+* its kind, its image, and its build.
+*/
+const shippedClientRecord = (kind, image) => ({
+	kind,
+	image,
+	...buildOf(SHIPPED_CLIENTS[kind].image)
+});
 /**
 * This function gives the image and the command of the client of each run, its kind for the
 * manifest, and the source of its image. A client by its name is a client that the harness ships
@@ -2492,7 +3150,9 @@ const openLab = async (target, kind = "tcp", topology) => {
 	const opened = [];
 	const networks = [];
 	const serverLines = lines();
+	let fabric;
 	const close = async () => {
+		await fabric?.close();
 		for (const container of opened.reverse()) await container.stop(STOP).catch(() => void 0);
 		for (const network of networks.reverse()) await network.stop().catch(() => void 0);
 	};
@@ -2633,7 +3293,18 @@ const openLab = async (target, kind = "tcp", topology) => {
 		opened.push(admin);
 		const injector = await new GenericContainer(image).withNetworkMode(`container:${server.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW").start();
 		opened.push(injector);
-		const clientIp = await ipOn(client, networks[0]);
+		if (topology?.fabric !== void 0) fabric = await openFabric({
+			fabric: topology.fabric,
+			image,
+			client,
+			server,
+			admin,
+			injector,
+			opened,
+			output
+		});
+		const device = fabric === void 0 ? "eth0" : FABRIC_DEVICE;
+		const clientIp = fabric === void 0 ? await ipOn(client, networks[0]) : FABRIC_CLIENT;
 		for (let at = 0; at < inChain.length; at += 1) {
 			const before = at === 0 ? clientIp : inChain[at - 1].ip.target;
 			inChain[at] = {
@@ -2716,7 +3387,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 		const begun = await records.until(["begin", "fail"], 3e4, "prober");
 		if (begun.ev === "fail") throw new Error(`the prober failed: ${String(begun["why"])}`);
 		const first = endpointAfter(hops, 0);
-		const targetIp = first === hops.length + 1 ? serverIp : inChain[first - 1].ip.client;
+		const targetIp = fabric !== void 0 ? FABRIC_TARGET : first === hops.length + 1 ? serverIp : inChain[first - 1].ip.client;
 		const targetPort = first === hops.length + 1 ? target.port : inChain[first - 1].hop.port;
 		const targetPeer = inChain.length === 0 ? clientIp : inChain.at(-1).ip.target;
 		const placed = inChain.length === 0 ? null : {
@@ -2769,8 +3440,8 @@ const openLab = async (target, kind = "tcp", topology) => {
 		};
 		/** This function gives the namespace and the device of netem at a place: an end, or the device of a hop toward a side. */
 		const deviceAt = (place) => {
-			if (place === "client") return [admin, "eth0"];
-			if (place === "target") return [injector, "eth0"];
+			if (place === "client") return [admin, device];
+			if (place === "target") return [injector, device];
 			const at = hopPlace(place);
 			const hop = hopNamed(at.hop);
 			return [hop.injector, hop.devices[at.toward]];
@@ -2797,6 +3468,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 			middlebox,
 			server: serverLines.seen,
 			kernel: records.seen,
+			...fabric === void 0 ? {} : { fabric },
 			async sysctl(values) {
 				const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);
 				if (settings.length > 0) await output(admin, [
@@ -3146,7 +3818,8 @@ const openLab = async (target, kind = "tcp", topology) => {
 					})(),
 					boot_id: await output(client, ["cat", "/proc/sys/kernel/random/boot_id"]),
 					docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
-					substrate: substrateOn("E1"),
+					substrate: substrateHere(fabric === void 0 ? "E1" : "E3"),
+					...fabric === void 0 ? {} : { fabric: fabric.manifest() },
 					target: {
 						image: targetImage,
 						...target.subject === void 0 ? {} : {
@@ -3157,7 +3830,8 @@ const openLab = async (target, kind = "tcp", topology) => {
 						settings,
 						sysctls: targetSysctls,
 						...target.files === void 0 ? {} : { files: Object.fromEntries(Object.entries(target.files).map(([path, content]) => [path, createHash("sha256").update(content).digest("hex")])) },
-						...serverLines.seen[0] === void 0 ? {} : { first: serverLines.seen[0] }
+						...serverLines.seen[0] === void 0 ? {} : { first: serverLines.seen[0] },
+						...buildOf(targetSourceOf(target))
 					},
 					harness: {
 						image,
@@ -3166,12 +3840,16 @@ const openLab = async (target, kind = "tcp", topology) => {
 							"apk",
 							"info",
 							"-v"
-						])).split("\n").sort()
+						])).split("\n").sort(),
+						...harnessRecord()
 					},
 					client: {
 						kind: clientKind,
 						image: clientImage,
-						...source === void 0 ? {} : { source }
+						...source === void 0 ? {} : {
+							source,
+							...buildOf(source)
+						}
 					},
 					netns,
 					sysctls,
@@ -3186,6 +3864,54 @@ const openLab = async (target, kind = "tcp", topology) => {
 	}
 };
 //#endregion
+//#region src/pins.ts
+/**
+* This module tells if a manifest pins each program that a run operated. A record that does not pin
+* them is not a record. A later run cannot operate the same programs, thus nobody can reproduce
+* its numbers (SYS-101). Each image is pulled by its digest, or the harness builds it. A build
+* records its bases by digest, the sha256 of each lockfile, and the versions of the packages that
+* its manifests name. The commit of the code that operated is a field of the record of a cell, and
+* `commitProblems` examines it.
+*/
+/** This pattern matches the id of an image: `sha256:` and 64 hex digits. */
+const IMAGE_ID = /^sha256:[0-9a-f]{64}$/;
+/** This pattern matches the tag of a build of the harness: its name, then 12 hex digits of the sha256 of its context. */
+const BUILD_TAG = /^[a-z0-9][a-z0-9._\-/]*:[0-9a-f]{12}$/;
+/** This pattern matches a commit of git: 40 hex digits. */
+const COMMIT = /^[0-9a-f]{40}$/;
+/** This function gives each problem of the record of a build. A problem is a base that no digest pins, or a tag of another form. */
+const buildProblems = (what, build) => [...BUILD_TAG.test(build.tag) ? [] : [`${what}: the build has the tag ${build.tag}, which is not a tag of a build`], ...build.bases.filter((base) => !PINNED.test(base)).map((base) => `${what}: the base ${base} is not pinned by digest`)];
+/**
+* This function gives each problem of an image of a manifest. A pulled image needs a digest. An
+* image that the harness builds needs its build record when the manifest can hold one (`build`).
+* Otherwise, it needs the tag of a build and the id of its image.
+*/
+const pinProblems = (what, image, id, build, needsBuild) => {
+	if (PINNED.test(image)) return [];
+	if (build !== void 0) return buildProblems(what, build);
+	if (BUILD_TAG.test(image) && !needsBuild && id !== void 0 && IMAGE_ID.test(id)) return [];
+	return BUILD_TAG.test(image) && needsBuild ? [`${what}: the image ${image} is a build with no build record`] : [`${what}: the image ${image} has no digest`];
+};
+/**
+* This function gives each problem of a manifest that stops it from pinning the programs of its
+* run. An empty list means that the manifest pins each program.
+* - the target: an image by its digest, or the build of a subject with its build record and its base
+*   by digest.
+* - the harness: the version of its package, and the build record of its image.
+* - the client: an image by its digest, or a build with its build record.
+* - each hop: an image by its digest, or the tag of a build and the id of its image.
+*/
+const manifestProblems = (manifest) => [
+	...pinProblems("the target", manifest.target.image, manifest.target.id, manifest.target.build, manifest.target.subject !== void 0),
+	...manifest.target.subject !== void 0 && (manifest.target.base === void 0 || !PINNED.test(manifest.target.base)) ? [`the target: the base ${manifest.target.base ?? "(none)"} of its subject is not pinned by digest`] : [],
+	...manifest.harness.version === void 0 ? ["the harness: the manifest has no version of the harness"] : [],
+	...manifest.harness.build === void 0 ? ["the harness: the image has no build record"] : buildProblems("the harness", manifest.harness.build),
+	...pinProblems("the client", manifest.client.image, void 0, manifest.client.build, true),
+	...(manifest.hops ?? []).flatMap((hop) => pinProblems(`the hop ${hop.name}`, hop.image, hop.id, void 0, false))
+];
+/** This function gives the problem of the commit of a record: no commit, or a text that is not a commit of git. */
+const commitProblems = (commit) => commit === void 0 ? ["the record has no commit of the code that operated"] : COMMIT.test(commit) ? [] : [`the record has the commit ${commit}, which is not 40 hex digits`];
+//#endregion
 //#region src/group.ts
 /** This function gives the configuration file of a member: its port, its settings, and the primary that a replica follows. */
 const memberConfig = (group, primary) => [
@@ -3675,12 +4401,10 @@ const openGroupLab = async (group, options = {}) => {
 							"apk",
 							"info",
 							"-v"
-						])).split("\n").sort()
-					},
-					client: {
-						kind: "ioredis",
-						image: node
+						])).split("\n").sort(),
+						...harnessRecord()
 					},
+					client: shippedClientRecord("ioredis", node),
 					netns: {
 						client: clientNetns,
 						target: parts.members[first].netns
@@ -3989,6 +4713,7 @@ const statusViewOf = (set, ips, status) => {
 		why: "healthy"
 	};
 };
+/** This function waits for a time, in ms. */
 const sleep$2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** These are the arguments of replSetStepDown: the time that the old primary cannot stand again, and the wait for a secondary to catch up. */
 const STEP_DOWN = {
@@ -4004,8 +4729,9 @@ const STEP_DOWN = {
 * - an injector sidecar for each member, in its network and process namespaces.
 * - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
 * - the prober, which watches the namespace of the client.
+* - a backup host for a recovery drill: the stock image, for its tools, with no server.
 */
-const openReplicaSetLab = async (set) => {
+const openReplicaSetLab = async (set, backups = false) => {
 	const image = await harnessImage();
 	const mongo = await mongoClientImage();
 	const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();
@@ -4064,6 +4790,12 @@ const openReplicaSetLab = async (set) => {
 			"while :; do sleep 3600; done"
 		]).start();
 		opened.push(commander);
+		const backupHost = backups ? await new GenericContainer(set.image).withNetwork(network).withCommand([
+			"sh",
+			"-c",
+			"while :; do sleep 3600; done"
+		]).start() : null;
+		if (backupHost !== null) opened.push(backupHost);
 		const clientNetns = await netnsOf(admin);
 		const records = lines();
 		const prober = await new GenericContainer(image).withPrivilegedMode().withNetworkMode("none").withCommand(["/opt/harness/prober", String(clientNetns)]).withLogConsumer(records.consume).start();
@@ -4081,11 +4813,11 @@ const openReplicaSetLab = async (set) => {
 			return found;
 		};
 		/** This function starts a script of the shell on a member, against its own server only. */
-		const shell = (name, script) => output(box(name), [
+		const shell = (name, script, port = set.port) => output(box(name), [
 			set.programs.shell,
 			"--quiet",
 			"--norc",
-			`mongodb://127.0.0.1:${set.port}/?directConnection=true&serverSelectionTimeoutMS=3000`,
+			`mongodb://127.0.0.1:${port}/?directConnection=true&serverSelectionTimeoutMS=3000`,
 			"--eval",
 			script
 		]);
@@ -4239,6 +4971,14 @@ const openReplicaSetLab = async (set) => {
 					await sleep$2(500);
 				}
 			},
+			exec(member, argv) {
+				return output(box(member), argv);
+			},
+			script: shell,
+			vault(argv) {
+				if (backupHost === null) throw new Error("this lab has no backup host");
+				return output(backupHost, argv);
+			},
 			stepDown(primary) {
 				return stamped(commander, [
 					"node",
@@ -4291,7 +5031,7 @@ const openReplicaSetLab = async (set) => {
 					architecture: await output(holder, ["uname", "-m"]),
 					hz: /^CONFIG_HZ=(\d+)$/m.exec(config)?.[1] ?? null,
 					boot_id: await output(holder, ["cat", "/proc/sys/kernel/random/boot_id"]),
-					substrate: substrateOn("E1"),
+					substrate: substrateHere("E1"),
 					docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
 					target: {
 						image: set.image,
@@ -4306,12 +5046,10 @@ const openReplicaSetLab = async (set) => {
 							"apk",
 							"info",
 							"-v"
-						])).split("\n").sort()
-					},
-					client: {
-						kind: "mongodb",
-						image: mongo
+						])).split("\n").sort(),
+						...harnessRecord()
 					},
+					client: shippedClientRecord("mongodb", mongo),
 					netns: {
 						client: clientNetns,
 						target: members[set.members[0]].netns
@@ -4331,8 +5069,6 @@ const openReplicaSetLab = async (set) => {
 		throw error;
 	}
 };
-/** This count is the number of accepted writes that a run waits for before its fault: the stream is steady then. */
-const STEADY$1 = 20;
 /** This time is how long a run continues after its outcome, so that the record shows what came next. */
 const TAIL_MS$2 = 2e3;
 /** This function gives the arguments of the client of a replica set cell: each member, and the stream of writes. */
@@ -4351,7 +5087,7 @@ const replicaSetArgs = (cell, members) => {
 		"--interval-ms",
 		String(cell.probe.interval === void 0 ? 50 : valueIn(cell.probe.interval, "ms")),
 		"--max-s",
-		String(Math.ceil(deadlineOf(cell) / 1e3 + (cell.hold === void 0 ? 0 : valueIn(cell.hold, "s")) + 120)),
+		String(Math.ceil(deadlineOf(cell) / 1e3 + (cell.hold === void 0 ? 0 : valueIn(cell.hold, "s")) + (cell.drill === void 0 ? 0 : drillSpan(cell.drill)) + 120)),
 		"--w",
 		String(cell.writeConcern ?? "majority"),
 		...cell.driverOptions === void 0 ? [] : ["--options", JSON.stringify(cell.driverOptions)]
@@ -4403,7 +5139,7 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 	const undo = [];
 	try {
 		await client.until(["ready"], 6e4);
-		if (await waitFor$1(client, (event) => event.ev === "ack" && Number(event["n"]) >= STEADY$1, 3e4) === void 0) throw new Error(`the client did not write ${STEADY$1} times in 30 s`);
+		if (await waitFor$1(client, (event) => event.ev === "ack" && Number(event["n"]) >= 20, 3e4) === void 0) throw new Error(`the client did not write 20 times in 30 s`);
 		let healing = null;
 		switch (cell.fault) {
 			case "blackhole":
@@ -4487,6 +5223,296 @@ const replicaSetOnce = async (lab, cell, environment, index) => {
 	};
 };
 //#endregion
+//#region src/drill.ts
+/**
+* This module does one run of a recovery drill on the lab of a replica set. A drill stops the store
+* during writes, removes its disk, and recovers from real backups, or by an election.
+*
+* A backup is a real `mongodump`, on a host of its own. The dump has the oplog of its own time, and
+* each slice has the oplog after the backup before it. The fault comes at the age of the newest
+* backup, on the clock of the VM. Thus a run measures the worst case of its interval by construction.
+*
+* A restore cannot have a client of the set: the client writes to any primary. So the restore writes
+* to a server with no set, on its own port. That server then stops, each member starts with an empty
+* set, and the old primary starts the set. Only that member has the data, thus it wins.
+*
+* The driver refuses a primary with an election id under the highest one that it saw, and a new
+* set starts its ids again. So a restore has two clients. The first one writes up to the fault, and
+* the second one starts after the restore, with the sequence numbers that follow the first.
+*/
+/** This folder is the store of the backups, on the backup host. */
+const VAULT = "/backup";
+/** This time is the longest wait for a server to answer, in ms. */
+const STARTS_WITHIN = 6e4;
+/** This function waits until a check holds, or it stops the run after `within` ms. */
+const until = async (what, check, within) => {
+	const started = Date.now();
+	while (!await check().catch(() => false)) {
+		if (Date.now() - started > within) throw new Error(`${what} did not come in ${within} ms`);
+		await sleep$2(500);
+	}
+};
+/** This function gives the time that the client sent a write. Zero is no write. */
+const writeOf = (events, n) => {
+	const sent = n === 0 ? void 0 : events.find((event) => event.ev === "sent" && Number(event["n"]) === n);
+	return sent === void 0 ? null : {
+		n,
+		t: sent.t
+	};
+};
+/** This function gives the size of a folder of the backup host. */
+const sizeOf = async (lab, path) => {
+	const [bytes] = (await lab.vault([
+		"du",
+		"-sb",
+		path
+	])).split(/\s+/);
+	return {
+		value: Number(bytes),
+		unit: "B"
+	};
+};
+/** This function takes the dump with its oplog, and each slice of the oplog at its interval. It gives each backup in sequence. */
+const backupsOf = async (lab, drill, primary, writer) => {
+	const host = `${lab.members[primary].ip}:${lab.set.port}`;
+	const started = Date.now();
+	await lab.vault([
+		"mongodump",
+		"--host",
+		host,
+		"--oplog",
+		"--out",
+		`${VAULT}/dump-0`
+	]);
+	const ended = Date.now();
+	const oplog = readOplog(await lab.vault([
+		"bsondump",
+		"--quiet",
+		`${VAULT}/dump-0/oplog.bson`
+	]));
+	const held = highestHeld(await lab.vault([
+		"bsondump",
+		"--quiet",
+		`${VAULT}/dump-0/cells/w.bson`
+	]));
+	if (oplog.last === null) throw new Error("the oplog of the dump has no entry");
+	let highest = Math.max(oplog.highest ?? 0, held ?? 0);
+	let from = oplog.last;
+	const backups = [{
+		kind: "dump",
+		started,
+		ended,
+		last: writeOf(writer.events, highest),
+		size: await sizeOf(lab, `${VAULT}/dump-0`)
+	}];
+	const every = valueIn(drill.sliceEvery, "s") * 1e3;
+	for (let slice = 1; slice <= drill.slices; slice += 1) {
+		await sleep$2(Math.max(0, ended + slice * every - Date.now()));
+		const out = `${VAULT}/slice-${slice}`;
+		const begun = Date.now();
+		await lab.vault([
+			"mongodump",
+			"--host",
+			host,
+			"--db",
+			"local",
+			"--collection",
+			"oplog.rs",
+			"--query",
+			sliceQuery(from),
+			"--out",
+			out
+		]);
+		const finished = Date.now();
+		const read = readOplog(await lab.vault([
+			"bsondump",
+			"--quiet",
+			`${out}/local/oplog.rs.bson`
+		]));
+		highest = Math.max(highest, read.highest ?? 0);
+		from = read.last ?? from;
+		backups.push({
+			kind: "slice",
+			started: begun,
+			ended: finished,
+			last: writeOf(writer.events, highest),
+			size: await sizeOf(lab, out)
+		});
+	}
+	return backups;
+};
+/**
+* This function stops the server of each member that the drill loses, and it removes each disk. A
+* restore loses each member, and an election loses the primary. The fault comes at the age of the
+* drill after the last write of the newest backup. A control has no later write, and an election
+* has no backup: their age counts from this point. The stamp is the one of the primary.
+*/
+const lose = async (lab, drill, primary, last) => {
+	const parts = drill.recovery === "restore" ? [primary, ...lab.set.members.filter((name) => name !== primary)] : [primary];
+	const age = valueIn(drill.age, "s");
+	const at = last === null || drill.quiet === true ? void 0 : last.t + age * 1e9;
+	if (at === void 0) await sleep$2(age * 1e3);
+	const stamps = await Promise.all(parts.map((part) => lab.signal(part, "KILL", at)));
+	for (const part of parts) {
+		await lab.exec(part, [
+			"sh",
+			"-c",
+			stopScript(false)
+		]);
+		await lab.exec(part, [
+			"find",
+			"/data/db",
+			"-mindepth",
+			"1",
+			"-delete"
+		]);
+	}
+	return stamps[0];
+};
+/**
+* This function restores the old primary from the backups and starts the set again:
+* - The slices go after the oplog of the dump, in their sequence, thus one replay does the whole restore.
+* - The restore writes to a server of the old primary with no set, on its own port.
+* - Each member that lost its disk starts before the set, because `rs.initiate` asks each member.
+* - The old primary starts the set. It has the data, and the other members make their first sync from it.
+*/
+const restore = async (lab, drill, primary) => {
+	const { set } = lab;
+	const started = Date.now();
+	const port = set.port + 1;
+	const slices = Array.from({ length: drill.slices }, (_, at) => `${VAULT}/slice-${at + 1}/local/oplog.rs.bson`);
+	if (slices.length > 0) await lab.vault([
+		"sh",
+		"-c",
+		`cat ${slices.join(" ")} >> ${VAULT}/dump-0/oplog.bson`
+	]);
+	await lab.exec(primary, [
+		"sh",
+		"-c",
+		`${standaloneCommand(set, port)} > /proc/1/fd/1 2>&1 & echo $! > /tmp/server.pid`
+	]);
+	await until("the server of the restore", async () => (await lab.script(primary, "print(db.hello().ok)", port)).includes("1"), STARTS_WITHIN);
+	await lab.vault([
+		"mongorestore",
+		"--host",
+		`${lab.members[primary].ip}:${port}`,
+		"--oplogReplay",
+		`${VAULT}/dump-0`
+	]);
+	await lab.exec(primary, [
+		"sh",
+		"-c",
+		stopScript(true)
+	]);
+	for (const name of set.members) if (name !== primary) await lab.restart(name);
+	await lab.restart(primary);
+	const ips = Object.fromEntries(Object.entries(lab.members).map(([name, at]) => [name, at.ip]));
+	const initiated = await lab.script(primary, initiateScript(set, ips));
+	if (!initiated.includes("\"ok\":1")) throw new Error(`rs.initiate failed after the restore: ${initiated}`);
+	await until("the primary after the restore", async () => (await lab.script(primary, "print(db.hello().isWritablePrimary)")).includes("true"), STARTS_WITHIN);
+	return {
+		started,
+		ended: Date.now()
+	};
+};
+/**
+* This function does one run of a recovery drill:
+* - The set converges, and it loses its writes. The tenants get their data, and a hash reads it.
+* - The client writes until its stream is steady. A control stops it here.
+* - The backups go, each at its interval. The fault comes at the age of the newest backup.
+* - An election continues the same client. A restore stops the client, and it starts a second one.
+* - The set converges again. The writes that it then holds tell which writes it kept, and a second
+*   hash tells if the tenants have the same data.
+*/
+const drillOnce = async (lab, cell, environment, index) => {
+	const drill = cell.drill;
+	const { set } = lab;
+	const primary = await lab.converge();
+	await lab.flush(primary);
+	await lab.unfreeze(primary);
+	await lab.forget();
+	await lab.vault([
+		"sh",
+		"-c",
+		`rm -rf ${VAULT} && mkdir -p ${VAULT}`
+	]);
+	const seeded = await lab.script(primary, seedScript(drill, set.members.length));
+	if (!seeded.includes("\"ok\":true")) throw new Error(`the seed failed: ${seeded}`);
+	const before = hashOf(await lab.script(primary, HASH_SCRIPT));
+	const tool = (await lab.vault(["mongodump", "--version"])).split("\n")[0].trim();
+	const from = lab.kernel.length;
+	const logFrom = lab.log.length;
+	const lostBefore = droppedSoFar$2(lab.kernel);
+	const writer = await lab.client(replicaSetArgs(cell, lab.members));
+	const restores = drill.recovery === "restore";
+	let successor = null;
+	let second = null;
+	let fault = null;
+	let restored = null;
+	const backups = [];
+	try {
+		await writer.until(["ready"], 6e4);
+		if (await waitFor$1(writer, (event) => event.ev === "ack" && Number(event["n"]) >= 20, 3e4) === void 0) throw new Error(`the client did not write 20 times in 30 s`);
+		if (drill.quiet === true) await writer.stop();
+		if (restores) backups.push(...await backupsOf(lab, drill, primary, writer));
+		const stamp = await lose(lab, drill, primary, backups.at(-1)?.last ?? null);
+		fault = stamp;
+		if (restores) {
+			await writer.stop();
+			restored = await restore(lab, drill, primary);
+			second = Math.max(0, ...writer.events.filter((event) => event.ev === "sent").map((event) => Number(event["n"]))) + 1;
+			successor = await lab.client([
+				...replicaSetArgs(cell, lab.members),
+				"--from",
+				String(second)
+			]);
+			if (await waitFor$1(successor, (event) => event.ev === "ack", deadlineOf(cell)) !== void 0) await sleep$2(TAIL_MS$2);
+		} else {
+			const old = `${lab.members[primary].ip}:${set.port}`;
+			const sentAt = (n) => writer.events.find((event) => event.ev === "sent" && Number(event["n"]) === n)?.t ?? -Infinity;
+			if (await waitFor$1(writer, (event) => event.ev === "ack" && sentAt(Number(event["n"])) > stamp.t1 && typeof event["at"] === "string" && event["at"] !== old, deadlineOf(cell)) !== void 0) await sleep$2(TAIL_MS$2);
+			await lab.restart(primary);
+		}
+	} finally {
+		await writer.stop();
+		await successor?.stop();
+	}
+	const settled = await lab.settle();
+	const lost = settled === null ? null : settled - lostBefore;
+	const after = await lab.converge().catch(() => null);
+	const holds = after === null ? null : await lab.holds(after);
+	const hash = after === null ? null : hashOf(await lab.script(after, HASH_SCRIPT));
+	const events = [...writer.events, ...successor?.events ?? []].sort((a, b) => a.t - b.t);
+	return {
+		environment,
+		index,
+		events,
+		fault,
+		kernel: lab.kernel.slice(from),
+		netns: lab.netns.client,
+		targetNetns: lab.members[primary].netns,
+		lost,
+		group: {
+			members: lab.members,
+			sentinels: {},
+			primary,
+			at: primary,
+			timeline: onMonotonic(lab.log.slice(logFrom), clockOffsetOf(events)),
+			after,
+			holds,
+			drill: {
+				tool,
+				seed: "generated",
+				backups,
+				before,
+				after: hash,
+				restored,
+				second
+			}
+		}
+	};
+};
+//#endregion
 //#region src/cluster.ts
 /** This form is a line of the console log of RabbitMQ 4.3: the time with its offset, the level, the process and the message. */
 const LINE = /^(\d{4}-\d\d-\d\d) (\d\d:\d\d:\d\d)\.(\d{3})(\d*)([+-]\d\d:\d\d) \[(\w+)\] <[^>]*> (.*)$/;
@@ -4979,7 +6005,7 @@ const openClusterLab = async (cluster) => {
 					architecture: await output(holder, ["uname", "-m"]),
 					hz: /^CONFIG_HZ=(\d+)$/m.exec(config)?.[1] ?? null,
 					boot_id: await output(holder, ["cat", "/proc/sys/kernel/random/boot_id"]),
-					substrate: substrateOn("E1"),
+					substrate: substrateHere("E1"),
 					docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,
 					target: {
 						image: cluster.image,
@@ -4994,12 +6020,10 @@ const openClusterLab = async (cluster) => {
 							"apk",
 							"info",
 							"-v"
-						])).split("\n").sort()
-					},
-					client: {
-						kind: "amqp",
-						image: amqp
+						])).split("\n").sort(),
+						...harnessRecord()
 					},
+					client: shippedClientRecord("amqp", amqp),
 					netns: {
 						client: clientNetns,
 						target: members[cluster.members[0]].netns
@@ -5367,6 +6391,8 @@ const requestSent = (probe, within) => new Promise((resolve, reject) => {
 });
 /** This time is how long a run of a driver cell continues after its outcome, so that the record shows what came next. */
 const TAIL_MS = 1e3;
+/** This time lets the last lines of a route monitor reach the harness after a stall ends. */
+const MONITOR_MS = 200;
 /** This function waits for the outcome of a run of a driver cell, or for `within` ms. The outcome is the end of its band and of each later band. */
 const driverOutcome = async (cell, current, within) => {
 	const started = Date.now();
@@ -5474,6 +6500,9 @@ const netemSpec = (cell, fault) => {
 */
 const once = async (lab, cell, environment, index, dry = false) => {
 	if (cell.before === true) return beforeOnce(lab, cell, environment, index);
+	await lab.fabric?.ready();
+	const sessions = await lab.fabric?.sessions();
+	const routesFrom = lab.fabric?.routes.length ?? 0;
 	await lab.forget();
 	const from = lab.kernel.length;
 	const middleboxFrom = lab.middlebox.length;
@@ -5493,6 +6522,10 @@ const once = async (lab, cell, environment, index, dry = false) => {
 		throw new Error(`${error.message}; the client's last lines: ${last.join(" ")}`);
 	});
 	const placed = lab.topology === null ? {} : { topology: lab.topology };
+	const fabricOf = () => {
+		if (lab.fabric === void 0) throw new Error(`${cell.name} silences a router of a lab with no fabric`);
+		return lab.fabric;
+	};
 	/** This function gives the run as the records show it so far: the waits read it. */
 	const partial = () => ({
 		environment,
@@ -5566,6 +6599,7 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					case "server-close":
 						await lab.serverClose();
 						return null;
+					case "silence": return fabricOf().silence();
 					case "hop-event": {
 						if (cell.event === void 0) throw new Error(`${cell.name} names no event`);
 						const stamp = await lab.event(cell.event);
@@ -5623,6 +6657,9 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					case "toxic":
 						await lab.toxic(null);
 						return;
+					case "silence":
+						await fabricOf().clear();
+						return;
 					default: return heal === null ? lab.lift("blackhole", faultAt) : void 0;
 				}
 			};
@@ -5654,6 +6691,21 @@ const once = async (lab, cell, environment, index, dry = false) => {
 						"eof",
 						"drop"
 					], window * 1e3).catch(() => void 0);
+				} else if (cell.fault === "silence") {
+					const window = valueIn(prediction.detects ? prediction.high : prediction.window, "s");
+					await requestSent(probe, 5e3);
+					await probe.until([
+						"data",
+						"error",
+						"eof",
+						"drop"
+					], window * 1e3).catch(() => void 0);
+					await sleep(MONITOR_MS);
+					const moved = laterMove(fabricOf().routes.slice(routesFrom), fault?.t0 ?? 0);
+					heal = moved === null ? null : {
+						t0: moved,
+						t1: moved
+					};
 				} else if (prediction.detects === null) {
 					const window = valueIn(prediction.window, "s") * 1e3;
 					await probe.until([
@@ -5695,14 +6747,21 @@ const once = async (lab, cell, environment, index, dry = false) => {
 		index,
 		events: [...probe.events],
 		fault,
-		...cell.fault === "stall" || thawing !== null ? { heal } : {},
+		...cell.fault === "stall" || cell.fault === "silence" || thawing !== null ? { heal } : {},
 		kernel: lab.kernel.slice(from),
 		netns: lab.netns.client,
 		targetNetns: lab.netns.target,
 		lost,
 		...placed,
 		...lab.topology === null ? {} : { middlebox: lab.middlebox.slice(middleboxFrom) },
-		...cell.target.lines === true ? { server: lab.server.slice(serverFrom) } : {}
+		...cell.target.lines === true ? { server: lab.server.slice(serverFrom) } : {},
+		...lab.fabric === void 0 || sessions === void 0 ? {} : { fabric: {
+			routes: lab.fabric.routes.slice(routesFrom),
+			established: {
+				before: sessions,
+				after: await lab.fabric.sessions()
+			}
+		} }
 	};
 };
 /** This function divides a count of runs across environments, as equally as possible. */
@@ -5714,6 +6773,17 @@ const spread = (runs, environments) => Array.from({ length: environments }, (_,
 const endsEnvironment = (cell) => cell.fresh === true || cell.fault === "fin" && (cell.observer === void 0 || cell.observer === "client");
 /** This function says one line for a run. */
 const line = (cell, run, read) => `${cell.name}: env ${run.environment} ${run.warmup === true ? "warm-up" : `run ${run.index}`} ${read.outcome.kind}${read.elapsed === null ? "" : ` ${read.elapsed.value} s`}${read.void === null ? "" : ` (void: ${read.void})`}`;
+/** This function adds to a run the share of the CPU that the host took while the run operated. */
+const withSteal = async (work) => {
+	const before = readCpu();
+	const run = await work();
+	const after = readCpu();
+	const steal = before === null || after === null ? null : stealShare(before, after);
+	return steal === null ? run : {
+		...run,
+		steal
+	};
+};
 /**
 * This function does the part of the runs of a cell in one environment, and records its manifest.
 * A warm-up goes first, and the verdict does not count it. On a loaded machine, a new environment is
@@ -5743,13 +6813,13 @@ const inEnvironment = async (cell, environment, count, say) => {
 			jitter: jitterOf(cell, run)
 		});
 		const warmup = {
-			...timed(await once(lab, cell, environment, -1, endsEnvironment(cell))),
+			...timed(await withSteal(() => once(lab, cell, environment, -1, endsEnvironment(cell)))),
 			warmup: true
 		};
 		say(line(cell, warmup, measure(cell, warmup)));
 		const runs = [];
 		for (let index = 0; index < count; index += 1) {
-			const run = timed(await once(lab, cell, environment, index));
+			const run = timed(await withSteal(() => once(lab, cell, environment, index)));
 			runs.push(run);
 			say(line(cell, run, measure(cell, run)));
 		}
@@ -5796,17 +6866,18 @@ const inGroupEnvironment = async (cell, environment, count, say) => {
 * is a whole run with its fault, as for a group under Sentinels. The set converges before each run.
 */
 const inReplicaSetEnvironment = async (cell, environment, count, say) => {
-	const lab = await openReplicaSetLab(cell.replicaSet);
+	const lab = await openReplicaSetLab(cell.replicaSet, cell.drill !== void 0);
+	const once = cell.drill === void 0 ? replicaSetOnce : drillOnce;
 	try {
 		const manifest = await lab.manifest(await lab.sysctl(cell.sysctls));
 		const warmup = {
-			...await replicaSetOnce(lab, cell, environment, -1),
+			...await once(lab, cell, environment, -1),
 			warmup: true
 		};
 		say(line(cell, warmup, measure(cell, warmup)));
 		const runs = [];
 		for (let index = 0; index < count; index += 1) {
-			const run = await replicaSetOnce(lab, cell, environment, index);
+			const run = await once(lab, cell, environment, index);
 			runs.push(run);
 			say(line(cell, run, measure(cell, run)));
 		}
@@ -5901,6 +6972,6 @@ const runCell = async (cell, say = () => {}) => {
 	}
 };
 //#endregion
-export { AWS_FROM, CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, DROP_HANDLING, GIVES_AWAY, INJECT, KEPT, LINGER_MS, LISTING, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, callOf, clockOffsetOf, closeOrExit, cloudOf, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, comparable, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, dropFinding, dropOf, endpointAfter, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, goHttpImage, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isHopPlace, jitterOf, jitterOfLines, judge, lagsOf, latestEndOf, leftAfter, logNameOf, matches, measure, measureDriver, measureGroup, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicOf, nodeClientImage, nodeCommand, nodeHttpImage, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, onMonotonic, openClusterLab, openGroupLab, openLab, openReplicaSetLab, pairingOf, pairingsOf, peersOf, pgClientImage, pingsOf, plainOf, planOf, programLines, programOf, quantileOf, readHost, reconnectFinding, reconnectOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routerImage, routesOf, runCell, sentinelConfig, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, statusViewOf, subjectClient, subjectProblems, subscriptionsOf, substrateKey, substrateOf, substrateOn, targetImageOf, targetSourceOf, templated, toolJsonOf, toxicBody, unlockedManifests, unpinnedBases, unpoolable, verdictOf, viewOf, writesOf };
+export { AWS_FROM, CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, DAEMONS, DROP_HANDLING, FABRIC_CLIENT, FABRIC_DEVICE, FABRIC_TARGET, FRR_IMAGE, GIVES_AWAY, HASH_SCRIPT, INJECT, KEPT, LINGER_MS, LISTING, LOCK_NAMES, LOSS_MARGIN, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, ROUTERS, SEED_DOCUMENT, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, basesOf, buildOf, buildRecordOf, callOf, cargoVersions, clockOffsetOf, closeOrExit, cloudOf, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, commitProblems, comparable, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, drillOnce, drillSpan, dropFinding, dropOf, endpointAfter, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, frrConf, goHttpImage, goVersions, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, harnessRecord, harnessVersion, hashOf, highestHeld, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isHopPlace, jarVersions, jitterOf, jitterOfLines, judge, lagsOf, laterMove, latestEndOf, leafLacks, leftAfter, locksOf, logNameOf, manifestProblems, matches, measure, measureDriver, measureGroup, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicAt, monotonicOf, nodeClientImage, nodeCommand, nodeHttpImage, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, npmVersions, onMonotonic, openClusterLab, openFabric, openGroupLab, openLab, openReplicaSetLab, pairingOf, pairingsOf, peersOf, pgClientImage, pingsOf, plainOf, planOf, programLines, programOf, quantileOf, readCpu, readHost, readOplog, reconnectFinding, reconnectOf, recoveryFindings, recoveryOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routeLine, routerImage, routesOf, runCell, seedScript, sentinelConfig, sessionsOf, shippedClientRecord, shippedSource, sliceQuery, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, standaloneCommand, statusViewOf, stealShare, stopScript, subjectClient, subjectProblems, subscriptionsOf, substrateHere, substrateKey, substrateOf, substrateOn, targetImageOf, targetSourceOf, templated, toolJsonOf, topologyOf, toxicBody, unlockedManifests, unpinnedBases, unpoolable, verdictOf, versionsOf, viewOf, writesOf };
 
 //# sourceMappingURL=index.js.map