@mark1russell7/harness 0.15.0 → 0.15.2

package/build/index.d.ts

@@ -1,4 +1,4 @@
-import { $ as ProxyHop, A as ImageSource, At as Subject, B as LoadStream, C as Hop, Ct as RunTopology, D as HopWrite, Dt as ShippedImage, E as HopSignal, Et as Shipped, F as LineCount, Ft as isHopPlace, G as Outcomes, H as MiddleboxExpectation, I as LineMatch, It as latestEndOf, J as PausePrediction, K as Pair, L as LoadBand, M as Kafka, Mt as Topology, N as KafkaPause, Nt as Toxic, O as HostSysctl, Ot as Stage, P as KernelRecord, Pt as WsProbe, Q as ProbeEvent, R as LoadPlan, S as GroupRole, St as RunStream, T as HopPlace, Tt as SentinelStage, U as MiddleboxForm, V as MiddleboxEvent, W as Observer, X as PortsPrediction, Y as PinnedImage, Z as Prediction, _ as Fault, _t as RunFabric, a as ClientCapability, at as Reconnects, b as GroupPlace, bt as RunPause, c as ClusterClientAt, ct as ReplicaSet, d as CountBand, dt as RequestPrediction, et as ProxyTime, f as Drill, ft as RouterHop, g as FabricManifest, gt as RunDrill, h as Fabric, ht as RunBackup, i as Client, it as QueuePrediction, j as Jitter, jt as Target, k as HttpProbe, kt as StageBand, l as ClusterStage, lt as ReplicaSetStage, m as DriverStep, mt as Run, n as CLIENT_CAPABILITIES, nt as QueueConnection, o as ClientImage, ot as Recovers, p as DriverBand, pt as RttRange, q as PauseBand, r as Cell, rt as QueueEnd, s as Cluster, st as Reference, t as BuildImage, tt as QueueBand, u as ConntrackPrediction, ut as Replies, v as FaultPlace, vt as RunGroup, w as HopEvent, wt as SentinelEvent, x as GroupPrediction, xt as RunRoute, y as Group, yt as RunPart, z as LoadPrediction } from "./cell-biffSiEJ.js";
+import { $ as ProxyHop, A as ImageSource, At as Subject, B as LoadStream, C as Hop, Ct as RunTopology, D as HopWrite, Dt as ShippedImage, E as HopSignal, Et as Shipped, F as LineCount, Ft as environmentProblems, G as Outcomes, H as MiddleboxExpectation, I as LineMatch, It as isHopPlace, J as PausePrediction, K as Pair, L as LoadBand, Lt as latestEndOf, M as Kafka, Mt as Topology, N as KafkaPause, Nt as Toxic, O as HostSysctl, Ot as Stage, P as KernelRecord, Pt as WsProbe, Q as ProbeEvent, R as LoadPlan, S as GroupRole, St as RunStream, T as HopPlace, Tt as SentinelStage, U as MiddleboxForm, V as MiddleboxEvent, W as Observer, X as PortsPrediction, Y as PinnedImage, Z as Prediction, _ as Fault, _t as RunFabric, a as ClientCapability, at as Reconnects, b as GroupPlace, bt as RunPause, c as ClusterClientAt, ct as ReplicaSet, d as CountBand, dt as RequestPrediction, et as ProxyTime, f as Drill, ft as RouterHop, g as FabricManifest, gt as RunDrill, h as Fabric, ht as RunBackup, i as Client, it as QueuePrediction, j as Jitter, jt as Target, k as HttpProbe, kt as StageBand, l as ClusterStage, lt as ReplicaSetStage, m as DriverStep, mt as Run, n as CLIENT_CAPABILITIES, nt as QueueConnection, o as ClientImage, ot as Recovers, p as DriverBand, pt as RttRange, q as PauseBand, r as Cell, rt as QueueEnd, s as Cluster, st as Reference, t as BuildImage, tt as QueueBand, u as ConntrackPrediction, ut as Replies, v as FaultPlace, vt as RunGroup, w as HopEvent, wt as SentinelEvent, x as GroupPrediction, xt as RunRoute, y as Group, yt as RunPart, z as LoadPrediction } from "./cell-OwQdpSkd.js";
 import { Quantity } from "@mark1russell7/quantity";
 import { LoadMeasure, Schedule } from "@mark1russell7/rigor-chaos";
 import { StartedTestContainer } from "testcontainers";
@@ -181,6 +181,35 @@ export declare const streamsLoadOf: (events: readonly ProbeEvent[], streams: rea
   readonly pool: RunLoad;
   readonly streams: Readonly<Record<string, RunLoad>>;
 } | null;
+/** This function gives the pool of some measures: it sums the counts and joins the times, in the order of the measures. */
+export declare const poolOf: (measures: readonly RunLoad[]) => RunLoad & {
+  readonly failed: number;
+};
+/**
+ * This time is the tolerance of a closed-loop estimate, in ms. A send later than its expected time
+ * by more than this tolerance is a pause of the client: a driver gap.
+ */
+export declare const CLOSED_TOLERANCE = 5;
+/**
+ * A run estimate is the latency of the open loop that a closed-loop stream hid (SYS-91). The package
+ * rigor-chaos puts back each send that a stall or a pause of the client hid (`closedLoopPlan`).
+ * `hidden` counts those sends, and `driverGaps` counts the pauses of the client. Its times are in ms.
+ */
+type RunEstimate = LoadMeasure & {
+  readonly hidden: number;
+  readonly driverGaps: number;
+  readonly driverGapTotal: number;
+};
+/**
+ * This function gives the estimate of one closed-loop stream from its requests, in the order of their
+ * sends. Each time is a time of a line of the client. The client waits `interval` ms after each reply,
+ * thus the interval is also the pause of the driver. The function gives undefined for no request, and
+ * for a request with no reply before the last request.
+ */
+export declare const estimateOf: (requests: ReadonlyArray<{
+  readonly sent: number;
+  readonly done: number | null;
+}>, end: number, interval: number) => RunEstimate | undefined;
 /** A load spread is a part of the distribution of one measure of an open-loop cell, over its counted runs. */
 interface LoadSpread {
   readonly p50: Quantity;
@@ -224,6 +253,22 @@ export declare const loadVerdict: (prediction: LoadPrediction, runs: ReadonlyArr
   readonly name: string;
   readonly load: RunLoad | undefined;
 }>) => LoadJudgement;
+/**
+ * An estimate verdict sums the closed-loop estimates of a cell over its counted runs. Its latency
+ * is an estimate of the open loop, not a measure of it: a record of an open-loop client stays the
+ * better measure. Each hidden send after a request with no reply counts at its age at the end.
+ */
+interface EstimateVerdict {
+  readonly planned: number;
+  readonly hidden: number;
+  readonly driverGaps: number;
+  readonly driverGapTotal: Quantity;
+  readonly latency: LoadSpread | null;
+  /** This text gives the estimate in words, for the verdict of the cell. */
+  readonly words: string;
+}
+/** This function gives the estimate verdict of the counted runs, or undefined when no run has an estimate. */
+export declare const estimateVerdict: (estimates: ReadonlyArray<RunEstimate | undefined>) => EstimateVerdict | undefined;
 //#endregion
 //#region src/driver.d.ts
 /**
@@ -785,6 +830,8 @@ interface Verdict {
   readonly reordering?: Reordering;
   /** This record is the load of an open-loop cell over its counted runs. */
   readonly load?: LoadVerdict;
+  /** This record is the estimate of the open loop of a closed-loop stream over its counted runs (SYS-91). */
+  readonly estimate?: EstimateVerdict;
   /** This record is the time of the proxy of a calibration cell, in each direction. */
   readonly proxy?: {
     readonly hop: string;
@@ -974,6 +1021,8 @@ interface Measured {
   readonly samples?: readonly Quantity[];
   /** This measure is the run of an open-loop cell against its schedule (`Run.schedule`), with its failed replies. Its times are in ms. */
   readonly load?: RunLoad;
+  /** This estimate is the open loop that a closed-loop stream of the client of the harness hid (SYS-91). Its times are in ms. */
+  readonly estimate?: RunEstimate;
   /** This count is the number of keepalive probes that the kernel sent. */
   readonly probes: number;
   /** This count is the number of segments that the kernel sent again. */
@@ -1415,9 +1464,11 @@ export declare const targetImageOf: (target: Target) => Promise<string>;
  * A lab in a guest of QEMU writes E4 (`substrateHere`). The other layers do not have a runner at this
  * time.
  */
-type Layer = "E0" | "E1" | "E2" | "E3" | "E4" | "E5" | "E6";
+export declare const LAYERS: readonly ["E0", "E1", "E2", "E3", "E4", "E5", "E6"];
+type Layer = (typeof LAYERS)[number];
 /** A cloud is where the runner operates. GitHub can schedule a standard Linux job on Azure or on AWS. */
-type Cloud = "azure" | "aws" | "unknown";
+export declare const CLOUDS: readonly ["azure", "aws", "unknown"];
+type Cloud = (typeof CLOUDS)[number];
 /** A host is what a process reads about its own machine. */
 interface Host {
   /** This text is the kernel release. */
@@ -2650,6 +2701,12 @@ export declare const QUIET_MS = 250;
 export declare const LINGER_MS = 1e3;
 /** This function divides a count of runs across environments, as equally as possible. */
 export declare const spread: (runs: number, environments: number) => number[];
+/**
+ * This function gives the count of runs of each new environment of a cell, for `runs` more runs. A
+ * pair cell has an environment for each run, because its run promotes the standby. Other cells
+ * go across their environments, and at least one for each environment at the same time.
+ */
+export declare const environmentCounts: (cell: Cell, runs: number) => number[];
 /**
  * This function does the runs of a cell. They go across new environments, and each environment records
  * its manifest. Then the verdict judges the runs. A median across the edge of the band extends
@@ -2661,5 +2718,5 @@ export declare const spread: (runs: number, environments: number) => number[];
  */
 export declare const runCell: (cell: Cell, say?: (line: string) => void) => Promise<CellResult>;
 //#endregion
-export { type Agent, type BuildImage, type BuildRecord, CLIENT_CAPABILITIES, type CallReading, type Cell, type CellResult, type Client, type ClientCapability, type ClientImage, type ClockPair, type Cloud, type Cluster, type ClusterClientAt, type ClusterLab, type ClusterStage, type ConntrackPrediction, type ConntrackReading, type ContextFile, type CountBand, type Crossings, type Cut, type Decision, type Drill, type DriverBand, type DriverSocket, type DriverStep, type DriverStepTime, type ErrorOnTheWay, type Fabric, type FabricLab, type FabricManifest, type FabricParts, type Fault, type FaultPlace, type FaultSide, type Group, type GroupClient, type GroupLab, type GroupMeasured, type GroupNetworkFault, type GroupPlace, type GroupPrediction, type GroupReading, type GroupRole, type GroupVerdict, type GroupView, type Hop, type HopEvent, type HopPlace, type HopSignal, type HopWrite, type Host, type HostSetting, type HostSysctl, type HttpProbe, type ImagePlan, type ImageSource, type Interval, type Jitter, type Kafka, type KafkaLab, type KafkaMember, type KafkaPause, type KernelRecord, type Lab, type Layer, type LeafViews, type LineCount, type LineMatch, type LoadBand, type LoadJudgement, type LoadPlan, type LoadPrediction, type LoadSpread, type LoadStream, type LoadVerdict, type Manifest, type ManifestHop, type Measured, type MemberLine, type MiddleboxEvent, type MiddleboxExpectation, type MiddleboxForm, type NetworkFault, type NodeLine, type Observer, type OplogRead, type OplogTime, type Outcome, type Outcomes, type Pair, type PairSets, type Pairing, type PartitionView, type PauseBand, type PauseCommit, type PausePrediction, type PauseReading, type Pinnable, type PinnedImage, type PortsPrediction, type PortsReading, type Prediction, type Probe, type ProbeEvent, type ProgramEvent, type ProxyHop, type ProxyTime, type QueueBand, type QueueConnection, type QueueConnectionReading, type QueueEnd, type QueuePrediction, type QueueReading, type ReconnectReading, type ReconnectVerdict, type Reconnects, type RecordCode, type Recovers, type RecoveryMeasured, type Reference, type Reordering, type ReplicaSet, type ReplicaSetLab, type ReplicaSetStage, type Replies, type RequestPrediction, type RequestReading, type RouteLine, type RouterHop, type RttRange, type Run, type RunBackup, type RunDrill, type RunFabric, type RunGroup, type RunLoad, type RunPart, type RunPause, type RunRoute, type RunStream, type RunTopology, type SentinelEvent, type SentinelStage, type Shipped, type ShippedImage, type Sibling, type SocketPlace, type Spread, type Stage, type StageBand, type StageVerdict, type StallReading, type Stamp, type Status, type StatusRow, type Subject, type Substrate, type SubstrateSource, type Target, type Topology, type Toxic, type Verdict, type WallStamp, type Writes, type WsProbe, isHopPlace, latestEndOf };
+export { type Agent, type BuildImage, type BuildRecord, CLIENT_CAPABILITIES, type CallReading, type Cell, type CellResult, type Client, type ClientCapability, type ClientImage, type ClockPair, type Cloud, type Cluster, type ClusterClientAt, type ClusterLab, type ClusterStage, type ConntrackPrediction, type ConntrackReading, type ContextFile, type CountBand, type Crossings, type Cut, type Decision, type Drill, type DriverBand, type DriverSocket, type DriverStep, type DriverStepTime, type ErrorOnTheWay, type EstimateVerdict, type Fabric, type FabricLab, type FabricManifest, type FabricParts, type Fault, type FaultPlace, type FaultSide, type Group, type GroupClient, type GroupLab, type GroupMeasured, type GroupNetworkFault, type GroupPlace, type GroupPrediction, type GroupReading, type GroupRole, type GroupVerdict, type GroupView, type Hop, type HopEvent, type HopPlace, type HopSignal, type HopWrite, type Host, type HostSetting, type HostSysctl, type HttpProbe, type ImagePlan, type ImageSource, type Interval, type Jitter, type Kafka, type KafkaLab, type KafkaMember, type KafkaPause, type KernelRecord, type Lab, type Layer, type LeafViews, type LineCount, type LineMatch, type LoadBand, type LoadJudgement, type LoadPlan, type LoadPrediction, type LoadSpread, type LoadStream, type LoadVerdict, type Manifest, type ManifestHop, type Measured, type MemberLine, type MiddleboxEvent, type MiddleboxExpectation, type MiddleboxForm, type NetworkFault, type NodeLine, type Observer, type OplogRead, type OplogTime, type Outcome, type Outcomes, type Pair, type PairSets, type Pairing, type PartitionView, type PauseBand, type PauseCommit, type PausePrediction, type PauseReading, type Pinnable, type PinnedImage, type PortsPrediction, type PortsReading, type Prediction, type Probe, type ProbeEvent, type ProgramEvent, type ProxyHop, type ProxyTime, type QueueBand, type QueueConnection, type QueueConnectionReading, type QueueEnd, type QueuePrediction, type QueueReading, type ReconnectReading, type ReconnectVerdict, type Reconnects, type RecordCode, type Recovers, type RecoveryMeasured, type Reference, type Reordering, type ReplicaSet, type ReplicaSetLab, type ReplicaSetStage, type Replies, type RequestPrediction, type RequestReading, type RouteLine, type RouterHop, type RttRange, type Run, type RunBackup, type RunDrill, type RunEstimate, type RunFabric, type RunGroup, type RunLoad, type RunPart, type RunPause, type RunRoute, type RunStream, type RunTopology, type SentinelEvent, type SentinelStage, type Shipped, type ShippedImage, type Sibling, type SocketPlace, type Spread, type Stage, type StageBand, type StageVerdict, type StallReading, type Stamp, type Status, type StatusRow, type Subject, type Substrate, type SubstrateSource, type Target, type Topology, type Toxic, type Verdict, type WallStamp, type Writes, type WsProbe, environmentProblems, isHopPlace, latestEndOf };
 //# sourceMappingURL=index.d.ts.map

package/build/index.js

@@ -1,8 +1,8 @@
-import { A as unlockedManifests, B as isHopPlace, C as programOf, D as subjectProblems, E as subjectClient, F as jarVersions, H as requestFirst, I as locksOf, L as npmVersions, M as LOCK_NAMES, N as cargoVersions, O as targetImageOf, P as goVersions, R as versionsOf, S as planOf, T as shippedSource, V as latestEndOf, _ as kafkaClientImage, a as SHIPPED_NAMES, b as nodeHttpImage, c as buildRecordOf, d as harnessImage, f as harnessVersion, g as inContext, h as imageProblems, i as SHIPPED_CLIENTS, j as unpinnedBases, k as targetSourceOf, l as goHttpImage, m as imageOf, n as PINNED, o as amqpClientImage, p as hikariClientImage, r as REPOSITORY, s as basesOf, t as CLIENT_NAMES, u as grpcClientImage, v as mongoClientImage, w as routerImage, x as pgClientImage, y as nodeClientImage, z as CLIENT_CAPABILITIES } from "./image-BxVTFZ1o.js";
+import { A as unlockedManifests, B as environmentProblems, C as programOf, D as subjectProblems, E as subjectClient, F as jarVersions, H as latestEndOf, I as locksOf, L as npmVersions, M as LOCK_NAMES, N as cargoVersions, O as targetImageOf, P as goVersions, R as versionsOf, S as planOf, T as shippedSource, U as requestFirst, V as isHopPlace, _ as kafkaClientImage, a as SHIPPED_NAMES, b as nodeHttpImage, c as buildRecordOf, d as harnessImage, f as harnessVersion, g as inContext, h as imageProblems, i as SHIPPED_CLIENTS, j as unpinnedBases, k as targetSourceOf, l as goHttpImage, m as imageOf, n as PINNED, o as amqpClientImage, p as hikariClientImage, r as REPOSITORY, s as basesOf, t as CLIENT_NAMES, u as grpcClientImage, v as mongoClientImage, w as routerImage, x as pgClientImage, y as nodeClientImage, z as CLIENT_CAPABILITIES } from "./image-B8yh_FF7.js";
 import { quantity, valueIn } from "@mark1russell7/quantity";
 import { median, medianInterval as medianInterval$1, quantile, quantileSorted } from "@mark1russell7/rigor-statistics";
 import { rungVerdict } from "@mark1russell7/rigor-bench";
-import { measureLoad, schedule, scheduleHeld } from "@mark1russell7/rigor-chaos";
+import { closedLoopPlan, measureLoad, schedule, scheduleHeld } from "@mark1russell7/rigor-chaos";
 import { createHash, randomBytes } from "node:crypto";
 import { execFile } from "node:child_process";
 import { mkdtemp, rm, writeFile } from "node:fs/promises";
@@ -99,21 +99,56 @@ const streamsLoadOf = (events, streams) => {
 	const start = events.find((event) => event.ev === "scheduled")?.t;
 	if (start === void 0) return null;
 	const each = streams.map((stream) => [stream.key, measureOf(events, start, stream.schedule, stream.key)]);
-	const measures = each.map(([, measure]) => measure);
+	return {
+		pool: poolOf(each.map(([, measure]) => measure)),
+		streams: Object.fromEntries(each)
+	};
+};
+/** This function gives the pool of some measures: it sums the counts and joins the times, in the order of the measures. */
+const poolOf = (measures) => {
 	const sum = (count) => measures.reduce((total, measure) => total + count(measure), 0);
 	const join = (times) => measures.flatMap(times);
 	return {
-		pool: {
-			planned: sum((measure) => measure.planned),
-			unsent: sum((measure) => measure.unsent),
-			unfinished: sum((measure) => measure.unfinished),
-			failed: sum((measure) => measure.failed ?? 0),
-			latency: join((measure) => measure.latency),
-			service: join((measure) => measure.service),
-			lag: join((measure) => measure.lag),
-			censored: join((measure) => measure.censored)
-		},
-		streams: Object.fromEntries(each)
+		planned: sum((measure) => measure.planned),
+		unsent: sum((measure) => measure.unsent),
+		unfinished: sum((measure) => measure.unfinished),
+		failed: sum((measure) => measure.failed ?? 0),
+		latency: join((measure) => measure.latency),
+		service: join((measure) => measure.service),
+		lag: join((measure) => measure.lag),
+		censored: join((measure) => measure.censored)
+	};
+};
+/**
+* This time is the tolerance of a closed-loop estimate, in ms. A send later than its expected time
+* by more than this tolerance is a pause of the client: a driver gap.
+*/
+const CLOSED_TOLERANCE = 5;
+/**
+* This function gives the estimate of one closed-loop stream from its requests, in the order of their
+* sends. Each time is a time of a line of the client. The client waits `interval` ms after each reply,
+* thus the interval is also the pause of the driver. The function gives undefined for no request, and
+* for a request with no reply before the last request.
+*/
+const estimateOf = (requests, end, interval) => {
+	const start = requests[0]?.sent;
+	if (start === void 0 || requests.slice(0, -1).some((request) => request.done === null)) return void 0;
+	const loop = requests.map((request) => ({
+		sent: fromStart(start, request.sent),
+		done: request.done === null ? null : fromStart(start, request.done)
+	}));
+	const stop = Math.max(fromStart(start, end), ...loop.map((request) => request.done ?? request.sent), interval);
+	const run = closedLoopPlan(loop, {
+		interval,
+		end: stop,
+		tolerance: 5,
+		pause: interval
+	});
+	return {
+		...measureLoad(run.plan, run.completions, stop),
+		hidden: run.hidden,
+		driverGaps: run.driverGaps,
+		driverGapTotal: run.driverGapTotal
 	};
 };
 /** This function gives the spread of a list of times in ms, or null for no time. */
@@ -246,6 +281,28 @@ const loadVerdict = (prediction, runs) => {
 		}
 	};
 };
+/** This function gives the estimate verdict of the counted runs, or undefined when no run has an estimate. */
+const estimateVerdict = (estimates) => {
+	const runs = estimates.filter((estimate) => estimate !== void 0);
+	if (runs.length === 0) return void 0;
+	const pool = poolOf(runs);
+	const total = (count) => runs.reduce((sum, estimate) => sum + count(estimate), 0);
+	const hidden = total((estimate) => estimate.hidden);
+	const driverGaps = total((estimate) => estimate.driverGaps);
+	const driverGapTotal = total((estimate) => estimate.driverGapTotal);
+	const latency = spreadOf$1([...pool.latency, ...pool.censored]);
+	const seconds = (time) => (valueIn(time, "ms") / 1e3).toFixed(3);
+	const spread = latency === null ? "no latency" : `median ${seconds(latency.p50)} s, p99 ${seconds(latency.p99)} s, slowest ${seconds(latency.max)} s`;
+	const gaps = driverGaps === 0 ? "no pause of the client" : `${driverGaps} pauses of the client, ${Math.round(driverGapTotal)} ms in all`;
+	return {
+		planned: pool.planned,
+		hidden,
+		driverGaps,
+		driverGapTotal: quantity(driverGapTotal, "ms"),
+		latency,
+		words: `The estimate of the open loop puts back ${hidden} hidden sends, with ${gaps}: ${spread}.`
+	};
+};
 //#endregion
 //#region src/verdict.ts
 /** This function gives the words of the outcome of a second request: its connection, a retry, and its end. */
@@ -687,7 +744,9 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
 			reordering.fast,
 			reordering.seen
 		].some((count) => (count ?? 0) > 0) ? ` Reordering: ${reorderingWords(reordering)}.` : "";
-		const why = samples.length === 0 ? `${counted.length} runs measured, with no reply: ${prediction.why}` : `${counted.length} runs, ${samples.length} replies: median ${s(median(samples)).value} s, p99 ${s(quantileOf(samples, .99)).value} s, slowest ${s(samples.at(-1)).value} s, ${retransmissions} retransmissions, ${ended.length} connections ended.${reordered} ${prediction.why}`;
+		const estimate = estimateVerdict(counted.map((run) => run.estimate));
+		const estimated = estimate === void 0 ? "" : ` ${estimate.words}`;
+		const why = samples.length === 0 ? `${counted.length} runs measured, with no reply: ${prediction.why}` : `${counted.length} runs, ${samples.length} replies: median ${s(median(samples)).value} s, p99 ${s(quantileOf(samples, .99)).value} s, slowest ${s(samples.at(-1)).value} s, ${retransmissions} retransmissions, ${ended.length} connections ended.${reordered}${estimated} ${prediction.why}`;
 		return {
 			...empty,
 			decision: "measured",
@@ -696,6 +755,7 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
 			median: samples.length === 0 ? null : s(median(samples)),
 			...samples.length === 0 ? {} : { slowest: s(samples.at(-1)) },
 			...reordering === void 0 ? {} : { reordering },
+			...estimate === void 0 ? {} : { estimate },
 			why
 		};
 	}
@@ -3019,11 +3079,18 @@ const measureSocket = (cell, run) => {
 		};
 		const sends = load === void 0 ? events.filter((event) => event.ev === "sent").slice(1) : [];
 		const answers = events.some((event) => event.ev === "replied") ? "replied" : "data";
-		const samples = sends.flatMap((send, n) => {
+		const requests = sends.map((send, n) => {
 			const next = sends[n + 1]?.t ?? Infinity;
-			const reply = events.find((event) => event.ev === answers && event.t > send.t && event.t < next);
-			return reply === void 0 ? [] : [between(send.t, reply.t)];
+			return {
+				send,
+				reply: events.find((event) => event.ev === answers && event.t > send.t && event.t < next)
+			};
 		});
+		const samples = requests.flatMap(({ send, reply }) => reply === void 0 ? [] : [between(send.t, reply.t)]);
+		const estimate = cell.probe.mode === "stream" && cell.probe.driver === void 0 && cell.probe.interval !== void 0 ? estimateOf(requests.map(({ send, reply }) => ({
+			sent: send.t,
+			done: reply?.t ?? null
+		})), events.reduce((latest, event) => Math.max(latest, event.t), 0), valueIn(cell.probe.interval, "ms")) : void 0;
 		const from = prediction.from === void 0 ? void 0 : referenceAt(prediction.from);
 		const until = ended ?? after.at(-1)?.t;
 		const peerRecords = peerCookie === void 0 ? [] : kernel.filter((record) => record.netns === peerNetns && record.cookie === peerCookie);
@@ -3056,6 +3123,7 @@ const measureSocket = (cell, run) => {
 			steps: retransmitted.filter((record) => record.mono > request.t).map((record) => between(request.t, record.mono)),
 			samples,
 			...load === void 0 ? {} : { load },
+			...estimate === void 0 ? {} : { estimate },
 			reordering: reorderingOf(socket.filter((record) => record.mono > request.t), peerRecords.filter((record) => record.mono > request.t)),
 			...crossings,
 			void: voided
@@ -3664,6 +3732,27 @@ const openFabric = async (parts) => {
 };
 //#endregion
 //#region src/substrate.ts
+/**
+* A layer is a rung of the laboratory (`docs/S7.md`). E1 is containers on one kernel, and E2 is a
+* cluster of containers on one kernel. A lab with `Topology.fabric` writes E3: routers in containers.
+* A lab in a guest of QEMU writes E4 (`substrateHere`). The other layers do not have a runner at this
+* time.
+*/
+const LAYERS = [
+	"E0",
+	"E1",
+	"E2",
+	"E3",
+	"E4",
+	"E5",
+	"E6"
+];
+/** A cloud is where the runner operates. GitHub can schedule a standard Linux job on Azure or on AWS. */
+const CLOUDS = [
+	"azure",
+	"aws",
+	"unknown"
+];
 /** GitHub announced Linux jobs on AWS on this date. Each earlier hosted job ran on Azure. */
 const AWS_FROM = "2026-10-28";
 /** This function gives the cloud of a DMI vendor. A kernel name does not tell: one image can start on each cloud. */
@@ -7493,6 +7582,19 @@ const clusterOnce = async (lab, cell, environment, index) => {
 * not in the namespace that the prober watches. The record of a run is the lines of the client and
 * the whole partition, read to its high watermark after the group converged.
 */
+/**
+* This function gives the record of the Kafka client for a manifest: its image, its source and its
+* build record. The pins check refuses a record of a built client without its build record.
+*/
+const kafkaClientRecord = (image) => {
+	const source = shippedSource("kafka");
+	return {
+		kind: "image",
+		image,
+		source,
+		...buildOf(source)
+	};
+};
 /** This path is the class path of the Kafka image: the JVM expands the star itself. */
 const CLASS_PATH = "/opt/harness/kafka/classes:/opt/harness/kafka/lib/*";
 /** This option is the heap of the client and of the admin. It has no source: the two hold a producer or a read of one partition. */
@@ -7892,12 +7994,10 @@ const openKafkaLab = async (kafka) => {
 							"apk",
 							"info",
 							"-v"
-						])).split("\n").sort()
-					},
-					client: {
-						kind: "image",
-						image: clientImage
+						])).split("\n").sort(),
+						...harnessRecord()
 					},
+					client: kafkaClientRecord(clientImage),
 					netns: {
 						client: clientNetns,
 						target: members[kafka.brokers[0]].netns
@@ -8663,6 +8763,12 @@ const once = async (lab, cell, environment, index, dry = false) => {
 /** This function divides a count of runs across environments, as equally as possible. */
 const spread = (runs, environments) => Array.from({ length: environments }, (_, at) => Math.floor(runs / environments) + (at < runs % environments ? 1 : 0)).filter((count) => count > 0);
 /**
+* This function gives the count of runs of each new environment of a cell, for `runs` more runs. A
+* pair cell has an environment for each run, because its run promotes the standby. Other cells
+* go across their environments, and at least one for each environment at the same time.
+*/
+const environmentCounts = (cell, runs) => spread(runs, cell.pair !== void 0 ? runs : Math.max(cell.environments, Math.max(1, cell.concurrency ?? 1)));
+/**
 * This function tells if the fault of a cell ends its environment. A server that stopped cannot serve
 * the next run. Also, a cell can say that its fault changes the environment (`fresh`).
 */
@@ -8799,6 +8905,7 @@ const inReplicaSetEnvironment = async (cell, environment, count, say) => {
 * run. The run promotes the standby, thus a pair cell has an environment for each run.
 */
 const inPairEnvironment = async (cell, environment, count, say) => {
+	if (count !== 1) throw new Error(`${cell.name}: a pair environment has one run, not ${count}: the run promotes the standby`);
 	const lab = await openLab(cell.target, cell.client ?? cell.probe.driver?.name ?? "tcp", cell.topology);
 	try {
 		await lab.warm();
@@ -8891,6 +8998,8 @@ const inKafkaEnvironment = async (cell, environment, count, say) => {
 * Each environment does a warm-up first, and the result keeps it apart from the counted runs.
 */
 const runCell = async (cell, say = () => {}) => {
+	const problems = environmentProblems(cell);
+	if (problems.length > 0) throw new Error(problems.join(", "));
 	const started = (/* @__PURE__ */ new Date()).toISOString();
 	const manifests = [];
 	const runs = [];
@@ -8899,7 +9008,7 @@ const runCell = async (cell, say = () => {}) => {
 	let target = cell.runs;
 	let next = 0;
 	for (;;) {
-		const counts = spread(target - runs.length, Math.max(cell.environments, concurrency));
+		const counts = environmentCounts(cell, target - runs.length);
 		for (let at = 0; at < counts.length; at += concurrency) {
 			const batch = counts.slice(at, at + concurrency).map((count) => ({
 				count,
@@ -8935,6 +9044,6 @@ const runCell = async (cell, say = () => {}) => {
 	}
 };
 //#endregion
-export { AWS_FROM, CLIENT_CAPABILITIES, CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, DAEMONS, DEBUG, DIRTY, DROP_HANDLING, FABRIC_CLIENT, FABRIC_DEVICE, FABRIC_TARGET, FRR_IMAGE, GIVES_AWAY, HASH_SCRIPT, INJECT, INSERT, KAFKA_TAIL_MS, KEPT, LINGER_MS, LISTING, LOAD_FORMAT, LOAD_PATH, LOCK_NAMES, LOSS_MARGIN, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, RESTART, ROUTERS, SCHEDULE_PATH, SEED_DOCUMENT, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STANDBY_READY, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, basesOf, bootstrapOf, brokerEnv, buildOf, buildRecordOf, callOf, cancelsOf, cargoVersions, clockOffsetOf, closeOrExit, cloudOf, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, commitProblems, comparable, controllerEnv, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, drillOnce, drillSpan, dropFinding, dropOf, endpointAfter, envOf, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, frrConf, goHttpImage, goVersions, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, harnessRecord, harnessVersion, hashOf, highestHeld, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isFenceLine, isHopPlace, jarVersions, jitterOf, jitterOfLines, judge, kafkaArgs, kafkaClientImage, kafkaCutOf, kafkaOnce, kafkaPauseOnce, kafkaViewOf, lagsOf, laterMove, latestEndOf, leafLacks, leftAfter, loadOf, loadPlanOf, loadVerdict, locksOf, logNameOf, logTimeOf, manifestProblems, matches, measure, measureConntrack, measureDriver, measureGroup, measurePause, measurePorts, measureQueue, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicAt, monotonicOf, namesGroup, nodeClientImage, nodeCommand, nodeHttpImage, nodeIdOf, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, npmVersions, numbersOf, onMonotonic, openClusterLab, openFabric, openGroupLab, openKafkaLab, openLab, openReplicaSetLab, openStandby, pairArgs, pairOnce, pairingOf, pairingsOf, pauseReady, peersOf, pgClientImage, pingsOf, plainOf, planOf, programLines, programOf, quantileOf, readCpu, readHost, readOplog, reconnectFinding, reconnectOf, recordProblems, recoveryFindings, recoveryOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routeLine, routerImage, routesOf, runCell, seedScript, sentinelConfig, sessionsOf, shippedClientRecord, shippedSource, sliceQuery, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, standaloneCommand, standbyCommand, statusViewOf, stealShare, stopScript, streamsLoadOf, subjectClient, subjectProblems, subscriptionsOf, substrateHere, substrateKey, substrateOf, substrateOn, targetImageOf, targetSourceOf, templated, toolJsonOf, topicOf, topologyOf, toxicBody, unlockedManifests, unpinnedBases, unpoolable, verdictOf, versionsOf, viewOf, writesOf };
+export { AWS_FROM, CLIENT_CAPABILITIES, CLIENT_NAMES, CLOSED_TOLERANCE, CLOUDS, CLUSTER_STAGES, CONVERGE_MS, DAEMONS, DEBUG, DIRTY, DROP_HANDLING, FABRIC_CLIENT, FABRIC_DEVICE, FABRIC_TARGET, FRR_IMAGE, GIVES_AWAY, HASH_SCRIPT, INJECT, INSERT, KAFKA_TAIL_MS, KEPT, LAYERS, LINGER_MS, LISTING, LOAD_FORMAT, LOAD_PATH, LOCK_NAMES, LOSS_MARGIN, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, RESTART, ROUTERS, SCHEDULE_PATH, SEED_DOCUMENT, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STANDBY_READY, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, basesOf, bootstrapOf, brokerEnv, buildOf, buildRecordOf, callOf, cancelsOf, cargoVersions, clockOffsetOf, closeOrExit, cloudOf, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, commitProblems, comparable, controllerEnv, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, drillOnce, drillSpan, dropFinding, dropOf, endpointAfter, envOf, environmentCounts, environmentProblems, errorsOf, estimateOf, estimateVerdict, exitOf, fieldOf, fieldsOf, firstUnanswered, frrConf, goHttpImage, goVersions, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, harnessRecord, harnessVersion, hashOf, highestHeld, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isFenceLine, isHopPlace, jarVersions, jitterOf, jitterOfLines, judge, kafkaArgs, kafkaClientImage, kafkaCutOf, kafkaOnce, kafkaPauseOnce, kafkaViewOf, lagsOf, laterMove, latestEndOf, leafLacks, leftAfter, loadOf, loadPlanOf, loadVerdict, locksOf, logNameOf, logTimeOf, manifestProblems, matches, measure, measureConntrack, measureDriver, measureGroup, measurePause, measurePorts, measureQueue, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicAt, monotonicOf, namesGroup, nodeClientImage, nodeCommand, nodeHttpImage, nodeIdOf, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, npmVersions, numbersOf, onMonotonic, openClusterLab, openFabric, openGroupLab, openKafkaLab, openLab, openReplicaSetLab, openStandby, pairArgs, pairOnce, pairingOf, pairingsOf, pauseReady, peersOf, pgClientImage, pingsOf, plainOf, planOf, poolOf, programLines, programOf, quantileOf, readCpu, readHost, readOplog, reconnectFinding, reconnectOf, recordProblems, recoveryFindings, recoveryOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routeLine, routerImage, routesOf, runCell, seedScript, sentinelConfig, sessionsOf, shippedClientRecord, shippedSource, sliceQuery, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, standaloneCommand, standbyCommand, statusViewOf, stealShare, stopScript, streamsLoadOf, subjectClient, subjectProblems, subscriptionsOf, substrateHere, substrateKey, substrateOf, substrateOn, targetImageOf, targetSourceOf, templated, toolJsonOf, topicOf, topologyOf, toxicBody, unlockedManifests, unpinnedBases, unpoolable, verdictOf, versionsOf, viewOf, writesOf };
 
 //# sourceMappingURL=index.js.map