@mark1russell7/harness 0.14.2 → 0.14.4

package/build/index.d.ts

@@ -1,6 +1,6 @@
-import { $ as StageBand, A as PinnedImage, B as Replies, C as Jitter, D as MiddleboxExpectation, E as MiddleboxEvent, F as Reconnects, G as RunGroup, H as RouterHop, I as Recovers, J as SentinelEvent, K as RunPart, L as Reference, M as ProbeEvent, N as ProxyHop, O as MiddleboxForm, P as ProxyTime, Q as Stage, R as ReplicaSet, S as ImageSource, T as LineMatch, U as RttRange, V as RequestPrediction, W as Run, X as Shipped, Y as SentinelStage, Z as ShippedImage, _ as HopEvent, a as Cluster, at as isHopPlace, b as HopWrite, c as DriverBand, d as FaultPlace, et as Subject, f as Group, g as Hop, h as GroupRole, i as ClientImage, it as WsProbe, j as Prediction, k as Observer, l as DriverStep, m as GroupPrediction, n as Cell, nt as Topology, o as ClusterClientAt, ot as latestEndOf, p as GroupPlace, q as RunTopology, r as Client, rt as Toxic, s as ClusterStage, t as BuildImage, tt as Target, u as Fault, v as HopPlace, w as KernelRecord, x as HttpProbe, y as HopSignal, z as ReplicaSetStage } from "./cell-BccWKiXh.js";
+import { $ as Replies, A as LineCount, B as Prediction, C as HopSignal, Ct as latestEndOf, D as Jitter, E as ImageSource, F as Observer, G as QueueConnection, H as ProxyHop, I as Outcomes, J as Reconnects, K as QueueEnd, L as Pair, M as MiddleboxEvent, N as MiddleboxExpectation, O as Kafka, P as MiddleboxForm, Q as ReplicaSetStage, R as PinnedImage, S as HopPlace, St as isHopPlace, T as HttpProbe, U as ProxyTime, V as ProbeEvent, W as QueueBand, X as Reference, Y as Recovers, Z as ReplicaSet, _ as GroupPlace, _t as Subject, a as Cluster, at as RunDrill, b as Hop, bt as Toxic, c as CountBand, ct as RunPart, d as DriverStep, dt as SentinelEvent, et as RequestPrediction, f as Fabric, ft as SentinelStage, g as Group, gt as StageBand, h as FaultPlace, ht as Stage, i as ClientImage, it as RunBackup, j as LineMatch, k as KernelRecord, l as Drill, lt as RunRoute, m as Fault, mt as ShippedImage, n as Cell, nt as RttRange, o as ClusterClientAt, ot as RunFabric, p as FabricManifest, pt as Shipped, q as QueuePrediction, r as Client, rt as Run, s as ClusterStage, st as RunGroup, t as BuildImage, tt as RouterHop, u as DriverBand, ut as RunTopology, v as GroupPrediction, vt as Target, w as HopWrite, x as HopEvent, xt as WsProbe, y as GroupRole, yt as Topology, z as PortsPrediction } from "./cell-DZm8zXN4.js";
 import { Quantity } from "@mark1russell7/quantity";
-import "testcontainers";
+import { StartedTestContainer } from "testcontainers";
 //#region src/application.d.ts
 /**
  * This module reads the acts of the program of the observer: its heartbeat, its drop of the
@@ -204,6 +204,74 @@ export declare const referenceOf: (run: Run, from: Reference, role: string, data
  */
 export declare const measureDriver: (cell: Cell, run: Run) => Measured;
 //#endregion
+//#region src/recovery.d.ts
+/**
+ * This module holds the pure parts of a recovery drill. These are the scripts of the shell, the reader
+ * of the output of `bsondump`, and the set check of a recovery. It has no container, thus unit tests
+ * run it. `drill.ts` does the drill on a lab.
+ *
+ * A dump with `--oplog` reads the oplog from the start of the dump to its end. A slice reads the
+ * oplog after the last entry of the backup before it. Thus the backups together hold each write up
+ * to the last entry of the last backup, and no write after it. One writer sends the writes one at a
+ * time, thus the highest sequence number is also the latest write.
+ */
+/** This size is one document of generated data. A tenant holds as many documents as its size needs. */
+export declare const SEED_DOCUMENT = 65536;
+/** This count is how far the lost writes can be from the writes after the last backup: the writes in flight at the fault. */
+export declare const LOSS_MARGIN = 2;
+/** This function gives the time of a drill from the start of the backups to the fault, in seconds. */
+export declare const drillSpan: (drill: Drill) => number;
+/**
+ * This script writes the generated data of each tenant into the collection `tenants`. Each write
+ * waits for `copies` members, thus each member holds the data when the script ends.
+ */
+export declare const seedScript: (drill: Drill, copies: number) => string;
+/** This script prints the hash of the collection `tenants`. It takes locks, thus a run reads it with no writer. */
+export declare const HASH_SCRIPT: string;
+/** This function gives the hash from the output of `HASH_SCRIPT`. */
+export declare const hashOf: (text: string) => string;
+/** This function gives the command of a server with no set, on its own port. A client of the set cannot reach that server, and a restore writes to it. */
+export declare const standaloneCommand: (set: ReplicaSet, port: number) => string;
+/**
+ * This script waits for the end of the server of a box, and it can stop the server first. `kill -0`
+ * also succeeds on a zombie, and the shell of the box can leave its child as a zombie for a long
+ * time. Thus the script reads the state of the process, and a zombie has ended.
+ */
+export declare const stopScript: (terminate: boolean) => string;
+/** An oplog time is the `ts` of an oplog entry: a time in seconds, and the count of the entry in that second. */
+interface OplogTime {
+  readonly t: number;
+  readonly i: number;
+}
+/** An oplog read is what an oplog file holds: its last entry, and the highest sequence number that it inserts. */
+interface OplogRead {
+  readonly last: OplogTime | null;
+  readonly highest: number | null;
+}
+/** This function reads an oplog file from the output of `bsondump`. Only an insert into `cells.w` is a write of a client. */
+export declare const readOplog: (text: string) => OplogRead;
+/** This function gives the highest sequence number of the file of the writes in a dump, from the output of `bsondump`, or null. */
+export declare const highestHeld: (text: string) => number | null;
+/** This function gives the query of a slice of the oplog: each entry after an oplog time, in Extended JSON. */
+export declare const sliceQuery: (after: OplogTime) => string;
+/** A recovery measurement is what a drill found, beside the writes of the run. */
+interface RecoveryMeasured {
+  /** This time is from the reply to the last accepted write that the store kept to the fault, or null with no such write. */
+  readonly rpo: Quantity | null;
+  /** This time is `rpo` less the age of the drill: what a schedule adds to its interval. It is null for a control and for an election. */
+  readonly excess: Quantity | null;
+  /** These are the accepted writes that the store lost, below a write that it kept. A restore must leave none. */
+  readonly holes: readonly number[];
+  /** This count is the number of accepted writes after the last write of the last backup, or null without a backup. */
+  readonly afterBackup: number | null;
+  /** This flag tells if the tenants have the same data after the recovery, or null when that is not known. */
+  readonly sameData: boolean | null;
+}
+/** This function gives the measurement of a drill, from its run. `t0` is the fault. */
+export declare const recoveryOf: (drill: Drill, run: Run, t0: number) => RecoveryMeasured;
+/** This function gives the findings of a drill: each rule of a recovery that its run broke. */
+export declare const recoveryFindings: (measured: RecoveryMeasured, writes: Writes) => string[];
+//#endregion
 //#region src/timeline.d.ts
 /**
  * This module measures a run of a group cell. It gives the first write that a primary accepted
@@ -227,6 +295,9 @@ interface ErrorOnTheWay {
  *   of a split brain. It is null when the group did not converge after the run.
  * - `unknown` are the writes that the client sent and did not see accepted: an error, or no answer.
  *   `held` tells if the group kept each one, or null when it did not converge.
+ * - `duplicates` has one entry for each extra copy that the group holds: a write held three times
+ *   has two entries. It is null when the group did not converge. A key holds one copy by
+ *   construction, thus only a list can show a copy.
  */
 interface Writes {
   readonly sent: number;
@@ -237,6 +308,18 @@ interface Writes {
     readonly n: number;
     readonly held: boolean | null;
   }>;
+  readonly duplicates?: readonly number[] | null;
+  /** These are the counts of a pair (`Cell.pair`), from its two final reads. */
+  readonly sets?: PairSets;
+}
+/** These are the counts that the bands of `Outcomes` judge: each one has the same name there. */
+interface PairSets {
+  readonly unknown: number;
+  readonly kept: number;
+  readonly stranded: number;
+  readonly absent: number;
+  readonly missing: number;
+  readonly warnings: number;
 }
 /** A group measurement is what one run of a group cell measured, beside its time. */
 interface GroupMeasured {
@@ -249,6 +332,8 @@ interface GroupMeasured {
   readonly errors: readonly ErrorOnTheWay[];
   /** These are the delays of the stamps of the observer after the writes of the Sentinels, for a calibration. */
   readonly lags?: readonly Quantity[];
+  /** This is what a recovery drill measured, for a drill only. */
+  readonly recovery?: RecoveryMeasured;
 }
 /** This function gives the time of each stage of a timeline from an instant: the first announcement of each, after the instant. */
 export declare const stagesOf: (timeline: readonly SentinelEvent[], from: number) => Partial<Record<Stage, Quantity>>;
@@ -336,6 +421,78 @@ interface GroupReading {
  */
 export declare const measureGroup: (cell: Cell, run: Run) => GroupReading;
 //#endregion
+//#region src/ports.d.ts
+/** This record is what a run of a port cell measured. */
+interface PortsReading {
+  /** This count is the rounds of the run, each destination once. */
+  readonly rounds: number;
+  /** This record gives the count of searches for each count of connects, by that count. */
+  readonly counts: Readonly<Record<string, number>>;
+  /** These offsets from the low port are the ports that the run used, in rising order. */
+  readonly seen: readonly number[];
+  /** These errnos ended the searches of the run, each one time. */
+  readonly errnos: readonly string[];
+  /** This time is the longest connect that failed. */
+  readonly refusal: Quantity;
+  /** This time is from the `closing` line to the first `freed` line. */
+  readonly firstFreed: Quantity | null;
+  /** This time is from the `closing` line to the last `freed` line. */
+  readonly lastFreed: Quantity | null;
+  /** This count is the `freed` lines of the run. */
+  readonly freed: number;
+  /** This count is the connects that failed before the last `freed` line. */
+  readonly tries: number;
+}
+/**
+ * This function reads one run of a port cell. The `round` lines give each search. The `closing` and
+ * `freed` lines give when each held port comes back. The records of the prober give the
+ * retransmissions.
+ */
+export declare const measurePorts: (run: Run, expected: PortsPrediction) => Measured;
+//#endregion
+//#region src/queue.d.ts
+/** This record is what one connection of a queue cell did, from its lines and the records of its socket. */
+interface QueueConnectionReading {
+  readonly name: string;
+  readonly connected: boolean;
+  /** These times are the waits of the SYN. The first one counts from the dial. */
+  readonly syns: readonly Quantity[];
+  /** These times are the waits of each SYN-ACK sent again. The first one counts from the dial. */
+  readonly synAcks: readonly Quantity[];
+  /** This count is the segments that the socket sent again. */
+  readonly retransmissions: number;
+  readonly errno: string | null;
+  /** This time is from the line of the prediction to the end of the connection. It is null for a live connection, or with no prediction. */
+  readonly ended: Quantity | null;
+  /** This time is from the last retransmission to the end of the connection. */
+  readonly afterLast: Quantity | null;
+  /** This count is the segments of the socket with no acknowledgment at the end of the run. */
+  readonly unacked: number | null;
+}
+/** This record is what a run of a queue cell measured. The counts are the changes in the run. */
+interface QueueReading {
+  readonly max: number;
+  readonly held: number;
+  readonly connected: number;
+  readonly overflows: number;
+  readonly drops: number;
+  readonly cookies: number;
+  readonly timeouts: number;
+  readonly resets: number;
+  readonly drained: number;
+  /** This time is from the first dial of the burst to the last `TCPTimeouts` of the prediction. */
+  readonly expiry: Quantity | null;
+  /** This time is from the first dial of the burst to the rise of the count of the queue. */
+  readonly rise: Quantity | null;
+  readonly connections: readonly QueueConnectionReading[];
+}
+/**
+ * This function reads one run of a queue cell. The client lines give the connections and the
+ * counters at the two ends of the run. The target lines give the count of the queue. The records of
+ * the prober give the waits of each socket.
+ */
+export declare const measureQueue: (run: Run, expected: QueuePrediction) => Measured;
+//#endregion
 //#region src/verdict.d.ts
 /** This function gives the words of the outcome of a second request: its connection, a retry, and its end. */
 export declare const requestWords: (outcome: {
@@ -435,6 +592,8 @@ interface Verdict {
   readonly median: Quantity | null;
   /** This interval is the order-statistic interval of the median at 95 %, or the whole range for fewer than six runs. */
   readonly medianInterval: Interval | null;
+  /** This time is the slowest reply of a cell that measures a distribution. A closed loop hides the sends in a stall, thus a quantile is a lower bound, and this maximum is not. */
+  readonly slowest?: Quantity;
   /** This record is the sum of what reordering did in each counted run of a measurement-only cell. */
   readonly reordering?: Reordering;
   /** This record is the time of the proxy of a calibration cell, in each direction. */
@@ -505,6 +664,8 @@ interface StageVerdict {
  * - `unknown` counts the writes whose outcome the client did not see, and how many the group kept.
  * - `errors` are the errors on the way: the runs that met each, and the median of its first time.
  * - `lags` is the spread of the delays of the stamps, for a calibration.
+ * - `duplicates` counts the extra copies that the group holds, in each run. A cell with no band of
+ *   copies has no such count.
  */
 interface GroupVerdict {
   readonly stages: readonly StageVerdict[];
@@ -529,6 +690,12 @@ interface GroupVerdict {
     readonly first: Quantity;
   }>;
   readonly lags?: Spread | null;
+  readonly duplicates?: {
+    readonly runs: number;
+    readonly total: number;
+    readonly median: number | null;
+    readonly max: number | null;
+  };
 }
 /**
  * This function gives the value at quantile p of sorted values, by nearest rank: x_(⌈p·n⌉). It is
@@ -640,6 +807,10 @@ interface Measured {
   readonly crossings?: Crossings;
   /** This record is what a run of a group cell measured: the stages of its timeline, and what became of each write. */
   readonly group?: GroupMeasured;
+  /** This record is what a run of a queue cell measured: the count of the queue, its counters, and each connection. */
+  readonly queue?: QueueReading;
+  /** This record is what a run of a port cell measured: the count of each search, and when each port came back. */
+  readonly ports?: PortsReading;
   /** This record is what the second request of an HTTP client met, for a cell whose prediction says it. */
   readonly request?: RequestReading | null;
   /** This record is the time of each later band of a driver cell (`DriverBand.then`). */
@@ -761,10 +932,293 @@ export declare const jitterOf: (cell: Cell, run: Run) => Jitter;
  */
 export declare const requestOf: (events: readonly ProbeEvent[]) => RequestReading | null;
 //#endregion
+//#region src/fabric.d.ts
+/**
+ * A fabric is a routed network between the client and the target, on layer E3 (docs/S7.md). Four
+ * routers of FRRouting, two leaves and two spines, run real BGP over veth pairs that containerlab
+ * makes. The client hangs off leaf1 and the target off leaf2. The path between them goes through
+ * spine1 by a weight, and spine2 holds the other path. A cell silences spine1, and the routes of
+ * the leaves move to spine2 when a detector of the routers fires.
+ *
+ * This file holds the plan of the fabric as pure functions, and `openFabric` that starts it. A lab
+ * opens a fabric with `Topology.fabric`. Thus the client, the target, the prober and the verdict
+ * stay the same as on layer E1. The fabric needs a Linux host with containerlab and sudo.
+ */
+/** This image is FRRouting 10.7.1, pinned by digest. */
+export declare const FRR_IMAGE = "quay.io/frrouting/frr:10.7.1@sha256:e995beaa50fdc9edb35eadcfefa29b7f062cc06f2b812613789b68fa541554d2";
+/** This device is the side of the fabric in the client and in the target. Docker keeps eth0. */
+export declare const FABRIC_DEVICE = "eth1";
+/** This address is the client on its network behind leaf1. */
+export declare const FABRIC_CLIENT = "10.1.0.2";
+/** This address is the target on its network behind leaf2. */
+export declare const FABRIC_TARGET = "10.2.0.2";
+/** A router is one box of the fabric, as data. */
+interface Router {
+  readonly name: string;
+  readonly as: number;
+  readonly id: string;
+  readonly interfaces: Readonly<Record<string, string>>;
+  readonly neighbors: ReadonlyArray<{
+    readonly address: string;
+    readonly as: number;
+    readonly weight?: number;
+  }>;
+  /** This prefix is the network behind a leaf, which its BGP advertises. */
+  readonly network?: string;
+  /** This prefix is the network behind the other leaf. A move is a change of its route. */
+  readonly remote?: string;
+}
+/** These are the routers. Each leaf prefers spine1 by a weight of 100, thus the path of each run is known. */
+export declare const ROUTERS: readonly Router[];
+/** This text is the `daemons` file of each router: BGP and BFD, on top of what FRRouting always starts. */
+export declare const DAEMONS: string;
+/**
+ * This function gives the configuration of a router. Graceful restart is off: after a hold expiry,
+ * a helper can keep the stale routes, and the move that a cell measures does not come.
+ * `maximum-paths 1` keeps one path, thus a move is one change of one route.
+ */
+export declare const frrConf: (router: Router, fabric: Fabric) => string;
+/**
+ * This function gives the topology file of containerlab. The client and the target are containers
+ * of the lab, and the fabric adds a device to each. Each link has the MTU of a Docker network, thus
+ * a segment is as long as on layer E1. The default of containerlab is 9500. The management network
+ * belongs to the lab, and Docker chooses its subnet.
+ */
+export declare const topologyOf: (lab: string, dir: string, client: string, target: string) => string;
+/** A route line is one line of `ip -ts -o monitor route`, with its time on the wall clock in ms. */
+interface RouteLine {
+  readonly wall: number;
+  readonly deleted: boolean;
+  readonly prefix: string;
+  readonly via: string | null;
+  readonly dev: string | null;
+}
+/** This function reads a line of `ip -ts -o monitor route`, or it gives null for another line. The clock of a container is UTC. */
+export declare const routeLine: (text: string) => RouteLine | null;
+/** A clock pair is one instant on the monotonic clock in ns and on the wall clock in ms. */
+interface ClockPair {
+  readonly mono: number;
+  readonly wall: number;
+}
+/**
+ * This function takes a time on the wall clock to the monotonic clock. The harness runs on the host
+ * of the containers, thus `process.hrtime` reads the CLOCK_MONOTONIC of the injector and the
+ * prober. The error of the pair is approximately 1 ms, the step of `Date.now`.
+ */
+export declare const monotonicAt: (wall: number, pair: ClockPair) => number;
+/** This function gives the state of the BGP sessions from `show bgp neighbors json`. */
+export declare const sessionsOf: (json: string) => {
+  readonly up: number;
+  readonly established: number;
+};
+/** These are the views of a leaf that tell if a run can start on it. */
+interface LeafViews {
+  readonly neighbors: string;
+  /** This text is the kernel's route to the far network, from `ip -o route show`. */
+  readonly route: string;
+  /** This text is `show bgp ipv4 unicast PREFIX json`. */
+  readonly paths: string;
+  /** This text is `show bfd peers json`, or null when the fabric has no BFD. */
+  readonly bfd: string | null;
+}
+/**
+ * This function tells what a leaf still lacks before a run, or it gives null. The two sessions must
+ * be up and the route must be on spine1. Spine2 must hold the other path. Without these, the fault
+ * of a run acts on another path, or nothing can take over.
+ */
+export declare const leafLacks: (views: LeafViews) => string | null;
+/**
+ * This function gives the later move of the leaves after an instant: for each leaf, the first route
+ * to its far network that leaves spine1. It gives null when a leaf has no move. The later move ends
+ * the outage in the two directions.
+ */
+export declare const laterMove: (routes: readonly RunRoute[], after: number) => number | null;
+/** A fabric lab is the open fabric of a lab. */
+interface FabricLab {
+  /** These are the lines of the route monitors of the leaves, on the monotonic clock. */
+  readonly routes: readonly RunRoute[];
+  /** This method drops each packet of spine1 in one commit, and the links stay up. */
+  silence(): Promise<{
+    t0: number;
+    t1: number;
+  }>;
+  /** This method ends a silence. */
+  clear(): Promise<{
+    t0: number;
+    t1: number;
+  }>;
+  /** This method waits until each leaf is on spine1 with each session up. */
+  ready(): Promise<void>;
+  /** This method gives the count of established sessions of each leaf so far. */
+  sessions(): Promise<Readonly<Record<string, number>>>;
+  /** This method gives the record of the fabric for a manifest. */
+  manifest(): FabricManifest;
+  /** This method destroys the routers. */
+  close(): Promise<void>;
+}
+/** These are the parts of a lab that a fabric uses. */
+interface FabricParts {
+  readonly fabric: Fabric;
+  readonly image: string;
+  readonly client: StartedTestContainer;
+  readonly server: StartedTestContainer;
+  readonly admin: StartedTestContainer;
+  readonly injector: StartedTestContainer;
+  readonly opened: StartedTestContainer[];
+  readonly output: (container: StartedTestContainer, argv: string[]) => Promise<string>;
+}
+/**
+ * This function deploys the routers of a lab and joins the client and the target to them. It also
+ * starts the sidecars: the injector of spine1 and a route monitor on each leaf. The caller stops the
+ * sidecars with the other containers of the lab, and it calls `close` first.
+ */
+export declare const openFabric: (parts: FabricParts) => Promise<FabricLab>;
+//#endregion
+//#region src/image.d.ts
+/** These are the names of the images that the harness ships. */
+export declare const SHIPPED_NAMES: readonly [Shipped, ...Shipped[]];
+/** This function gives the source of a shipped image. */
+export declare const shippedSource: (name: Shipped) => ImageSource;
+/** This function gives the version of the package of the harness, from its package.json. */
+export declare const harnessVersion: () => string;
+/**
+ * These are the clients that the harness ships, by their names (`Client`): the image of each, and
+ * its command. A cell that names a client gets this image and this command.
+ */
+export declare const SHIPPED_CLIENTS: Readonly<Record<Client, ClientImage>>;
+/** These are the names of the clients that the harness ships. */
+export declare const CLIENT_NAMES: readonly [Client, ...Client[]];
+/** This function gives the command of a program of a subject by its role. It fails when the subject has no such program. */
+export declare const programOf: (subject: Subject, role: string) => readonly string[];
+/**
+ * This function gives the client of a subject for a cell (`Cell.client`): the image of the subject,
+ * with the command of a program. A client that the harness ships goes by its name, thus its records
+ * stay the same.
+ */
+export declare const subjectClient: (subject: Subject, role?: string) => Client | ClientImage;
+/** This pattern matches an image pinned by digest: a name, a tag or none, then `@sha256:` and 64 hex digits. */
+export declare const PINNED: RegExp;
+/** This pattern matches the repository of a tag that the harness builds. Its parts are lower case, with `/` between them. */
+export declare const REPOSITORY: RegExp;
+/** This function tells if a path stays in its context: relative, with `/` between its parts, and no `..`. */
+export declare const inContext: (path: string) => boolean;
+/**
+ * This function gives the problems of the source of an image, as data. A pinned image names a
+ * digest. A build names a repository, a context, and paths that stay in the context. A shipped image
+ * is one that the harness ships.
+ */
+export declare const imageProblems: (source: ImageSource) => string[];
+/**
+ * This function gives the problems of a subject, as data. A subject has a name and a base that a
+ * digest pins. Its image source has no problems, and each of its programs is a command.
+ */
+export declare const subjectProblems: (subject: Subject) => string[];
+/**
+ * This function gives each `FROM` of a Dockerfile that no digest pins. A `FROM` of an earlier stage
+ * by its name, or of `scratch`, needs no digest. A base from an `ARG` has no digest that the harness
+ * can read, thus it is in the list too.
+ */
+export declare const unpinnedBases: (dockerfile: string) => string[];
+/**
+ * This function gives each base of a Dockerfile, in the sequence of the file. A base is the image of
+ * a `FROM` that is not `scratch` or an earlier stage by its name.
+ */
+export declare const basesOf: (dockerfile: string) => string[];
+/**
+ * This function gives each manifest of packages in a context (`package.json`, `Cargo.toml`) that
+ * has no lockfile in its folder or in a folder above it. The paths are the paths in the context.
+ */
+export declare const unlockedManifests: (paths: readonly string[]) => string[];
+/**
+ * An image plan tells how the harness gets the image of a source. It is a pull of an image by its
+ * name, or a build of a context. A build has its tag, its Dockerfile, and each file with its path.
+ */
+type ImagePlan = {
+  readonly kind: "pull";
+  readonly image: string;
+} | {
+  readonly kind: "build";
+  readonly tag: string;
+  readonly dockerfile: string;
+  readonly files: ReadonlyArray<{
+    readonly file: string;
+    readonly at: string;
+  }>;
+  /** This record tells what the image holds, for the manifest of a run. */
+  readonly record: BuildRecord;
+};
+/**
+ * A build record tells what an image that the harness builds holds, for the manifest of a run. The
+ * tag is a sha256 of each file of the context. The bases are pinned by digest, and each lockfile
+ * has its sha256. Thus the record pins each program of the image. The versions are the part that a
+ * person reads.
+ */
+interface BuildRecord {
+  readonly tag: string;
+  /** These are the bases of its Dockerfile, each pinned by digest (`basesOf`). */
+  readonly bases: readonly string[];
+  /** These are the lockfiles of its context by their path, each with the sha256 of its content (`locksOf`). */
+  readonly locks: Readonly<Record<string, string>>;
+  /** These are the versions of the packages that its manifests name directly, by their names (`versionsOf`). */
+  readonly versions: Readonly<Record<string, string>>;
+}
+/**
+ * This function gives the plan of the image of a source. A stock image by its name, or a pinned
+ * image, is a pull. A build or a shipped image is a build. Its tag is its name, then 12 digits of a
+ * sha256. The sha256 reads the path of the Dockerfile, and the path and the content of each file.
+ * Thus a changed program cannot operate as an old one.
+ *
+ * A build fails before it starts on a problem of its source. It also fails on a `FROM` that no
+ * digest pins, and on a manifest of packages without a lockfile.
+ */
+export declare const planOf: (source: string | ImageSource) => ImagePlan;
+/**
+ * This function gives an image that a lab can start, from its source. It is the name of a stock
+ * image or of a pinned image, which the lab pulls, or the tag of a build. A build goes through the CLI
+ * of Docker.
+ */
+export declare const imageOf: (source: string | ImageSource) => Promise<string>;
+/**
+ * This function gives the build record of the image of a source, for the manifest of a run. A
+ * stock image by its name, or a pinned image, is a pull, and it has no build record.
+ */
+export declare const buildRecordOf: (source: string | ImageSource) => BuildRecord | undefined;
+/** This function gives the image of the harness: the prober, the TCP client, the injector and the namespace admin. */
+export declare const harnessImage: () => Promise<string>;
+/** This function gives the image of the Node client. */
+export declare const nodeClientImage: () => Promise<string>;
+/** This function gives the image of the router. */
+export declare const routerImage: () => Promise<string>;
+/** This function gives the image of the Go HTTP client. */
+export declare const goHttpImage: () => Promise<string>;
+/** This function gives the image of the Node HTTP clients. */
+export declare const nodeHttpImage: () => Promise<string>;
+/** This function gives the image of the MongoDB client. */
+export declare const mongoClientImage: () => Promise<string>;
+/** This function gives the image of the PostgreSQL client. */
+export declare const pgClientImage: () => Promise<string>;
+/** This function gives the image of the client of HikariCP over pgjdbc. */
+export declare const hikariClientImage: () => Promise<string>;
+/** This function gives the image of the RabbitMQ client: amqplib, with the client and the admin of a cluster cell. */
+export declare const amqpClientImage: () => Promise<string>;
+/** This function gives the image of the Kafka client: kafka-clients on Java, with the client and the admin of a Kafka cell. */
+export declare const kafkaClientImage: () => Promise<string>;
+/** This function gives the image of the gRPC client of the harness: grpc-js 1.14.6 on Node 24.21.0, with its client. */
+export declare const grpcClientImage: () => Promise<string>;
+/**
+ * This function gives the source of the image that a target operates. It is the image of its
+ * subject (`Target.subject`), or its stock image.
+ */
+export declare const targetSourceOf: (target: Target) => string | ImageSource;
+/** This function gives the image that a target operates: the image of its subject, or its stock image. */
+export declare const targetImageOf: (target: Target) => Promise<string>;
+//#endregion
 //#region src/substrate.d.ts
 /**
  * A layer is a rung of the laboratory (`docs/S7.md`). E1 is containers on one kernel, and E2 is a
- * cluster of containers on one kernel. The other layers do not have a runner at this time.
+ * cluster of containers on one kernel. A lab with `Topology.fabric` writes E3: routers in containers.
+ * A lab in a guest of QEMU writes E4 (`substrateHere`). The other layers do not have a runner at this
+ * time.
  */
 type Layer = "E0" | "E1" | "E2" | "E3" | "E4" | "E5" | "E6";
 /** A cloud is where the runner operates. GitHub can schedule a standard Linux job on Azure or on AWS. */
@@ -804,6 +1258,27 @@ export declare const cloudOf: (vendor: string | null) => Cloud;
 export declare const readHost: (root?: string, env?: Readonly<Record<string, string | undefined>>, kernel?: string) => Host;
 /** This function gives the substrate of a layer on a host. */
 export declare const substrateOn: (layer: Layer, host?: Host) => Substrate;
+/**
+ * This function gives the substrate of a lab at this place. A guest cannot read its own runner: its
+ * DMI vendor is QEMU, and `ImageOS` is not set. Thus the runner writes its host to `LAB_RUNNER`, and
+ * the guest keeps only its own kernel and HZ. A value of that variable makes the layer E4.
+ */
+export declare const substrateHere: (layer: Layer, host?: Host, env?: Readonly<Record<string, string | undefined>>) => Substrate;
+/** A CPU reading is two counters of `/proc/stat`, in ticks of the clock. */
+interface CpuTime {
+  /** This count is the time that the host gave to another guest while this one had work. */
+  readonly steal: number;
+  /** This count is the sum of all states of each CPU. */
+  readonly total: number;
+}
+/** This function reads the CPU time of this machine, or gives null without `/proc/stat`. `root` is for tests. */
+export declare const readCpu: (root?: string) => CpuTime | null;
+/**
+ * This function gives the share of the CPU time between two readings that the host took. The share
+ * goes from zero to one. It is null if no tick passed. A guest sees only the host above it. On a hosted
+ * runner that host is itself a guest, and this number does not show the contention of the layer below it.
+ */
+export declare const stealShare: (before: CpuTime, after: CpuTime) => number | null;
 /** A part of a record that can hold a substrate: a manifest of the harness, or the fields of a kind manifest. */
 interface SubstrateSource {
   readonly substrate?: Substrate;
@@ -865,6 +1340,8 @@ interface FaultOptions {
   readonly code?: "host" | "net" | "port" | "admin";
   /** This length is the longest segment in bytes that an MTU drop lets through. */
   readonly above?: number;
+  /** These are the addresses that the rule faces, in place of the peer of its place. */
+  readonly peers?: readonly string[];
 }
 /**
  * A fault side is the namespace that gets the rules of a fault: the namespace of the peer of the
@@ -901,6 +1378,8 @@ interface Manifest {
   readonly docker: string;
   /** This block tells the layer, the cloud and the kernel of the record. A record from before it reads through `substrateOf`. */
   readonly substrate?: Substrate;
+  /** This record is the fabric of a cell with `Topology.fabric`. */
+  readonly fabric?: FabricManifest;
   /** This record is the target, and its settings as it read them back. */
   readonly target: {
     readonly image: string;
@@ -915,11 +1394,20 @@ interface Manifest {
     readonly files?: Readonly<Record<string, string>>;
     /** This line is the first line of a target that writes them: a server that tells its settings as it reads them. */
     readonly first?: ProbeEvent;
+    /** This record tells what the image of a target holds when the harness builds it: the image of a subject (`buildRecordOf`). */
+    readonly build?: BuildRecord;
   };
+  /**
+   * This record is the harness: its image, the id of that image and its Alpine packages. It also
+   * has the version of the package of the harness, and the build of its image. The record of a
+   * cell gives the commit of the code that operated.
+   */
   readonly harness: {
     readonly image: string;
     readonly id: string;
     readonly packages: string[];
+    readonly version?: string;
+    readonly build?: BuildRecord;
   };
   /**
    * This record is the client of each run: the TCP client of the harness, a driver, or an image that
@@ -929,6 +1417,7 @@ interface Manifest {
     readonly kind: Client | "image";
     readonly image: string;
     readonly source?: ImageSource;
+    readonly build?: BuildRecord;
   };
   /** These are the namespaces that the prober watched, by inode. */
   readonly netns: {
@@ -994,8 +1483,17 @@ interface Agent {
   exec(argv: readonly string[]): Promise<string>;
   stop(): Promise<void>;
 }
+/** A sibling is a server that a lab starts on the network of the target, for example a standby. */
+interface Sibling {
+  readonly ip: string;
+  readonly netns: number;
+  /** This method starts a command in the sibling, and gives its standard output. */
+  exec(argv: readonly string[]): Promise<string>;
+}
 /** A lab is one environment: the target, the client, the hops of a topology, the injectors and the prober. */
 interface Lab {
+  /** This address is the address of the target on its own network. */
+  readonly serverIp: string;
   /** This address is the address that the client connects to: the target, or the first proxy of a topology. */
   readonly targetIp: string;
   /** This port is the port that the client connects to. */
@@ -1026,6 +1524,10 @@ interface Lab {
   agent(image: string, command: readonly string[], options?: {
     readonly pid?: boolean;
   }): Promise<Agent>;
+  /** This method starts a sibling, and waits for a line of its log. */
+  sibling(image: string, command: readonly string[], ready: string): Promise<Sibling>;
+  /** This method starts a command in the target, and gives its standard output. */
+  exec(argv: readonly string[]): Promise<string>;
   /** These are all the records that the prober wrote, in sequence. */
   readonly kernel: readonly KernelRecord[];
   /** This method sets the sysctls of the client namespace through its privileged sidecar, and reads them back. */
@@ -1086,6 +1588,8 @@ interface Lab {
    * independent of the run before it.
    */
   forget(): Promise<void>;
+  /** This part is the routed fabric of a lab with `Topology.fabric`. */
+  readonly fabric?: FabricLab;
   /** This method freezes the target with the cgroup freezer: its process stops, and its kernel continues to answer. */
   freeze(): Promise<void>;
   thaw(): Promise<void>;
@@ -1152,6 +1656,20 @@ export declare const hzOf: (first: {
   now: number;
   jiffies: number;
 }) => string | null;
+/** This function gives the build record of a source as a part of a manifest: `{ build }`, or nothing for an image that the harness pulls. */
+export declare const buildOf: (source: Parameters<typeof buildRecordOf>[0]) => {
+  readonly build?: BuildRecord;
+};
+/** This function gives the version of the harness and the build of its image, as a part of the harness record of a manifest. */
+export declare const harnessRecord: () => {
+  readonly version: string;
+  readonly build?: BuildRecord;
+};
+/**
+ * This function gives the record of the client of a run that the harness ships, for a manifest:
+ * its kind, its image, and its build.
+ */
+export declare const shippedClientRecord: (kind: Client, image: string) => Manifest["client"];
 /** This function gives the device of a route from its line in `ip -o route get`: "10.0.0.2 dev eth0 src …". */
 export declare const deviceOf: (route: string) => string;
 /**
@@ -1176,111 +1694,108 @@ export declare const deviceOf: (route: string) => string;
  */
 export declare const openLab: (target: Target, kind?: Client | ClientImage, topology?: Topology) => Promise<Lab>;
 //#endregion
-//#region src/image.d.ts
-/** These are the names of the images that the harness ships. */
-export declare const SHIPPED_NAMES: readonly [Shipped, ...Shipped[]];
-/**
- * These are the clients that the harness ships, by their names (`Client`): the image of each, and
- * its command. A cell that names a client gets this image and this command.
- */
-export declare const SHIPPED_CLIENTS: Readonly<Record<Client, ClientImage>>;
-/** These are the names of the clients that the harness ships. */
-export declare const CLIENT_NAMES: readonly [Client, ...Client[]];
-/** This function gives the command of a program of a subject by its role. It fails when the subject has no such program. */
-export declare const programOf: (subject: Subject, role: string) => readonly string[];
-/**
- * This function gives the client of a subject for a cell (`Cell.client`): the image of the subject,
- * with the command of a program. A client that the harness ships goes by its name, thus its records
- * stay the same.
- */
-export declare const subjectClient: (subject: Subject, role?: string) => Client | ClientImage;
-/** This pattern matches an image pinned by digest: a name, a tag or none, then `@sha256:` and 64 hex digits. */
-export declare const PINNED: RegExp;
-/** This pattern matches the repository of a tag that the harness builds. Its parts are lower case, with `/` between them. */
-export declare const REPOSITORY: RegExp;
-/** This function tells if a path stays in its context: relative, with `/` between its parts, and no `..`. */
-export declare const inContext: (path: string) => boolean;
-/**
- * This function gives the problems of the source of an image, as data. A pinned image names a
- * digest. A build names a repository, a context, and paths that stay in the context. A shipped image
- * is one that the harness ships.
- */
-export declare const imageProblems: (source: ImageSource) => string[];
-/**
- * This function gives the problems of a subject, as data. A subject has a name and a base that a
- * digest pins. Its image source has no problems, and each of its programs is a command.
- */
-export declare const subjectProblems: (subject: Subject) => string[];
-/**
- * This function gives each `FROM` of a Dockerfile that no digest pins. A `FROM` of an earlier stage
- * by its name, or of `scratch`, needs no digest. A base from an `ARG` has no digest that the harness
- * can read, thus it is in the list too.
- */
-export declare const unpinnedBases: (dockerfile: string) => string[];
-/**
- * This function gives each manifest of packages in a context (`package.json`, `Cargo.toml`) that
- * has no lockfile in its folder or in a folder above it. The paths are the paths in the context.
- */
-export declare const unlockedManifests: (paths: readonly string[]) => string[];
+//#region src/standby.d.ts
+/** This text is the line of the log of a standby that serves reads (src/backend/postmaster/postmaster.c:3740). */
+export declare const STANDBY_READY: string;
+/** This text is the write of the client, with its sequence number. */
+export declare const INSERT: string;
+/** This function gives the command of a standby: a base backup of the primary, then the server in recovery. */
+export declare const standbyCommand: (primary: string) => string[];
+/** This command starts the old primary again, after the stop of each of its processes. Its crash recovery runs before the wait ends. */
+export declare const RESTART: readonly string[];
+/** This function reads the output of a query for sequence numbers into a sorted list. */
+export declare const numbersOf: (text: string) => number[];
+/** This function gives the arguments of the client of a run. */
+export declare const pairArgs: (cell: Cell, lab: Pick<Lab, "targetIp" | "targetPort">) => string[];
+/** This function gives the instants of the cancels of a hold, in ns from the fault: equal parts of the hold. */
+export declare const cancelsOf: (pair: Pair, holdNs: number) => number[];
+/**
+ * This function opens the pair of an environment. The table comes first and the standby next: the
+ * base backup has it. A synchronous standby is named last, thus no setup step waits for it.
+ */
+export declare const openStandby: (lab: Lab, cell: Cell) => Promise<Sibling>;
+/**
+ * This function does one run of a pair. A dry run writes and stops, with no fault: the warm-up of the
+ * environment. It removes its rows, thus the numbers of the run start on an empty table.
+ */
+export declare const pairOnce: (lab: Lab, standby: Sibling, cell: Cell, environment: number, index: number, dry?: boolean) => Promise<Run>;
+//#endregion
+//#region src/locks.d.ts
 /**
- * An image plan tells how the harness gets the image of a source. It is a pull of an image by its
- * name, or a build of a context. A build has its tag, its Dockerfile, and each file with its path.
+ * This module reads the lockfiles of a build context for the manifest of a run. A lockfile pins each
+ * package that a build installs, thus its sha256 pins the packages that the image holds. The
+ * versions of the packages that each manifest names directly are the part that a person reads.
  */
-type ImagePlan = {
-  readonly kind: "pull";
-  readonly image: string;
-} | {
-  readonly kind: "build";
-  readonly tag: string;
-  readonly dockerfile: string;
-  readonly files: ReadonlyArray<{
-    readonly file: string;
-    readonly at: string;
-  }>;
-};
-/**
- * This function gives the plan of the image of a source. A stock image by its name, or a pinned
- * image, is a pull. A build or a shipped image is a build. Its tag is its name, then 12 digits of a
- * sha256. The sha256 reads the path of the Dockerfile, and the path and the content of each file.
- * Thus a changed program cannot operate as an old one.
- *
- * A build fails before it starts on a problem of its source. It also fails on a `FROM` that no
- * digest pins, and on a manifest of packages without a lockfile.
- */
-export declare const planOf: (source: string | ImageSource) => ImagePlan;
+/** A context file is a file of a build context: its path in the context, and its content. */
+interface ContextFile {
+  readonly at: string;
+  readonly content: Buffer | string;
+}
 /**
- * This function gives an image that a lab can start, from its source. It is the name of a stock
- * image or of a pinned image, which the lab pulls, or the tag of a build. A build goes through the CLI
- * of Docker.
- */
-export declare const imageOf: (source: string | ImageSource) => Promise<string>;
-/** This function gives the image of the harness: the prober, the TCP client, the injector and the namespace admin. */
-export declare const harnessImage: () => Promise<string>;
-/** This function gives the image of the Node client. */
-export declare const nodeClientImage: () => Promise<string>;
-/** This function gives the image of the router. */
-export declare const routerImage: () => Promise<string>;
-/** This function gives the image of the Go HTTP client. */
-export declare const goHttpImage: () => Promise<string>;
-/** This function gives the image of the Node HTTP clients. */
-export declare const nodeHttpImage: () => Promise<string>;
-/** This function gives the image of the MongoDB client. */
-export declare const mongoClientImage: () => Promise<string>;
-/** This function gives the image of the PostgreSQL client. */
-export declare const pgClientImage: () => Promise<string>;
-/** This function gives the image of the client of HikariCP over pgjdbc. */
-export declare const hikariClientImage: () => Promise<string>;
-/** This function gives the image of the RabbitMQ client: amqplib, with the client and the admin of a cluster cell. */
-export declare const amqpClientImage: () => Promise<string>;
-/** This function gives the image of the gRPC client of the harness: grpc-js 1.14.6 on Node 24.21.0, with its client. */
-export declare const grpcClientImage: () => Promise<string>;
+ * These are the names of the lockfiles that the harness records. A `go.mod` is the lockfile of a Go
+ * module, because its `require` lines give exact versions. For them, Go selects the same packages
+ * each time, and its checksum database examines each package.
+ */
+export declare const LOCK_NAMES: ReadonlySet<string>;
+/** This function gives each lockfile of a context by its path, with the sha256 of its content in hex. */
+export declare const locksOf: (files: readonly ContextFile[]) => Record<string, string>;
+/** This function gives the version of each direct dependency of an npm lockfile (lockfileVersion 2 or 3). */
+export declare const npmVersions: (lock: string) => Record<string, string>;
+/**
+ * This function gives the version of each direct dependency of the root package of a Cargo.lock: the
+ * package that the Cargo.toml beside it names. An entry of a dependency list is a name, or a name
+ * and a version when the lock holds two versions of that name.
+ */
+export declare const cargoVersions: (lock: string, manifest: string) => Record<string, string>;
+/** This function gives the version of each direct requirement of a go.mod. A requirement that is `// indirect` is not direct. */
+export declare const goVersions: (mod: string) => Record<string, string>;
+/** This function gives the version of each jar of a list of jars by their SHA-256: `<hex>  HikariCP-7.1.0.jar` gives HikariCP 7.1.0. */
+export declare const jarVersions: (list: string) => Record<string, string>;
+/**
+ * This function gives the version of each package that the manifests of a context name directly,
+ * by its name. It reads each npm lockfile, Cargo.lock, go.mod and list of jars. A package of a
+ * workspace in the context is a package of the image too, with the version of its package.json.
+ * Such a package has a name and a version, and it is not private. The record has the sha256 of
+ * the lockfile of a pnpm, yarn or bun workspace (`locksOf`). This function does not read versions
+ * from such a lockfile.
+ */
+export declare const versionsOf: (files: readonly ContextFile[]) => Record<string, string>;
+//#endregion
+//#region src/pins.d.ts
+/**
+ * This function gives each problem of a manifest that stops it from pinning the programs of its
+ * run. An empty list means that the manifest pins each program.
+ * - the target: an image by its digest, or the build of a subject with its build record and its base
+ *   by digest.
+ * - the harness: the version of its package, and the build record of its image.
+ * - the client: an image by its digest, or a build with its build record.
+ * - each hop: an image by its digest, or the tag of a build and the id of its image.
+ */
+export declare const manifestProblems: (manifest: Manifest) => string[];
+/** This function gives the problem of the commit of a record: no commit, or a text that is not a commit of git. */
+export declare const commitProblems: (commit: string | undefined) => string[];
+/** The code of a record is the commit of the checkout that operated, and if its tracked files had changes. */
+interface RecordCode {
+  readonly commit: string;
+  readonly dirty: boolean;
+}
+/** These are the parts of a record that tell if it pins the programs of its runs. */
+interface Pinnable {
+  readonly code?: RecordCode | undefined;
+  readonly manifests: readonly Manifest[];
+  /** A local debug run marks its record so. No page reads such a record. */
+  readonly debug?: boolean | undefined;
+}
+/** This problem is the problem of a checkout with changes: its commit does not hold the code that operated. */
+export declare const DIRTY = "the checkout has changes to tracked files that no commit holds";
+/** This problem is the problem of a record of a local debug run. */
+export declare const DEBUG = "the record is of a local debug run, and no page reads it";
 /**
- * This function gives the source of the image that a target operates. It is the image of its
- * subject (`Target.subject`), or its stock image.
+ * This function gives each problem that stops a record from pinning the programs of its runs. A
+ * record with a problem is not a record (SYS-101). The writer of the cells refuses it, and the read
+ * of a schedule refuses it. A commit with changes to tracked files is not a pin.
  */
-export declare const targetSourceOf: (target: Target) => string | ImageSource;
-/** This function gives the image that a target operates: the image of its subject, or its stock image. */
-export declare const targetImageOf: (target: Target) => Promise<string>;
+export declare const recordProblems: (record: Pinnable) => string[];
 //#endregion
 //#region src/group.d.ts
 /**
@@ -1357,8 +1872,8 @@ export declare const memberConfig: (group: Group, primary: string | null) => str
 export declare const sentinelConfig: (group: Group, primary: string) => string;
 /**
  * This function gives the peers that a fault on a part cuts it from, by their addresses. `all` is
- * each other part and the client, `group` is the other parts, and `clients` is the client. `one`
- * and `one-way` are the first other member. The observer is not a peer, thus the timeline of the
+ * each other part and the client, `group` is the other parts, and `clients` and `to-clients` are
+ * the client. `one` and `one-way` are the first other member. The observer is not a peer, thus the timeline of the
  * group stays whole.
  */
 export declare const peersOf: (parts: {
@@ -1527,6 +2042,12 @@ interface ReplicaSetLab {
   signal(part: string, signal: "KILL" | "STOP" | "CONT", at?: number): Promise<Stamp>;
   /** This method starts the server of a member again, after a fin. */
   restart(member: string): Promise<void>;
+  /** This method runs a command in the box of a member, and gives its output. */
+  exec(member: string, argv: string[]): Promise<string>;
+  /** This method runs a script of the shell on a member, against its own server only, and gives its output. Another port reads a server with no set. */
+  script(member: string, script: string, port?: number): Promise<string>;
+  /** This method runs a command on the backup host of a drill, and gives its output. A lab without that host refuses it. */
+  vault(argv: string[]): Promise<string>;
   /** This method sends replSetStepDown to the primary. */
   stepDown(primary: string): Promise<Stamp>;
   /** This method removes the TCP metrics of the client namespace: each run starts from a kernel that keeps nothing. */
@@ -1550,8 +2071,9 @@ export declare const STEP_DOWN: {
  * - an injector sidecar for each member, in its network and process namespaces.
  * - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
  * - the prober, which watches the namespace of the client.
+ * - a backup host for a recovery drill: the stock image, for its tools, with no server.
  */
-export declare const openReplicaSetLab: (set: ReplicaSet) => Promise<ReplicaSetLab>;
+export declare const openReplicaSetLab: (set: ReplicaSet, backups?: boolean) => Promise<ReplicaSetLab>;
 /** This function gives the arguments of the client of a replica set cell: each member, and the stream of writes. */
 export declare const replicaSetArgs: (cell: Cell, members: Readonly<Record<string, RunPart>>) => string[];
 /**
@@ -1569,6 +2091,18 @@ export declare const replicaSetArgs: (cell: Cell, members: Readonly<Record<strin
  */
 export declare const replicaSetOnce: (lab: ReplicaSetLab, cell: Cell, environment: number, index: number) => Promise<Run>;
 //#endregion
+//#region src/drill.d.ts
+/**
+ * This function does one run of a recovery drill:
+ * - The set converges, and it loses its writes. The tenants get their data, and a hash reads it.
+ * - The client writes until its stream is steady. A control stops it here.
+ * - The backups go, each at its interval. The fault comes at the age of the newest backup.
+ * - An election continues the same client. A restore stops the client, and it starts a second one.
+ * - The set converges again. The writes that it then holds tell which writes it kept, and a second
+ *   hash tells if the tenants have the same data.
+ */
+export declare const drillOnce: (lab: ReplicaSetLab, cell: Cell, environment: number, index: number) => Promise<Run>;
+//#endregion
 //#region src/cluster.d.ts
 /**
  * This module is the lab of a cluster of RabbitMQ with one quorum queue. The members of the queue
@@ -1639,6 +2173,7 @@ export declare const clusterViewOf: (cluster: Cluster, rows: readonly StatusRow[
  * This function gives the peers that a fault on a node cuts it from. `all` is each other node and the
  * client, `group` is the other nodes, and `clients` is the client. `one` is the first other node in
  * the sequence of the cluster, and `one-way` is the same node for the segments from the part only.
+ * `to-clients` is the client, for the segments from the part only.
  */
 export declare const clusterCutOf: (cluster: Cluster, members: Readonly<Record<string, RunPart>>, client: string, part: string, cuts: NonNullable<GroupPlace["cuts"]>) => Cut;
 /** A wall stamp is the wall clock in ns around a command of a node: the run puts it on the monotonic clock. */
@@ -1739,6 +2274,106 @@ export declare const restamped: (line: NodeLine, events: readonly ProbeEvent[])
  */
 export declare const clusterOnce: (lab: ClusterLab, cell: Cell, environment: number, index: number) => Promise<Run>;
 //#endregion
+//#region src/kafka.d.ts
+/** This function gives the node id of a name. A broker has its place in the list from 1, and the controller has the id after them. */
+export declare const nodeIdOf: (kafka: Kafka, name: string) => number;
+/**
+ * This function gives the variable of a server property for the image. The wrapper of the image
+ * reverses this map (KafkaDockerWrapper.scala:188-203). After the prefix `KAFKA_`, `___` is a
+ * hyphen, `__` is an underscore and `_` is a dot.
+ */
+export declare const envOf: (property: string) => string;
+/** This function gives the environment of a broker. */
+export declare const brokerEnv: (kafka: Kafka, name: string) => Record<string, string>;
+/** This function gives the environment of the controller. The start script of the image stops on an advertised listener here (configure:43-48). */
+export declare const controllerEnv: (kafka: Kafka) => Record<string, string>;
+/** A partition view is the output of the command `view` of the admin. Without a topic it has only the brokers. */
+interface PartitionView {
+  readonly brokers: readonly number[];
+  readonly leader?: number | null;
+  readonly isr?: readonly number[];
+  readonly replicas?: readonly number[];
+}
+/**
+ * This function judges a view of a partition: its leader, and if the group is healthy. A healthy
+ * group has each broker not fenced, one leader, and each broker in the in-sync replicas.
+ */
+export declare const kafkaViewOf: (kafka: Kafka, view: PartitionView) => GroupView;
+/**
+ * This function gives the peers that a fault on a broker cuts it from. Only `group` has a law: the
+ * other brokers and the controller, and the client keeps its route to the broker.
+ */
+export declare const kafkaCutOf: (kafka: Kafka, members: Readonly<Record<string, RunPart>>, controller: RunPart, part: string, cuts: NonNullable<GroupPlace["cuts"]>) => Cut;
+/** This function gives the addresses that the client and the admin bootstrap from. */
+export declare const bootstrapOf: (kafka: Kafka) => string;
+/** This function gives the name of the topic of a run. A warm-up has the index -1. */
+export declare const topicOf: (environment: number, index: number) => string;
+/** A Kafka lab is one environment of a Kafka cell. */
+interface KafkaLab {
+  readonly kafka: Kafka;
+  readonly clientIp: string;
+  readonly members: Readonly<Record<string, RunPart>>;
+  readonly controller: RunPart;
+  readonly netns: {
+    readonly client: number;
+  };
+  /** These are all the records that the prober wrote, in sequence. */
+  readonly kernel: readonly KernelRecord[];
+  /** These are the last raw lines of the console of each node, for the diagnosis of a lab: at most RAW_TAIL of each. */
+  readonly raw: Readonly<Record<string, readonly string[]>>;
+  /** This method sets the sysctls of the client namespace through its privileged sidecar, and reads them back. */
+  sysctl(values: Readonly<Record<string, string>>): Promise<Record<string, string>>;
+  /** This method waits until each broker is not fenced. */
+  brokersUp(within?: number): Promise<void>;
+  /** This method makes the topic of a run. */
+  create(topic: string): Promise<void>;
+  /** This method waits until the partition is whole and calm, and gives the name of the broker of its leader. */
+  converge(topic: string, within?: number): Promise<string>;
+  /** This method gives the sequence numbers that the partition holds, one entry for each copy. */
+  holds(topic: string): Promise<number[]>;
+  /** This method starts the client of a run, with the arguments of `ClientKafka`. */
+  client(args: readonly string[]): Promise<GroupClient>;
+  /** This method applies a network fault between a broker and some of its peers, at once or at an instant of the VM. */
+  fault(kind: GroupNetworkFault, part: string, cuts: NonNullable<GroupPlace["cuts"]>, at?: number): Promise<Stamp>;
+  /** This method removes a network fault, at once or at an instant of the VM. */
+  lift(kind: GroupNetworkFault, part: string, cuts: NonNullable<GroupPlace["cuts"]>, at?: number): Promise<Stamp>;
+  /** This method removes the TCP metrics of the client namespace: each run starts from a kernel that keeps nothing. */
+  forget(): Promise<void>;
+  /** This method waits for the next clock line of the prober, which gives the count of lost records. */
+  settle(): Promise<number | null>;
+  manifest(sysctls: Readonly<Record<string, string>>): Promise<Manifest>;
+  close(): Promise<void>;
+}
+/**
+ * This function opens one environment of a Kafka cell:
+ * - one new network, with each node, the client and the commander on it.
+ * - the controller first, then each broker: the stock image with its own command. The brokers
+ *   register with the controller, thus they need it first.
+ * - an injector sidecar for each node, in its network and process namespaces.
+ * - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
+ * - the prober, which watches the namespace of the client.
+ */
+export declare const openKafkaLab: (kafka: Kafka) => Promise<KafkaLab>;
+/**
+ * This time is how long a run continues after its lift. The record then shows the step-down of the
+ * leader and the writes after it. It has no source. It is above the largest step-down of the
+ * law, which is approximately 2 s after the lift.
+ */
+export declare const KAFKA_TAIL_MS = 1e4;
+/** This function gives the arguments of the client of a Kafka cell: the topic, the writers, the stream of writes and the settings. */
+export declare const kafkaArgs: (cell: Cell, topic: string) => string[];
+/**
+ * This function does one run of a Kafka cell:
+ * - Each broker is up, the topic of the run exists, and its partition is whole. The client starts,
+ *   and its stream of writes gets steady.
+ * - The fault cuts the leader from the other brokers and the controller, and it lifts at its instant
+ *   in the VM. The client keeps its route to the leader, thus only the group knows of the cut.
+ * - The run waits for the lift and for the tail after it.
+ * - Each fault goes, the client stops, and the partition converges again. The admin then reads
+ *   the whole partition, thus the record tells which writes the group kept.
+ */
+export declare const kafkaOnce: (lab: KafkaLab, cell: Cell, environment: number, index: number) => Promise<Run>;
+//#endregion
 //#region src/run.d.ts
 /** A cell result is a cell with its manifests, its runs, their measurements and its verdict. */
 interface CellResult {
@@ -1787,5 +2422,5 @@ export declare const spread: (runs: number, environments: number) => number[];
  */
 export declare const runCell: (cell: Cell, say?: (line: string) => void) => Promise<CellResult>;
 //#endregion
-export { type Agent, type BuildImage, type CallReading, type Cell, type CellResult, type Client, type ClientImage, type Cloud, type Cluster, type ClusterClientAt, type ClusterLab, type ClusterStage, type Crossings, type Cut, type Decision, type DriverBand, type DriverSocket, type DriverStep, type DriverStepTime, type ErrorOnTheWay, type Fault, type FaultPlace, type FaultSide, type Group, type GroupClient, type GroupLab, type GroupMeasured, type GroupNetworkFault, type GroupPlace, type GroupPrediction, type GroupReading, type GroupRole, type GroupVerdict, type GroupView, type Hop, type HopEvent, type HopPlace, type HopSignal, type HopWrite, type Host, type HttpProbe, type ImagePlan, type ImageSource, type Interval, type Jitter, type KernelRecord, type Lab, type Layer, type LineMatch, type Manifest, type ManifestHop, type Measured, type MemberLine, type MiddleboxEvent, type MiddleboxExpectation, type MiddleboxForm, type NetworkFault, type NodeLine, type Observer, type Outcome, type Pairing, type PinnedImage, type Prediction, type Probe, type ProbeEvent, type ProgramEvent, type ProxyHop, type ProxyTime, type ReconnectReading, type ReconnectVerdict, type Reconnects, type Recovers, type Reference, type Reordering, type ReplicaSet, type ReplicaSetLab, type ReplicaSetStage, type Replies, type RequestPrediction, type RequestReading, type RouterHop, type RttRange, type Run, type RunGroup, type RunPart, type RunTopology, type SentinelEvent, type SentinelStage, type Shipped, type ShippedImage, type SocketPlace, type Spread, type Stage, type StageBand, type StageVerdict, type Stamp, type Status, type StatusRow, type Subject, type Substrate, type SubstrateSource, type Target, type Topology, type Toxic, type Verdict, type WallStamp, type Writes, type WsProbe, isHopPlace, latestEndOf };
+export { type Agent, type BuildImage, type BuildRecord, type CallReading, type Cell, type CellResult, type Client, type ClientImage, type ClockPair, type Cloud, type Cluster, type ClusterClientAt, type ClusterLab, type ClusterStage, type ContextFile, type CountBand, type Crossings, type Cut, type Decision, type Drill, type DriverBand, type DriverSocket, type DriverStep, type DriverStepTime, type ErrorOnTheWay, type Fabric, type FabricLab, type FabricManifest, type FabricParts, type Fault, type FaultPlace, type FaultSide, type Group, type GroupClient, type GroupLab, type GroupMeasured, type GroupNetworkFault, type GroupPlace, type GroupPrediction, type GroupReading, type GroupRole, type GroupVerdict, type GroupView, type Hop, type HopEvent, type HopPlace, type HopSignal, type HopWrite, type Host, type HttpProbe, type ImagePlan, type ImageSource, type Interval, type Jitter, type Kafka, type KafkaLab, type KernelRecord, type Lab, type Layer, type LeafViews, type LineCount, type LineMatch, type Manifest, type ManifestHop, type Measured, type MemberLine, type MiddleboxEvent, type MiddleboxExpectation, type MiddleboxForm, type NetworkFault, type NodeLine, type Observer, type OplogRead, type OplogTime, type Outcome, type Outcomes, type Pair, type PairSets, type Pairing, type PartitionView, type Pinnable, type PinnedImage, type PortsPrediction, type PortsReading, type Prediction, type Probe, type ProbeEvent, type ProgramEvent, type ProxyHop, type ProxyTime, type QueueBand, type QueueConnection, type QueueConnectionReading, type QueueEnd, type QueuePrediction, type QueueReading, type ReconnectReading, type ReconnectVerdict, type Reconnects, type RecordCode, type Recovers, type RecoveryMeasured, type Reference, type Reordering, type ReplicaSet, type ReplicaSetLab, type ReplicaSetStage, type Replies, type RequestPrediction, type RequestReading, type RouteLine, type RouterHop, type RttRange, type Run, type RunBackup, type RunDrill, type RunFabric, type RunGroup, type RunPart, type RunRoute, type RunTopology, type SentinelEvent, type SentinelStage, type Shipped, type ShippedImage, type Sibling, type SocketPlace, type Spread, type Stage, type StageBand, type StageVerdict, type Stamp, type Status, type StatusRow, type Subject, type Substrate, type SubstrateSource, type Target, type Topology, type Toxic, type Verdict, type WallStamp, type Writes, type WsProbe, isHopPlace, latestEndOf };
 //# sourceMappingURL=index.d.ts.map