@mark1russell7/harness 0.6.0 → 0.8.0

package/build/index.d.ts

@@ -36,7 +36,7 @@ import "testcontainers";
  * one side. It is a black hole, a reset or an ICMP unreachable. It can also be a simplex drop, an
  * MTU drop or netem.
  *
- * These faults are of a replicated group (`Cell.group`), on the part that `Cell.faultAt` names:
+ * These faults are of a replicated group (`Cell.group` or `Cell.replicaSet`), on the part that `Cell.faultAt` names:
  * - `blackhole` drops each segment between the part and each of its peers. `rst` answers each
  *   segment from a peer with a reset, thus the part refuses each connection.
  * - `partition` is a black hole between the part and some of its peers only (`GroupPlace.cuts`).
@@ -47,8 +47,11 @@ import "testcontainers";
  * This fault is of an HTTP server:
  * - `app-stall` injects nothing. The server process holds the reply to the request, because the
  *   request asks for that (`HttpProbe.hold`), while its kernel stays up.
+ *
+ * This fault is of a replica set (`Cell.replicaSet`):
+ * - `step-down` sends `replSetStepDown` to the primary, as a client sends a command.
  */
-type Fault = "blackhole" | "freeze-process" | "rst" | "stall" | "fin" | "unreachable" | "simplex" | "mtu-blackhole" | "netem" | "middlebox-forget" | "proxy-idle" | "toxic" | "server-close" | "partition" | "planned-failover" | "app-stall" | "none";
+type Fault = "blackhole" | "freeze-process" | "rst" | "stall" | "fin" | "unreachable" | "simplex" | "mtu-blackhole" | "netem" | "middlebox-forget" | "proxy-idle" | "toxic" | "server-close" | "partition" | "planned-failover" | "app-stall" | "step-down" | "none";
 /**
  * A middlebox form tells what a router does with a flow that it forgot, when a late packet comes:
  * - `reset-both` sends a reset to the two ends when it forgets the flow.
@@ -158,6 +161,32 @@ interface Group {
     readonly sentinel: string;
   };
 }
+/**
+ * A replica set is a replicated group with no external authority, as data: its members elect their
+ * own primary. Each member is a container from the same stock image, pinned by digest, in its own
+ * network namespace. The client finds the primary by its own monitors. The harness starts the set
+ * with the address of each member, and it waits for a primary.
+ */
+interface ReplicaSet {
+  /** This name is the name of the set, for example `rs0`. */
+  readonly name: string;
+  /** This image is the stock image of each member, pinned by digest. */
+  readonly image: string;
+  /** These are the programs of the image: the server and its shell. */
+  readonly programs: {
+    readonly server: string;
+    readonly shell: string;
+  };
+  /** These are the options of the server of each member, after the name of the set: for example the size of its cache. */
+  readonly args?: readonly string[];
+  readonly port: number;
+  /** These are the names of the members. The first one starts the set. */
+  readonly members: readonly string[];
+  /** These are the settings of the configuration of the set, as `rs.initiate` takes them: for example `electionTimeoutMillis`. */
+  readonly settings: Readonly<Record<string, number>>;
+  /** This line tells that a member serves. */
+  readonly ready: string;
+}
 /** A group role is a part of a group by its role at the start of a run: the primary, a replica, or a Sentinel. */
 type GroupRole = "primary" | "replica" | "sentinel";
 /**
@@ -166,10 +195,12 @@ type GroupRole = "primary" | "replica" | "sentinel";
  * - `all`: each other part of the group, and each client. This is the default.
  * - `group`: the other members and the Sentinels, but not the clients.
  * - `clients`: the clients, but not the group.
+ * - `one`: one other member, in both directions.
+ * - `one-way`: one other member, for the segments from the part only.
  */
 interface GroupPlace {
   readonly role: GroupRole;
-  readonly cuts?: "all" | "group" | "clients";
+  readonly cuts?: "all" | "group" | "clients" | "one" | "one-way";
 }
 /** A toxic is one toxic of Toxiproxy, as its API takes it. Its toxicity is 1. */
 interface Toxic {
@@ -185,8 +216,15 @@ interface Toxic {
  * send of the client. A FIN counts from the change of state of the socket of the peer when its
  * process dies (`peer-closed`). A toxic of Toxiproxy counts from the start of its commit, as the
  * injector stamps it (`fault`).
+ *
+ * A client that starts into a fault counts from its `connecting` line, or from its first line
+ * without one (`client-start`).
+ *
+ * A heartbeat of a program counts from its first ping that got no answer (`ping-sent`). The
+ * program writes a `ping` line before each ping, and the kernel stamps the write (`tx`). No
+ * segment with data reaches the socket after that write, up to the detection.
  */
-type Reference = "last-received" | "first-retransmission" | "request-sent" | "peer-closed" | "fault";
+type Reference = "last-received" | "first-retransmission" | "request-sent" | "peer-closed" | "fault" | "client-start" | "ping-sent";
 /**
  * This prediction tells when a connection moves again after a stall. That occurs at the first
  * firing of its retransmission timer after the stall stops, and its reply comes back one round
@@ -245,8 +283,18 @@ interface ProxyTime {
  * failover and the election. Then the selection of a replica, the promotion, the switch to the new
  * primary, and the conversion of the old primary into a replica.
  */
-type Stage = "+sdown" | "+odown" | "+try-failover" | "+elected-leader" | "+selected-slave" | "+promoted-slave" | "+switch-master" | "+convert-to-slave";
-/** A stage band is when the first Sentinel announces a stage, from the reference of the cell. */
+type SentinelStage = "+sdown" | "+odown" | "+try-failover" | "+elected-leader" | "+selected-slave" | "+promoted-slave" | "+switch-master" | "+convert-to-slave";
+/**
+ * A replica set stage is one step of the timeline of a replica set after a fault. A line of the log
+ * of a member marks it, or a line of the client. The stages are a member that marks the primary
+ * DOWN, the start of an election and its win, and the end of the catch-up. Then come the first
+ * write of the new primary and the step-down of the old one. The client's lines give its discovery
+ * of the new primary, the write-off of the old one, and the first write that a primary accepts.
+ */
+type ReplicaSetStage = "member-down" | "election-start" | "elected" | "caught-up" | "writable" | "old-primary-steps-down" | "discovered" | "written-off" | "first-write";
+/** A stage is one step of the timeline of a group: of a group under Sentinels, or of a replica set. */
+type Stage = SentinelStage | ReplicaSetStage;
+/** A stage band is when the first part announces a stage, from the reference of the cell. */
 interface StageBand {
   readonly stage: Stage;
   readonly low: Quantity;
@@ -260,6 +308,7 @@ interface StageBand {
  *   stamps of the observer (`stamps`).
  * - `errors` are the errors that the client must meet on the way, each with the band of its
  *   first occurrence. An example is the error of the offline queue when it gives up.
+ * - `quiet` are the stages that must not occur after the fault. Each one that a run shows is a finding.
  */
 interface GroupPrediction {
   readonly lands: "new" | "same";
@@ -271,6 +320,7 @@ interface GroupPrediction {
     readonly low: Quantity;
     readonly high: Quantity;
   }>;
+  readonly quiet?: readonly Stage[];
 }
 /**
  * A middlebox expectation tells what the router of a topology does with the flow in the window of
@@ -298,6 +348,71 @@ interface RequestPrediction {
   readonly error: string | null;
   readonly pending?: true;
 }
+/**
+ * A line match is a line of the client that a driver band waits for. It has the `ev` of the line,
+ * and the value of each field that it names. A dotted name is a field in a field, for example
+ * `failure.name`. `name` is the field that names the error of the line, for its outcome.
+ */
+interface LineMatch {
+  readonly ev: string;
+  readonly where?: Readonly<Record<string, string | number | boolean>>;
+  readonly name?: string;
+}
+/**
+ * A driver step is one more band of a driver after the band of its cell. An example is the error
+ * of the retry after the interruption. It has its own reference on a socket of the driver, its
+ * line, and its band. A run whose step is outside its band, or absent, has a finding.
+ */
+interface DriverStep {
+  readonly what: string;
+  /** This name is the name of the error of the line, or OK. */
+  readonly name: string;
+  /** This role is the socket of the reference. Without it, the socket is the socket of the band of the cell. */
+  readonly socket?: string;
+  readonly from: Reference;
+  readonly ends: LineMatch;
+  readonly low: Quantity;
+  readonly high: Quantity;
+}
+/**
+ * A driver band is a band of a cell whose client is a driver with more than one socket, for
+ * example the MongoDB driver. It names the socket of the reference, and the line that ends the
+ * band:
+ * - `socket` is the role of the socket, as the client names it in its `socket` lines. The socket
+ *   `server` is the socket of the target for the first connection of the pool.
+ * - `ends` is the line of the client that marks the detection. Without it, the detection is the
+ *   report of the kernel on the socket: an error, or the end of the file.
+ * - `reply` is the line of a reply that a silent prediction times (`replies`), with its reference.
+ * - `quiet` are the lines that must not occur after the fault, in the window of the prediction.
+ * - `then` are the later bands, each one a test of its own.
+ *
+ * A band whose line ends it is a timer of the driver. Its `last-received` is the last segment with
+ * data that its socket received or sent. The idle timer of a Node socket starts again at each read
+ * and each write. A band of the kernel counts from the last segment of any kind that the socket
+ * received.
+ */
+interface DriverBand {
+  readonly socket: string;
+  readonly ends?: LineMatch;
+  readonly reply?: {
+    readonly from: Reference;
+    readonly ends: LineMatch;
+  };
+  readonly quiet?: readonly LineMatch[];
+  readonly then?: readonly DriverStep[];
+}
+/**
+ * A reconnect prediction tells when the program of the client tries a new connection after the
+ * detection. Each wait counts from the start of the reconnect timer of the program (a
+ * `reconnecting` line) to the try (a `dial` line). The first wait follows the detection, and each
+ * next wait follows the failure of the try before it.
+ */
+interface Reconnects {
+  readonly waits: ReadonlyArray<{
+    readonly low: Quantity;
+    readonly high: Quantity;
+  }>;
+}
 /**
  * A prediction is what a cell predicts. Either a detection occurs in an interval [T_lo, T_hi]
  * after the reference event, or nothing detects the fault in a window. Nothing detects a stall
@@ -318,6 +433,12 @@ interface RequestPrediction {
  *
  * For an HTTP cell, `request` tells what the second request meets. The errno `CLOSED` is a close
  * of the observer itself: its pool closes an idle connection, a FIN that it sends.
+ *
+ * A band of a driver (`driver`) counts from a socket of the driver to a line of the client.
+ *
+ * The errno `DROPPED` is a drop by the program of the observer itself. Its own timer gave up, and
+ * it closed its socket or wrote a close frame. A detection can also tell when the client tries a
+ * new connection (`reconnects`).
  */
 type Prediction = {
   readonly detects: true;
@@ -325,7 +446,8 @@ type Prediction = {
   readonly timer: string;
   /**
    * This name is the errno of the detection, or EOF for an end of file without an error. It is
-   * OK for a write that a primary accepts, and CLOSED for a close of the observer itself.
+   * OK for a write that a primary accepts, and CLOSED for a close of the observer itself. It is
+   * DROPPED for a drop by the program of the observer.
    */
   readonly errno: string;
   readonly from: Reference;
@@ -337,6 +459,14 @@ type Prediction = {
   readonly through?: readonly ProxyTime[];
   readonly group?: GroupPrediction;
   readonly request?: RequestPrediction;
+  readonly driver?: DriverBand;
+  readonly reconnects?: Reconnects;
+  /**
+   * This name tells how the program drops the connection, for the errno DROPPED. A close frame
+   * leaves its socket ESTABLISHED below (`frame`). A close of its socket takes it out of
+   * ESTABLISHED at once (`close`).
+   */
+  readonly drop?: "frame" | "close";
 } | {
   readonly detects: false;
   readonly window: Quantity;
@@ -347,6 +477,7 @@ type Prediction = {
   readonly through?: readonly ProxyTime[];
   readonly group?: GroupPrediction;
   readonly request?: RequestPrediction;
+  readonly driver?: DriverBand;
 } | {
   readonly detects: null;
   readonly window: Quantity;
@@ -390,13 +521,70 @@ interface Target {
    * output, on CLOCK_MONOTONIC. Each run then keeps its lines (`Run.server`).
    */
   readonly lines?: true;
+  /**
+   * This name is the server process, as /proc/PID/comm holds it. With it, a freeze of the target
+   * stops this process with SIGSTOP from a sidecar in its process namespace, and the injector
+   * stamps it. SIGCONT then thaws it. Without it, the freeze is the cgroup freezer, with no stamps.
+   */
+  readonly program?: string;
+  /**
+   * This name is an image of the harness that the target operates in place of `image`. An
+   * example is the image of the procedures client, which also holds the script of its server.
+   * `image` is then the stock base of that image, for the record.
+   */
+  readonly harness?: HarnessImage;
 }
+/**
+ * A harness image is an image of the harness that a target can operate. `procedures` is the
+ * image of `@procedures/client` on Node: its WebSocket client and its server script.
+ */
+type HarnessImage = "procedures";
 /**
  * A client is what the runs of a cell use: the TCP client of the harness, or a driver on its
  * socket. The HTTP clients are the `http.Transport` of Go (`go-http`), the `http.Agent` of Node
- * (`http-agent`) and an HTTP/1.1 `Client` of undici (`undici`).
+ * (`http-agent`) and an HTTP/1.1 `Client` of undici (`undici`). The MongoDB driver is `mongodb`, and the WebSocket client of
+ * `@procedures/client` is `procedures-ws`.
  */
-type Client = "tcp" | "ioredis" | "go-http" | "http-agent" | "undici";
+type Client = "tcp" | "ioredis" | "go-http" | "http-agent" | "undici" | "mongodb" | "procedures-ws";
+/**
+ * A WebSocket probe is the workload and the settings of the WebSocket client of
+ * `@procedures/client` (`procedures-ws`). The cell gives each setting, thus the client takes
+ * nothing from a default.
+ */
+interface WsProbe {
+  /**
+   * This name is the class of the sockets of the client: the global `WebSocket` of Node (`global`),
+   * or the `WebSocket` of the `ws` package (`ws`, the option `webSocket`).
+   */
+  readonly socket: "global" | "ws";
+  /**
+   * These are the settings of the heartbeat: a ping each interval, the wait for its pong, and the
+   * misses in sequence that drop the connection. Null turns the heartbeat off.
+   */
+  readonly heartbeat: {
+    readonly interval: Quantity;
+    readonly timeout: Quantity;
+    readonly misses: number;
+  } | null;
+  /**
+   * These are the settings of the reconnect: the first delay, the multiplier of each next delay,
+   * the longest delay and the most tries. `jitter` is the random part of each delay, from 0 to 1.
+   * Null turns the reconnect off.
+   */
+  readonly reconnect: {
+    readonly initialDelay: Quantity;
+    readonly multiplier: number;
+    readonly maxDelay: Quantity;
+    readonly attempts: number;
+    readonly jitter: number;
+  } | null;
+  /** This time is how long a call waits for its first frame. */
+  readonly requestTimeout: Quantity;
+  /** This time is how long a call waits for a connection. */
+  readonly connectionTimeout: Quantity;
+  /** This time is how long the server holds the reply to the call in flight. Without it, the server replies at once. */
+  readonly hold?: Quantity;
+}
 /**
  * An HTTP probe is the workload of an HTTP client. The first request is a GET of the root. The
  * second request follows the mode of the probe: after the signal (`inflight`), or after a gap
@@ -466,16 +654,24 @@ interface Cell {
      */
     readonly request?: string;
     /**
-     * This driver replaces the TCP client of the harness, with its own options. It is ioredis 6 on
-     * Node, whose keepAlive is the keepalive delay of the socket. The driver then ignores `keepalive`
-     * and `userTimeout`.
+     * This driver replaces the TCP client of the harness, with its own options. The driver then
+     * ignores `keepalive` and `userTimeout`:
+     * - ioredis 6 on Node, whose keepAlive is the keepalive delay of the socket.
+     * - the MongoDB driver on Node, against the replica set `set`. Its operation is a write, or
+     *   the read of a long document (`read-long`). `Cell.driverOptions` are its options.
      */
     readonly driver?: {
       readonly name: "ioredis";
       readonly keepAlive: Quantity;
+    } | {
+      readonly name: "mongodb";
+      readonly set: string;
+      readonly op?: "write" | "read-long";
     };
     /** This workload is the workload of an HTTP client (`Cell.client` is an HTTP client). The client then ignores `keepalive` and `userTimeout`. */
     readonly http?: HttpProbe;
+    /** These are the settings of the WebSocket client (`Cell.client` is `procedures-ws`). The client then ignores `keepalive` and `userTimeout`. */
+    readonly ws?: WsProbe;
   };
   readonly fault: Fault;
   /** This time is the length of a stall, from the send of the request. */
@@ -530,12 +726,28 @@ interface Cell {
    * record, and the group does not read it.
    */
   readonly group?: Group;
-  /** This time is how long a fault on a group holds after its commit. The injector then lifts it at that instant, on the clock of the VM. */
+  /**
+   * This replica set replaces the target: a replicated group whose members elect their own
+   * primary. The client is then the MongoDB driver, and it writes a stream of sequence numbers.
+   * `target` stays for the record, and the set does not read it.
+   */
+  readonly replicaSet?: ReplicaSet;
+  /**
+   * This time is how long a fault on a group holds after its commit. The injector then lifts it at
+   * that instant, on the clock of the VM. On a direct pair, a frozen target thaws after it.
+   */
   readonly hold?: Quantity;
   /** This flag makes a planned failover coordinated: `SENTINEL FAILOVER <group> COORDINATED`. */
   readonly coordinated?: boolean;
-  /** These are options of ioredis over its defaults, for the client of a group. A cell at the defaults has none. */
+  /**
+   * These are options of the driver over its defaults: of ioredis for the client of a group, or of
+   * the MongoDB driver for its client. A cell at the defaults has none.
+   */
   readonly driverOptions?: Readonly<Record<string, unknown>>;
+  /** This write concern is the w of each write of the MongoDB client. Without it, it is majority. */
+  readonly writeConcern?: "majority" | 1;
+  /** This flag makes the fault hold before the client starts. The client then meets it at its first selection. */
+  readonly before?: boolean;
   /** This count is the number of runs of the cell (11), across new environments (two at least). */
   readonly runs: number;
   readonly environments: number;
@@ -639,7 +851,11 @@ interface RunGroup {
   readonly primary: string;
   /** This name is the part that got the fault, or null for a fault on the whole group. */
   readonly at: string | null;
-  /** These are the lines of the observer of the Sentinels from the start to the end of the run. */
+  /**
+   * These are the lines of the timeline from the start to the end of the run. For a group under
+   * Sentinels, they are the lines of the observer of the Sentinels. For a replica set, they are
+   * the log lines of the members (`ev` "mongod"), with their times on the monotonic clock.
+   */
   readonly timeline: readonly SentinelEvent[];
   /** This name is the member that was the primary when the group converged after the run, or null when it did not converge. */
   readonly after: string | null;
@@ -662,14 +878,147 @@ interface RunTopology {
   }>>;
 }
 //#endregion
+//#region src/application.d.ts
+/**
+ * A program event is a line of the program of the observer, with the kernel's stamp of its act.
+ * The act is a write, or a change of state of the socket. Without a record of the kernel in δ_abs
+ * after the line, the instant is the stamp of the line.
+ */
+interface ProgramEvent {
+  readonly line: ProbeEvent;
+  readonly at: number;
+  /** This flag tells that the kernel stamped the event. */
+  readonly kernel: boolean;
+  /** This name is the kind of the record of the kernel: a write, or a change of state. It is null without a record. */
+  readonly record: "write" | "state" | null;
+}
+/** This function gives the lines of the program of the observer: the client, or the target for the server. A proxy has no lines. */
+export declare const programLines: (cell: Cell, run: Run) => readonly ProbeEvent[];
+/**
+ * This function gives the time from a line of the program to the close of the socket by its own
+ * side. That close is its change from ESTABLISHED to FIN_WAIT1, or to CLOSE (an abort). A FIN of
+ * the peer (CLOSE_WAIT) is not such a close. It is null when the socket did not close itself.
+ */
+export declare const leftAfter: (socket: readonly KernelRecord[], line: ProbeEvent) => Quantity | null;
+/**
+ * This function tells how a drop differs from the way that its prediction gives, or gives null.
+ * A close closes the socket in δ_abs. A close frame is a write, and the program does not close
+ * the socket at the drop. A peer that answers the close frame can close it, and that is no finding.
+ */
+export declare const dropFinding: (expected: "frame" | "close", drop: ProgramEvent, left: Quantity | null) => string | null;
+/** This function gives each ping of the program on the connection of the run, with the kernel's stamp of its write. */
+export declare const pingsOf: (lines: readonly ProbeEvent[], socket: readonly KernelRecord[], local: unknown) => ProgramEvent[];
+/**
+ * This function gives the first ping that got no answer: no segment with data reached the socket
+ * after its write, up to `until`. Its write starts the wait of the heartbeat that the fault met.
+ */
+export declare const firstUnanswered: (pings: readonly ProgramEvent[], socket: readonly KernelRecord[], until?: number) => ProgramEvent | undefined;
+/**
+ * This function gives the first drop of the connection by its program. The kernel stamps it as the
+ * write of a close frame. Or it stamps the change of the socket out of ESTABLISHED: a close or an
+ * abort.
+ */
+export declare const dropOf: (lines: readonly ProbeEvent[], socket: readonly KernelRecord[], local: unknown) => ProgramEvent | undefined;
+/** A reconnect reading is what the client did after the detection, from its lines and from the records of the kernel. */
+interface ReconnectReading {
+  /** These are the waits of the program before each try: from a `reconnecting` line to the next `dial` line. */
+  readonly waits: readonly Quantity[];
+  /** This time is from the detection to the first SYN of a new socket. The kernel stamps the change to SYN_SENT at connect(2). */
+  readonly syn: Quantity | null;
+  /** This time is from the detection to the first new socket of the client that became ESTABLISHED. */
+  readonly established: Quantity | null;
+  /** These are the times from each `dial` line after the detection to the SYN of its socket: the work of the program before connect(2). */
+  readonly connects: readonly Quantity[];
+}
+/**
+ * This function reads the reconnect of the client after the detection at `detected`. The socket of
+ * the run has the cookie `cookie`. The lines of the program count from `from`: the drop line of a
+ * drop, which comes before the kernel's stamp of it.
+ */
+export declare const reconnectOf: (run: Run, detected: number, cookie: number | undefined, from?: number) => ReconnectReading;
+/** This function tells how the waits of a reconnect differ from their bands, each widened by δ_abs, or gives null. */
+export declare const reconnectFinding: (expected: Reconnects, read: ReconnectReading) => string | null;
+/**
+ * A call reading is what the call in flight of a WebSocket client met: its error, a reply, or
+ * nothing in the run. The time counts from the detection to its end.
+ */
+interface CallReading {
+  readonly outcome: string;
+  readonly after: Quantity | null;
+}
+/** This function reads the call that the client sent into the fault: its `call-error` or `data` line after the request. */
+export declare const callOf: (events: readonly ProbeEvent[], request: ProbeEvent, detected: number | undefined) => CallReading;
+//#endregion
+//#region src/driver.d.ts
+/**
+ * This module measures a run of a cell whose client is a driver with more than one socket, for
+ * example the MongoDB driver. Its monitor, its pool and its round-trip pinger each have their own
+ * socket, and the client names each socket by its role. A band counts from an event of one socket,
+ * as the kernel records it. It ends at a line of the client, or at the report of the kernel on
+ * that socket.
+ */
+/** A driver socket is one socket of the driver in a run: its role, its local address, its server, and when it opened and ended. */
+interface DriverSocket {
+  readonly role: string;
+  readonly local: string;
+  readonly server: string;
+  readonly opened: number;
+  readonly ended: number | null;
+}
+/** A driver step time is the time of one later band of a run. It is null when its line did not come. */
+interface DriverStepTime {
+  readonly what: string;
+  readonly name: string | null;
+  readonly elapsed: Quantity | null;
+}
+/** This function gives the value of a field of a line by its name. A dotted name is a field in a field. */
+export declare const fieldOf: (line: unknown, path: string) => unknown;
+/** This function tells if a line of the client matches a line match: its `ev`, and each field that the match names. */
+export declare const matches: (line: ProbeEvent, match: LineMatch) => boolean;
+/** This function gives the sockets of the driver in a run, from the `socket` and `socket-end` lines of the client. */
+export declare const socketsOf: (events: readonly ProbeEvent[]) => DriverSocket[];
+/**
+ * This function finds the socket of a role at an instant. The pool's socket is the first one, which
+ * the client names in its `connected` line. Each other role has the latest socket that opened at
+ * the instant or before it.
+ */
+export declare const socketAt: (events: readonly ProbeEvent[], role: string, at: number) => DriverSocket | undefined;
+/**
+ * This function gives the instant of a reference in a run, on a socket of the driver:
+ * - `last-received`: the last segment that the socket received before the fault held. A band of
+ *   the kernel counts from the last segment of any kind. A band that a line ends is a timer of
+ *   the driver. It counts from the last segment with data that the socket received or sent.
+ * - `request-sent`: the send of the operation into the fault, the second `sent` line.
+ * - `first-retransmission`: the first retransmission of the RTO timer on the socket.
+ * - `peer-closed`: the change of state of the socket of the target, when its process dies.
+ * - `fault`: the start of the commit of the fault. `client-start`: the `connecting` line.
+ */
+export declare const referenceOf: (run: Run, from: Reference, role: string, data: boolean) => number | undefined;
+/**
+ * This function reads one run of a driver cell.
+ * - The band of the cell counts from its reference on its socket. It ends at its line, or at the
+ *   report of the kernel on that socket.
+ * - Each later band (`then`) gets the same test. A later band outside its band, or absent, is a finding.
+ * - A line that the prediction names as quiet, after the fault and in its window, is a finding.
+ *   So is a flow of an open socket of the driver that a router forgot, for a silent prediction.
+ * - A silent prediction can time a reply (`reply`), and the verdict tests it against `replies`.
+ *
+ * A run is void when the prober lost records, or when the client did not get its first reply. It
+ * is also void when the operation went out before the fault held. A late lift of a hold and a
+ * keepalive probe before the fault also make it void. So do a delayed timer and a round trip
+ * above the assumption.
+ */
+export declare const measureDriver: (cell: Cell, run: Run) => Measured;
+//#endregion
 //#region src/timeline.d.ts
 /**
  * This module measures a run of a group cell. It gives the first write that a primary accepted
- * after the fault, and the timeline of the group as its Sentinels announced it. It also gives what
+ * after the fault, and the timeline of the group. The Sentinels of a group announce its timeline.
+ * The logs of the members of a replica set and its client tell it. The module also gives what
  * became of each write.
  */
-/** These are the stages of a timeline, in their sequence after a fault. */
-export declare const STAGES: readonly Stage[];
+/** These are the stages of a timeline under Sentinels, in their sequence after a fault. */
+export declare const STAGES: readonly SentinelStage[];
 /** An error on the way is one kind of error that the client met after the fault. It has its first time from the fault, and its count. */
 interface ErrorOnTheWay {
   readonly name: string;
@@ -708,6 +1057,30 @@ interface GroupMeasured {
 }
 /** This function gives the time of each stage of a timeline from an instant: the first announcement of each, after the instant. */
 export declare const stagesOf: (timeline: readonly SentinelEvent[], from: number) => Partial<Record<Stage, Quantity>>;
+/** These are the stages of the timeline of a replica set, in their usual sequence after a fault. */
+export declare const REPLICA_SET_STAGES: readonly ReplicaSetStage[];
+/**
+ * A member mark is how a log line of mongod marks a stage: by its id, or by its message. The
+ * message is the fallback for a version whose ids differ. `at` tells where the line comes from:
+ * the old primary, or another member.
+ */
+interface MemberMark {
+  readonly ids: readonly number[];
+  readonly message: RegExp;
+  readonly at: "old" | "other";
+}
+/** These are the log lines of mongod that mark each stage of a member. */
+export declare const MEMBER_MARKS: Readonly<Record<Exclude<ReplicaSetStage, "discovered" | "written-off" | "first-write">, MemberMark>>;
+/**
+ * This function gives the time of each stage of a replica set from an instant, in a run. The stages
+ * of the members come from their log lines (`ev` "mongod"), whose times the lab puts on the
+ * monotonic clock. The stages of the client come from its lines:
+ * - `discovered`: a member other than the old primary becomes RSPrimary in the view of the client.
+ * - `written-off`: the first heartbeat of the client to the old primary that fails.
+ * - `first-write`: the first write that a primary accepts, of a write that the client sent after
+ *   the commit of the fault (`sentAfter`). It is on a new primary when `lands` is `new`.
+ */
+export declare const replicaSetStagesOf: (run: Run, from: number, sentAfter: number, lands: "new" | "same") => Partial<Record<Stage, Quantity>>;
 /** This function gives what became of each write of a run, from the lines of the client and the keys that the group kept. */
 export declare const writesOf: (run: Run, after: number, primary: string) => Writes;
 /** This function gives the errors that the client met after an instant: each kind, its first time, and its count. */
@@ -737,7 +1110,10 @@ interface GroupReading {
  * - The reference is the start of the commit of the fault, as the injector stamped it. For a
  *   stall, it is the first write into the fault, as for the stall of a direct pair.
  * - The outcome is the first write that a primary accepted after the fault. The prediction says
- *   which primary: a new one, or the same one after the lift.
+ *   which primary: a new one, or the same one after the lift. In a replica set, it is the first
+ *   accepted write of a write that the client sent after the commit of the fault.
+ * - The stages come from the Sentinels, or from the logs of the members of a replica set and the
+ *   lines of its client (`replicaSetStagesOf`).
  * - Its time is the last segment with data that the client's socket to that primary received
  *   before the client saw the acceptance. The stamp of the client is the fallback.
  * - A run is void when the prober lost records, or when the client did not become ready. It is
@@ -750,7 +1126,8 @@ export declare const measureGroup: (cell: Cell, run: Run) => GroupReading;
 //#region src/observe.d.ts
 /**
  * An outcome tells how the wait of a run ended. The socket failed with an errno, or it read the
- * end of the file. Or nothing came, or the reply came back after a stall.
+ * end of the file. Or nothing came, or the reply came back after a stall. The observer can also
+ * close the connection itself (`closed`), or its program can drop it on its own timer (`dropped`).
  */
 type Outcome = {
   readonly kind: "error";
@@ -763,6 +1140,8 @@ type Outcome = {
   readonly kind: "answered";
 } | {
   readonly kind: "closed";
+} | {
+  readonly kind: "dropped";
 };
 /**
  * A request reading is what the second request of an HTTP client met, from the lines of the
@@ -826,6 +1205,20 @@ interface Measured {
   readonly group?: GroupMeasured;
   /** This record is what the second request of an HTTP client met, for a cell whose prediction says it. */
   readonly request?: RequestReading | null;
+  /** This record is the time of each later band of a driver cell (`DriverBand.then`). */
+  readonly driver?: {
+    readonly steps: readonly DriverStepTime[];
+  };
+  /** This record is what the client did after the detection, for a prediction that says when it reconnects. */
+  readonly reconnect?: ReconnectReading;
+  /** This record is what the call in flight of a WebSocket client met. */
+  readonly call?: CallReading;
+  /**
+   * This time is from the drop line of the program to the close of its socket by its own side: a
+   * FIN or an abort. A prediction that says how the program drops (`drop`) gets it. It is null
+   * when the socket did not close itself in the run.
+   */
+  readonly left?: Quantity | null;
   /** This text is the reason that the run cannot count, or null. A void run is not a pass and not a finding. */
   readonly void: string | null;
   /** This text tells what in the run disagrees with the prediction, other than its timing, or null. */
@@ -981,8 +1374,33 @@ interface Verdict {
    * one is a finding of its run.
    */
   readonly requests?: Readonly<Record<string, number>>;
+  /** This record is what the client did after the detection over the counted runs, for a prediction that says when it reconnects. */
+  readonly reconnect?: ReconnectVerdict;
+  /** These are the outcomes of the call in flight of a WebSocket client over the counted runs, each with its count. */
+  readonly calls?: Readonly<Record<string, number>>;
+  /**
+   * This record is the time from a drop of the program to the close of its socket by its own side,
+   * over the counted runs. It also counts the runs where the socket did not close itself.
+   */
+  readonly left?: {
+    readonly spread: Spread | null;
+    readonly stayed: number;
+  };
   readonly why: string;
 }
+/**
+ * A reconnect verdict sums the reconnect of the client over the counted runs. Each wait before a
+ * try gets its spread. A wait outside its band is a finding of its run, thus it is in `outside`.
+ * - `syn` is the time from the detection to the first SYN of a new socket.
+ * - `established` is the time from the detection to the first new socket that became ESTABLISHED.
+ * - `connect` is the time from a `dial` line to the SYN of its socket: the work of the program.
+ */
+interface ReconnectVerdict {
+  readonly waits: ReadonlyArray<Spread | null>;
+  readonly syn: Spread | null;
+  readonly established: Spread | null;
+  readonly connect: Spread | null;
+}
 /**
  * A stage verdict is the test of one stage of the timeline of a group against its band. The band
  * is [T_lo − δ_abs, T_hi + the error of the stamps + δ_abs]. A run that did not announce the stage
@@ -1069,6 +1487,12 @@ export declare const routerImage: () => Promise<string>;
 export declare const goHttpImage: () => Promise<string>;
 /** This function gives the image of the Node HTTP clients. */
 export declare const nodeHttpImage: () => Promise<string>;
+/** This function gives the image of the MongoDB client. */
+export declare const mongoClientImage: () => Promise<string>;
+/** This function gives the image of `@procedures/client`: its WebSocket client and its server. */
+export declare const proceduresImage: () => Promise<string>;
+/** This function gives the image that a target operates: its stock image, or an image of the harness (`Target.harness`). */
+export declare const targetImageOf: (target: Target) => Promise<string>;
 /**
  * This time is the interval of the listing of a router, when its hop does not set one. A listing
  * evicts each expired flow that it walks past, and the DESTROY event of that flow follows.
@@ -1126,6 +1550,9 @@ interface Manifest {
   /** This record is the target, and its settings as it read them back. */
   readonly target: {
     readonly image: string;
+    /** This name is the image of the harness that the target operates, with its stock base (`Target.harness`). */
+    readonly harness?: string;
+    readonly base?: string;
     readonly id: string;
     readonly settings: Readonly<Record<string, string>>;
     /** These are the sysctls of its namespace, as read back after the harness set them. */
@@ -1304,6 +1731,15 @@ interface Lab {
   /** This method freezes the target with the cgroup freezer: its process stops, and its kernel continues to answer. */
   freeze(): Promise<void>;
   thaw(): Promise<void>;
+  /**
+   * This method sends a signal to the server process of the target (`Target.program`), at once or
+   * at an instant of the VM. A sidecar in the network and process namespaces of the target sends
+   * it, and the injector stamps it. The lab starts that sidecar at the first signal.
+   */
+  signal(signal: "STOP" | "CONT" | "KILL", at?: number): Promise<{
+    t0: number;
+    t1: number;
+  }>;
   probe(args: readonly string[]): Promise<Probe>;
   /**
    * This method waits for the next clock line of the prober, which gives the count of lost
@@ -1491,6 +1927,157 @@ export declare const subscriptionsOf: (timeline: readonly SentinelEvent[]) => Se
  */
 export declare const groupOnce: (lab: GroupLab, cell: Cell, environment: number, index: number) => Promise<Run>;
 //#endregion
+//#region src/replset.d.ts
+/**
+ * This module is the lab of a replica set: a replicated group with no external authority, whose
+ * members elect their own primary. It opens one environment for a replica set cell, for the runs
+ * of the cell. Each member is a container from the stock image of the set, in its own network
+ * namespace. An injector sidecar shares its network and process namespaces.
+ *
+ * The client of each run is the MongoDB driver, which finds the primary with its own monitors. The
+ * timeline comes from the log of each member and from the lines of the client. The prober watches
+ * the namespace of the client.
+ */
+/**
+ * A member line is one log line of a member, as mongod writes it in JSON. `real` is its time on
+ * the wall clock in ms, and `t` is its time on the monotonic clock. The run gives `t` from the
+ * clock lines of its client.
+ */
+interface MemberLine extends SentinelEvent {
+  readonly ev: "mongod";
+  readonly member: string;
+  readonly real: number;
+  readonly id: number;
+  readonly c: string;
+  readonly msg: string;
+  readonly attr: Readonly<Record<string, unknown>>;
+}
+/** These are the components of the log of mongod that a run keeps: replication, its heartbeats, the elections, the rollbacks and the commands. */
+export declare const KEPT: ReadonlySet<string>;
+/** This function reads one log line of a member, or gives null for a line of another component or a line that is not JSON. */
+export declare const memberLineOf: (member: string, text: string) => MemberLine | null;
+/**
+ * This function gives the offset of the wall clock from the monotonic clock, in ns, from the clock
+ * lines of a client. Each line pairs its stamp with the wall clock in whole ms, which truncates.
+ * Thus each pair is the offset or less, and the largest pair is the nearest. It is null without a clock line.
+ */
+export declare const clockOffsetOf: (events: readonly ProbeEvent[]) => number | null;
+/** This function puts the log lines of the members on the monotonic clock, with the offset of a client. */
+export declare const onMonotonic: (log: readonly MemberLine[], offset: number | null) => MemberLine[];
+/** A cut is the peers that a network fault on a part cuts it from, by their addresses, and the direction of the rules. */
+interface Cut {
+  readonly peers: readonly string[];
+  readonly direction?: "out";
+}
+/**
+ * This function gives the peers that a fault on a member of a replica set cuts it from. `all` is
+ * each other member and the client, `group` is the other members, and `clients` is the client.
+ * `one` is the first other member in the sequence of the set, and `one-way` is the same member for
+ * the segments from the part only.
+ */
+export declare const cutOf: (set: ReplicaSet, members: Readonly<Record<string, RunPart>>, client: string, part: string, cuts: NonNullable<GroupPlace["cuts"]>) => Cut;
+/** This function gives the command of the server of each member. The server is the child of a shell, thus a fin can stop it and the container stays up. */
+export declare const memberCommand: (set: ReplicaSet) => string;
+/** This function gives the script of the shell that starts the set: each member by its address, and the settings of the set. */
+export declare const initiateScript: (set: ReplicaSet, ips: Readonly<Record<string, string>>) => string;
+/** This script prints the view of a member: itself, if it accepts writes, and the state of each member as it sees them. */
+export declare const STATUS_SCRIPT: string;
+/** A status is the view of one member, as `STATUS_SCRIPT` prints it. */
+interface Status {
+  readonly me: string;
+  readonly writable: boolean;
+  readonly members: ReadonlyArray<{
+    readonly name: string;
+    readonly state: string;
+    readonly health: number;
+    readonly optime: number | null;
+  }>;
+}
+/**
+ * This function judges the view of a member: the primary of the set, and if the set is healthy.
+ * The set is healthy when the member is the one primary and accepts writes. Each other member must
+ * be a healthy secondary at the optime of the primary.
+ */
+export declare const statusViewOf: (set: ReplicaSet, ips: Readonly<Record<string, string>>, status: Status) => GroupView;
+/** A replica set lab is one environment of a replica set cell. */
+interface ReplicaSetLab {
+  readonly set: ReplicaSet;
+  readonly clientIp: string;
+  readonly members: Readonly<Record<string, RunPart>>;
+  readonly netns: {
+    readonly client: number;
+  };
+  /** These are all the records that the prober wrote, in sequence. */
+  readonly kernel: readonly KernelRecord[];
+  /** These are the log lines of the members that a run keeps, in the sequence of their arrival, with `t` at 0. */
+  readonly log: readonly MemberLine[];
+  /** This method sets the sysctls of the client namespace through its privileged sidecar, and reads them back. */
+  sysctl(values: Readonly<Record<string, string>>): Promise<Record<string, string>>;
+  /** This method waits until the set is whole and calm, and gives the name of its primary. */
+  converge(within?: number): Promise<string>;
+  /** This method removes each write from the set, and waits until each member has the removal. */
+  flush(primary: string): Promise<void>;
+  /**
+   * This method lets each secondary stand for election again (`rs.freeze(0)`). A step-down holds its
+   * old primary out of the elections for `STEP_DOWN.seconds`. Without it, a run inherits that hold.
+   */
+  unfreeze(primary: string): Promise<void>;
+  /** This method gives the sequence numbers that the set holds, from its primary. */
+  holds(primary: string): Promise<number[]>;
+  /** This method gives the member that has a role in the set, when `primary` is the primary. */
+  partOf(role: GroupRole, primary: string): string;
+  /** This method starts the client of a run, with the arguments of `client-mongo.mjs`. */
+  client(args: readonly string[]): Promise<GroupClient>;
+  /** This method applies a network fault between a member and some of its peers, at once or at an instant of the VM. */
+  fault(kind: GroupNetworkFault, part: string, cuts: NonNullable<GroupPlace["cuts"]>, at?: number): Promise<Stamp>;
+  /** This method removes a network fault, at once or at an instant of the VM. */
+  lift(kind: GroupNetworkFault, part: string, cuts: NonNullable<GroupPlace["cuts"]>, at?: number): Promise<Stamp>;
+  /** This method sends a signal to the server process of a member, at once or at an instant of the VM. */
+  signal(part: string, signal: "KILL" | "STOP" | "CONT", at?: number): Promise<Stamp>;
+  /** This method starts the server of a member again, after a fin. */
+  restart(member: string): Promise<void>;
+  /** This method sends replSetStepDown to the primary. */
+  stepDown(primary: string): Promise<Stamp>;
+  /** This method removes the TCP metrics of the client namespace: each run starts from a kernel that keeps nothing. */
+  forget(): Promise<void>;
+  /** This method waits for the next clock line of the prober, which gives the count of lost records. */
+  settle(): Promise<number | null>;
+  manifest(sysctls: Readonly<Record<string, string>>): Promise<Manifest>;
+  close(): Promise<void>;
+}
+/** These are the arguments of replSetStepDown: the time that the old primary cannot stand again, and the wait for a secondary to catch up. */
+export declare const STEP_DOWN: {
+  readonly seconds: 60;
+  readonly catchUp: 10;
+};
+/**
+ * This function opens one environment of a replica set cell:
+ * - one new network, with each member, the client and the sender of a step-down on it.
+ * - each member: the stock image, its server as the child of a shell. A log consumer reads its
+ *   JSON log. The first member starts the set with the address of each member, and the lab
+ *   waits for a primary.
+ * - an injector sidecar for each member, in its network and process namespaces.
+ * - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
+ * - the prober, which watches the namespace of the client.
+ */
+export declare const openReplicaSetLab: (set: ReplicaSet) => Promise<ReplicaSetLab>;
+/** This function gives the arguments of the client of a replica set cell: each member, and the stream of writes. */
+export declare const replicaSetArgs: (cell: Cell, members: Readonly<Record<string, RunPart>>) => string[];
+/**
+ * This function does one run of a replica set cell:
+ * - The set converges, and its writes go. The client finds the primary with its monitors, and its
+ *   stream of writes gets steady.
+ * - The fault occurs on its member, as the primary of this run makes the roles. A fault with a
+ *   hold lifts at its instant in the VM. A stall lifts at its length after the first write into
+ *   it. A step-down is replSetStepDown to the primary.
+ * - The run waits for its outcome: the first accepted write of a write sent after the commit of
+ *   the fault. It is on a new primary, or on any primary. Then the run waits for the end of the
+ *   hold, thus the record shows what the set did in the whole hold.
+ * - Each fault goes, the client stops, and the set converges again. The writes that it then holds
+ *   tell which writes it kept. The log lines of the members go onto the monotonic clock.
+ */
+export declare const replicaSetOnce: (lab: ReplicaSetLab, cell: Cell, environment: number, index: number) => Promise<Run>;
+//#endregion
 //#region src/run.d.ts
 /** A cell result is a cell with its manifests, its runs, their measurements and its verdict. */
 interface CellResult {
@@ -1539,5 +2126,5 @@ export declare const spread: (runs: number, environments: number) => number[];
  */
 export declare const runCell: (cell: Cell, say?: (line: string) => void) => Promise<CellResult>;
 //#endregion
-export type { Agent, Cell, CellResult, Client, ClientImage, Crossings, Decision, ErrorOnTheWay, Fault, FaultPlace, FaultSide, Group, GroupClient, GroupLab, GroupMeasured, GroupNetworkFault, GroupPlace, GroupPrediction, GroupReading, GroupRole, GroupVerdict, GroupView, Hop, HopPlace, HttpProbe, Interval, KernelRecord, Lab, Manifest, ManifestHop, Measured, MiddleboxEvent, MiddleboxExpectation, MiddleboxForm, NetworkFault, Observer, Outcome, Prediction, Probe, ProbeEvent, ProxyHop, ProxyTime, Recovers, Reference, Reordering, Replies, RequestPrediction, RequestReading, RouterHop, RttRange, Run, RunGroup, RunPart, RunTopology, SentinelEvent, SocketPlace, Spread, Stage, StageBand, StageVerdict, Stamp, Target, Topology, Toxic, Verdict, Writes };
+export type { Agent, CallReading, Cell, CellResult, Client, ClientImage, Crossings, Cut, Decision, DriverBand, DriverSocket, DriverStep, DriverStepTime, ErrorOnTheWay, Fault, FaultPlace, FaultSide, Group, GroupClient, GroupLab, GroupMeasured, GroupNetworkFault, GroupPlace, GroupPrediction, GroupReading, GroupRole, GroupVerdict, GroupView, HarnessImage, Hop, HopPlace, HttpProbe, Interval, KernelRecord, Lab, LineMatch, Manifest, ManifestHop, Measured, MemberLine, MiddleboxEvent, MiddleboxExpectation, MiddleboxForm, NetworkFault, Observer, Outcome, Prediction, Probe, ProbeEvent, ProgramEvent, ProxyHop, ProxyTime, ReconnectReading, ReconnectVerdict, Reconnects, Recovers, Reference, Reordering, ReplicaSet, ReplicaSetLab, ReplicaSetStage, Replies, RequestPrediction, RequestReading, RouterHop, RttRange, Run, RunGroup, RunPart, RunTopology, SentinelEvent, SentinelStage, SocketPlace, Spread, Stage, StageBand, StageVerdict, Stamp, Status, Target, Topology, Toxic, Verdict, Writes, WsProbe };
 //# sourceMappingURL=index.d.ts.map