@mark1russell7/harness 0.5.0 → 0.7.0

package/build/index.d.ts

@@ -36,15 +36,22 @@ import "testcontainers";
  * one side. It is a black hole, a reset or an ICMP unreachable. It can also be a simplex drop, an
  * MTU drop or netem.
  *
- * These faults are of a replicated group (`Cell.group`), on the part that `Cell.faultAt` names:
+ * These faults are of a replicated group (`Cell.group` or `Cell.replicaSet`), on the part that `Cell.faultAt` names:
  * - `blackhole` drops each segment between the part and each of its peers. `rst` answers each
  *   segment from a peer with a reset, thus the part refuses each connection.
  * - `partition` is a black hole between the part and some of its peers only (`GroupPlace.cuts`).
  * - `freeze-process` stops the server process of the part with SIGSTOP, and SIGCONT thaws it. Its
  *   kernel stays up. `fin` stops the process with SIGKILL.
  * - `planned-failover` asks a Sentinel for a failover: `SENTINEL FAILOVER <group>`.
+ *
+ * This fault is of an HTTP server:
+ * - `app-stall` injects nothing. The server process holds the reply to the request, because the
+ *   request asks for that (`HttpProbe.hold`), while its kernel stays up.
+ *
+ * This fault is of a replica set (`Cell.replicaSet`):
+ * - `step-down` sends `replSetStepDown` to the primary, as a client sends a command.
  */
-type Fault = "blackhole" | "freeze-process" | "rst" | "stall" | "fin" | "unreachable" | "simplex" | "mtu-blackhole" | "netem" | "middlebox-forget" | "proxy-idle" | "toxic" | "server-close" | "partition" | "planned-failover" | "none";
+type Fault = "blackhole" | "freeze-process" | "rst" | "stall" | "fin" | "unreachable" | "simplex" | "mtu-blackhole" | "netem" | "middlebox-forget" | "proxy-idle" | "toxic" | "server-close" | "partition" | "planned-failover" | "app-stall" | "step-down" | "none";
 /**
  * A middlebox form tells what a router does with a flow that it forgot, when a late packet comes:
  * - `reset-both` sends a reset to the two ends when it forgets the flow.
@@ -154,6 +161,32 @@ interface Group {
     readonly sentinel: string;
   };
 }
+/**
+ * A replica set is a replicated group with no external authority, as data: its members elect their
+ * own primary. Each member is a container from the same stock image, pinned by digest, in its own
+ * network namespace. The client finds the primary by its own monitors. The harness starts the set
+ * with the address of each member, and it waits for a primary.
+ */
+interface ReplicaSet {
+  /** This name is the name of the set, for example `rs0`. */
+  readonly name: string;
+  /** This image is the stock image of each member, pinned by digest. */
+  readonly image: string;
+  /** These are the programs of the image: the server and its shell. */
+  readonly programs: {
+    readonly server: string;
+    readonly shell: string;
+  };
+  /** These are the options of the server of each member, after the name of the set: for example the size of its cache. */
+  readonly args?: readonly string[];
+  readonly port: number;
+  /** These are the names of the members. The first one starts the set. */
+  readonly members: readonly string[];
+  /** These are the settings of the configuration of the set, as `rs.initiate` takes them: for example `electionTimeoutMillis`. */
+  readonly settings: Readonly<Record<string, number>>;
+  /** This line tells that a member serves. */
+  readonly ready: string;
+}
 /** A group role is a part of a group by its role at the start of a run: the primary, a replica, or a Sentinel. */
 type GroupRole = "primary" | "replica" | "sentinel";
 /**
@@ -162,10 +195,12 @@ type GroupRole = "primary" | "replica" | "sentinel";
  * - `all`: each other part of the group, and each client. This is the default.
  * - `group`: the other members and the Sentinels, but not the clients.
  * - `clients`: the clients, but not the group.
+ * - `one`: one other member, in both directions.
+ * - `one-way`: one other member, for the segments from the part only.
  */
 interface GroupPlace {
   readonly role: GroupRole;
-  readonly cuts?: "all" | "group" | "clients";
+  readonly cuts?: "all" | "group" | "clients" | "one" | "one-way";
 }
 /** A toxic is one toxic of Toxiproxy, as its API takes it. Its toxicity is 1. */
 interface Toxic {
@@ -181,8 +216,11 @@ interface Toxic {
  * send of the client. A FIN counts from the change of state of the socket of the peer when its
  * process dies (`peer-closed`). A toxic of Toxiproxy counts from the start of its commit, as the
  * injector stamps it (`fault`).
+ *
+ * A client that starts into a fault counts from its `connecting` line, or from its first line
+ * without one (`client-start`).
  */
-type Reference = "last-received" | "first-retransmission" | "request-sent" | "peer-closed" | "fault";
+type Reference = "last-received" | "first-retransmission" | "request-sent" | "peer-closed" | "fault" | "client-start";
 /**
  * This prediction tells when a connection moves again after a stall. That occurs at the first
  * firing of its retransmission timer after the stall stops, and its reply comes back one round
@@ -241,8 +279,18 @@ interface ProxyTime {
  * failover and the election. Then the selection of a replica, the promotion, the switch to the new
  * primary, and the conversion of the old primary into a replica.
  */
-type Stage = "+sdown" | "+odown" | "+try-failover" | "+elected-leader" | "+selected-slave" | "+promoted-slave" | "+switch-master" | "+convert-to-slave";
-/** A stage band is when the first Sentinel announces a stage, from the reference of the cell. */
+type SentinelStage = "+sdown" | "+odown" | "+try-failover" | "+elected-leader" | "+selected-slave" | "+promoted-slave" | "+switch-master" | "+convert-to-slave";
+/**
+ * A replica set stage is one step of the timeline of a replica set after a fault. A line of the log
+ * of a member marks it, or a line of the client. The stages are a member that marks the primary
+ * DOWN, the start of an election and its win, and the end of the catch-up. Then come the first
+ * write of the new primary and the step-down of the old one. The client's lines give its discovery
+ * of the new primary, the write-off of the old one, and the first write that a primary accepts.
+ */
+type ReplicaSetStage = "member-down" | "election-start" | "elected" | "caught-up" | "writable" | "old-primary-steps-down" | "discovered" | "written-off" | "first-write";
+/** A stage is one step of the timeline of a group: of a group under Sentinels, or of a replica set. */
+type Stage = SentinelStage | ReplicaSetStage;
+/** A stage band is when the first part announces a stage, from the reference of the cell. */
 interface StageBand {
   readonly stage: Stage;
   readonly low: Quantity;
@@ -256,6 +304,7 @@ interface StageBand {
  *   stamps of the observer (`stamps`).
  * - `errors` are the errors that the client must meet on the way, each with the band of its
  *   first occurrence. An example is the error of the offline queue when it gives up.
+ * - `quiet` are the stages that must not occur after the fault. Each one that a run shows is a finding.
  */
 interface GroupPrediction {
   readonly lands: "new" | "same";
@@ -267,6 +316,7 @@ interface GroupPrediction {
     readonly low: Quantity;
     readonly high: Quantity;
   }>;
+  readonly quiet?: readonly Stage[];
 }
 /**
  * A middlebox expectation tells what the router of a topology does with the flow in the window of
@@ -279,6 +329,74 @@ interface MiddleboxExpectation {
     readonly high: Quantity;
   };
 }
+/**
+ * A request prediction tells what the second request of an HTTP client meets, beside the band of
+ * its cell. The client writes it in its lines: the connection of each try, and the outcome.
+ * - `connection` is the connection of its first try: the one that the first request used
+ *   (`reused`), or a new one.
+ * - `retried` tells that the client sent it again on a new connection by itself.
+ * - `error` is the error that reaches the caller, or null for a reply.
+ * - `pending` tells that nothing ends the request in the window of the cell.
+ */
+interface RequestPrediction {
+  readonly connection: "reused" | "new";
+  readonly retried: boolean;
+  readonly error: string | null;
+  readonly pending?: true;
+}
+/**
+ * A line match is a line of the client that a driver band waits for. It has the `ev` of the line,
+ * and the value of each field that it names. A dotted name is a field in a field, for example
+ * `failure.name`. `name` is the field that names the error of the line, for its outcome.
+ */
+interface LineMatch {
+  readonly ev: string;
+  readonly where?: Readonly<Record<string, string | number | boolean>>;
+  readonly name?: string;
+}
+/**
+ * A driver step is one more band of a driver after the band of its cell. An example is the error
+ * of the retry after the interruption. It has its own reference on a socket of the driver, its
+ * line, and its band. A run whose step is outside its band, or absent, has a finding.
+ */
+interface DriverStep {
+  readonly what: string;
+  /** This name is the name of the error of the line, or OK. */
+  readonly name: string;
+  /** This role is the socket of the reference. Without it, the socket is the socket of the band of the cell. */
+  readonly socket?: string;
+  readonly from: Reference;
+  readonly ends: LineMatch;
+  readonly low: Quantity;
+  readonly high: Quantity;
+}
+/**
+ * A driver band is a band of a cell whose client is a driver with more than one socket, for
+ * example the MongoDB driver. It names the socket of the reference, and the line that ends the
+ * band:
+ * - `socket` is the role of the socket, as the client names it in its `socket` lines. The socket
+ *   `server` is the socket of the target for the first connection of the pool.
+ * - `ends` is the line of the client that marks the detection. Without it, the detection is the
+ *   report of the kernel on the socket: an error, or the end of the file.
+ * - `reply` is the line of a reply that a silent prediction times (`replies`), with its reference.
+ * - `quiet` are the lines that must not occur after the fault, in the window of the prediction.
+ * - `then` are the later bands, each one a test of its own.
+ *
+ * A band whose line ends it is a timer of the driver. Its `last-received` is the last segment with
+ * data that its socket received or sent. The idle timer of a Node socket starts again at each read
+ * and each write. A band of the kernel counts from the last segment of any kind that the socket
+ * received.
+ */
+interface DriverBand {
+  readonly socket: string;
+  readonly ends?: LineMatch;
+  readonly reply?: {
+    readonly from: Reference;
+    readonly ends: LineMatch;
+  };
+  readonly quiet?: readonly LineMatch[];
+  readonly then?: readonly DriverStep[];
+}
 /**
  * A prediction is what a cell predicts. Either a detection occurs in an interval [T_lo, T_hi]
  * after the reference event, or nothing detects the fault in a window. Nothing detects a stall
@@ -296,12 +414,20 @@ interface MiddleboxExpectation {
  *
  * For a group cell, the band is the first write that a primary accepts after the fault, and its
  * errno is `OK`. `group` tells which primary, and the stages of the group on the way.
+ *
+ * For an HTTP cell, `request` tells what the second request meets. The errno `CLOSED` is a close
+ * of the observer itself: its pool closes an idle connection, a FIN that it sends.
+ *
+ * A band of a driver (`driver`) counts from a socket of the driver to a line of the client.
  */
 type Prediction = {
   readonly detects: true;
   /** This name is the timer of the machine that fires first, or the event that detects the fault, with its errno. */
   readonly timer: string;
-  /** This name is the errno of the detection, or EOF for an end of file without an error. It is OK for a write that a primary accepts. */
+  /**
+   * This name is the errno of the detection, or EOF for an end of file without an error. It is
+   * OK for a write that a primary accepts, and CLOSED for a close of the observer itself.
+   */
   readonly errno: string;
   readonly from: Reference;
   /** This time is the closed form, less the most that the timer can fire early. */
@@ -311,6 +437,8 @@ type Prediction = {
   readonly rtt?: RttRange;
   readonly through?: readonly ProxyTime[];
   readonly group?: GroupPrediction;
+  readonly request?: RequestPrediction;
+  readonly driver?: DriverBand;
 } | {
   readonly detects: false;
   readonly window: Quantity;
@@ -320,12 +448,15 @@ type Prediction = {
   readonly rtt?: RttRange;
   readonly through?: readonly ProxyTime[];
   readonly group?: GroupPrediction;
+  readonly request?: RequestPrediction;
+  readonly driver?: DriverBand;
 } | {
   readonly detects: null;
   readonly window: Quantity;
   readonly why: string;
   readonly from?: Reference;
   readonly group?: GroupPrediction;
+  readonly request?: RequestPrediction;
 };
 /** A target is the server that a cell connects to: a stock image. */
 interface Target {
@@ -352,9 +483,46 @@ interface Target {
   readonly kill?: readonly string[];
   /** This command makes the server close the connection of each client, as argv, for a `server-close` fault. */
   readonly close?: readonly string[];
+  /**
+   * These files go into the target before it starts, by their path, for example the script of a
+   * server on a stock Node image. The image stays stock.
+   */
+  readonly files?: Readonly<Record<string, string>>;
+  /**
+   * This flag tells that the target writes JSON lines with an `ev` and a `t` on its standard
+   * output, on CLOCK_MONOTONIC. Each run then keeps its lines (`Run.server`).
+   */
+  readonly lines?: true;
+  /**
+   * This name is the server process, as /proc/PID/comm holds it. With it, a freeze of the target
+   * stops this process with SIGSTOP from a sidecar in its process namespace, and the injector
+   * stamps it. SIGCONT then thaws it. Without it, the freeze is the cgroup freezer, with no stamps.
+   */
+  readonly program?: string;
+}
+/**
+ * A client is what the runs of a cell use: the TCP client of the harness, or a driver on its
+ * socket. The HTTP clients are the `http.Transport` of Go (`go-http`), the `http.Agent` of Node
+ * (`http-agent`) and an HTTP/1.1 `Client` of undici (`undici`). The MongoDB driver is `mongodb`.
+ */
+type Client = "tcp" | "ioredis" | "go-http" | "http-agent" | "undici" | "mongodb";
+/**
+ * An HTTP probe is the workload of an HTTP client. The first request is a GET of the root. The
+ * second request follows the mode of the probe: after the signal (`inflight`), or after a gap
+ * (`gap`).
+ */
+interface HttpProbe {
+  /** This method is the method of the second request. */
+  readonly method: "GET" | "POST";
+  /** This time is the gap of a `gap` workload: from the end of the first reply to the second request. */
+  readonly gap?: Quantity;
+  /** This time is how long the server holds the reply to the second request (`/hold?ms=`). */
+  readonly hold?: Quantity;
+  /** This length is the length of the body of the reply to the second request, in bytes (`/big?bytes=`). */
+  readonly bytes?: number;
+  /** These are settings of the client over its defaults, by the names of its flags: for example `idle-conn-timeout-ms`. */
+  readonly options?: Readonly<Record<string, number>>;
 }
-/** A client is what the runs of a cell use: the TCP client of the harness, or a driver on its socket. */
-type Client = "tcp" | "ioredis";
 /**
  * A client image is a process that a lab starts as the client of each run, in the network
  * namespace of the client. An example is an agent of an application on Node. Its command gets the
@@ -389,8 +557,9 @@ interface Cell {
     /**
      * This mode tells if the client is idle after its first reply, or sends a request into the
      * fault. In a stream, it sends `count` requests into it, each `interval` after the reply before.
+     * An HTTP client in a `gap` sends its second request a gap after its first reply, by itself.
      */
-    readonly mode: "idle" | "inflight" | "stream";
+    readonly mode: "idle" | "inflight" | "stream" | "gap";
     /** This count is the number of requests of a stream. */
     readonly count?: number;
     /** This time is the pause after each reply of a stream, before the next request. */
@@ -406,14 +575,22 @@ interface Cell {
      */
     readonly request?: string;
     /**
-     * This driver replaces the TCP client of the harness, with its own options. It is ioredis 6 on
-     * Node, whose keepAlive is the keepalive delay of the socket. The driver then ignores `keepalive`
-     * and `userTimeout`.
+     * This driver replaces the TCP client of the harness, with its own options. The driver then
+     * ignores `keepalive` and `userTimeout`:
+     * - ioredis 6 on Node, whose keepAlive is the keepalive delay of the socket.
+     * - the MongoDB driver on Node, against the replica set `set`. Its operation is a write, or
+     *   the read of a long document (`read-long`). `Cell.driverOptions` are its options.
      */
     readonly driver?: {
       readonly name: "ioredis";
       readonly keepAlive: Quantity;
+    } | {
+      readonly name: "mongodb";
+      readonly set: string;
+      readonly op?: "write" | "read-long";
     };
+    /** This workload is the workload of an HTTP client (`Cell.client` is an HTTP client). The client then ignores `keepalive` and `userTimeout`. */
+    readonly http?: HttpProbe;
   };
   readonly fault: Fault;
   /** This time is the length of a stall, from the send of the request. */
@@ -451,8 +628,8 @@ interface Cell {
   readonly idleBefore?: Quantity;
   /** This toxic is the toxic of a `toxic` fault, which the Toxiproxy hop gets. */
   readonly toxic?: Toxic;
-  /** This image replaces the TCP client of the harness for each run. */
-  readonly client?: ClientImage;
+  /** This image, or this client of the harness, replaces the TCP client of the harness for each run. An HTTP client is one. */
+  readonly client?: ClientImage | Client;
   /**
    * This name is the proxy hop whose own time a calibration cell measures. The time of a request
    * (`up`) starts at its arrival on the socket of the proxy toward the client. It stops at the
@@ -468,12 +645,28 @@ interface Cell {
    * record, and the group does not read it.
    */
   readonly group?: Group;
-  /** This time is how long a fault on a group holds after its commit. The injector then lifts it at that instant, on the clock of the VM. */
+  /**
+   * This replica set replaces the target: a replicated group whose members elect their own
+   * primary. The client is then the MongoDB driver, and it writes a stream of sequence numbers.
+   * `target` stays for the record, and the set does not read it.
+   */
+  readonly replicaSet?: ReplicaSet;
+  /**
+   * This time is how long a fault on a group holds after its commit. The injector then lifts it at
+   * that instant, on the clock of the VM. On a direct pair, a frozen target thaws after it.
+   */
   readonly hold?: Quantity;
   /** This flag makes a planned failover coordinated: `SENTINEL FAILOVER <group> COORDINATED`. */
   readonly coordinated?: boolean;
-  /** These are options of ioredis over its defaults, for the client of a group. A cell at the defaults has none. */
+  /**
+   * These are options of the driver over its defaults: of ioredis for the client of a group, or of
+   * the MongoDB driver for its client. A cell at the defaults has none.
+   */
   readonly driverOptions?: Readonly<Record<string, unknown>>;
+  /** This write concern is the w of each write of the MongoDB client. Without it, it is majority. */
+  readonly writeConcern?: "majority" | 1;
+  /** This flag makes the fault hold before the client starts. The client then meets it at its first selection. */
+  readonly before?: boolean;
   /** This count is the number of runs of the cell (11), across new environments (two at least). */
   readonly runs: number;
   readonly environments: number;
@@ -549,6 +742,8 @@ interface Run {
   readonly middlebox?: readonly MiddleboxEvent[];
   /** This record is the group of a group cell: its parts, and what it did in the run. */
   readonly group?: RunGroup;
+  /** These are the lines of a target that writes them (`Target.lines`), from the start to the end of the run. */
+  readonly server?: readonly ProbeEvent[];
 }
 /** A Sentinel event is one line of the observer of a group: an event that a Sentinel published, on the clock of the VM. */
 interface SentinelEvent {
@@ -575,7 +770,11 @@ interface RunGroup {
   readonly primary: string;
   /** This name is the part that got the fault, or null for a fault on the whole group. */
   readonly at: string | null;
-  /** These are the lines of the observer of the Sentinels from the start to the end of the run. */
+  /**
+   * These are the lines of the timeline from the start to the end of the run. For a group under
+   * Sentinels, they are the lines of the observer of the Sentinels. For a replica set, they are
+   * the log lines of the members (`ev` "mongod"), with their times on the monotonic clock.
+   */
   readonly timeline: readonly SentinelEvent[];
   /** This name is the member that was the primary when the group converged after the run, or null when it did not converge. */
   readonly after: string | null;
@@ -598,14 +797,76 @@ interface RunTopology {
   }>>;
 }
 //#endregion
+//#region src/driver.d.ts
+/**
+ * This module measures a run of a cell whose client is a driver with more than one socket, for
+ * example the MongoDB driver. Its monitor, its pool and its round-trip pinger each have their own
+ * socket, and the client names each socket by its role. A band counts from an event of one socket,
+ * as the kernel records it. It ends at a line of the client, or at the report of the kernel on
+ * that socket.
+ */
+/** A driver socket is one socket of the driver in a run: its role, its local address, its server, and when it opened and ended. */
+interface DriverSocket {
+  readonly role: string;
+  readonly local: string;
+  readonly server: string;
+  readonly opened: number;
+  readonly ended: number | null;
+}
+/** A driver step time is the time of one later band of a run. It is null when its line did not come. */
+interface DriverStepTime {
+  readonly what: string;
+  readonly name: string | null;
+  readonly elapsed: Quantity | null;
+}
+/** This function gives the value of a field of a line by its name. A dotted name is a field in a field. */
+export declare const fieldOf: (line: unknown, path: string) => unknown;
+/** This function tells if a line of the client matches a line match: its `ev`, and each field that the match names. */
+export declare const matches: (line: ProbeEvent, match: LineMatch) => boolean;
+/** This function gives the sockets of the driver in a run, from the `socket` and `socket-end` lines of the client. */
+export declare const socketsOf: (events: readonly ProbeEvent[]) => DriverSocket[];
+/**
+ * This function finds the socket of a role at an instant. The pool's socket is the first one, which
+ * the client names in its `connected` line. Each other role has the latest socket that opened at
+ * the instant or before it.
+ */
+export declare const socketAt: (events: readonly ProbeEvent[], role: string, at: number) => DriverSocket | undefined;
+/**
+ * This function gives the instant of a reference in a run, on a socket of the driver:
+ * - `last-received`: the last segment that the socket received before the fault held. A band of
+ *   the kernel counts from the last segment of any kind. A band that a line ends is a timer of
+ *   the driver. It counts from the last segment with data that the socket received or sent.
+ * - `request-sent`: the send of the operation into the fault, the second `sent` line.
+ * - `first-retransmission`: the first retransmission of the RTO timer on the socket.
+ * - `peer-closed`: the change of state of the socket of the target, when its process dies.
+ * - `fault`: the start of the commit of the fault. `client-start`: the `connecting` line.
+ */
+export declare const referenceOf: (run: Run, from: Reference, role: string, data: boolean) => number | undefined;
+/**
+ * This function reads one run of a driver cell.
+ * - The band of the cell counts from its reference on its socket. It ends at its line, or at the
+ *   report of the kernel on that socket.
+ * - Each later band (`then`) gets the same test. A later band outside its band, or absent, is a finding.
+ * - A line that the prediction names as quiet, after the fault and in its window, is a finding.
+ *   So is a flow of an open socket of the driver that a router forgot, for a silent prediction.
+ * - A silent prediction can time a reply (`reply`), and the verdict tests it against `replies`.
+ *
+ * A run is void when the prober lost records, or when the client did not get its first reply. It
+ * is also void when the operation went out before the fault held. A late lift of a hold and a
+ * keepalive probe before the fault also make it void. So do a delayed timer and a round trip
+ * above the assumption.
+ */
+export declare const measureDriver: (cell: Cell, run: Run) => Measured;
+//#endregion
 //#region src/timeline.d.ts
 /**
  * This module measures a run of a group cell. It gives the first write that a primary accepted
- * after the fault, and the timeline of the group as its Sentinels announced it. It also gives what
+ * after the fault, and the timeline of the group. The Sentinels of a group announce its timeline.
+ * The logs of the members of a replica set and its client tell it. The module also gives what
  * became of each write.
  */
-/** These are the stages of a timeline, in their sequence after a fault. */
-export declare const STAGES: readonly Stage[];
+/** These are the stages of a timeline under Sentinels, in their sequence after a fault. */
+export declare const STAGES: readonly SentinelStage[];
 /** An error on the way is one kind of error that the client met after the fault. It has its first time from the fault, and its count. */
 interface ErrorOnTheWay {
   readonly name: string;
@@ -644,6 +905,30 @@ interface GroupMeasured {
 }
 /** This function gives the time of each stage of a timeline from an instant: the first announcement of each, after the instant. */
 export declare const stagesOf: (timeline: readonly SentinelEvent[], from: number) => Partial<Record<Stage, Quantity>>;
+/** These are the stages of the timeline of a replica set, in their usual sequence after a fault. */
+export declare const REPLICA_SET_STAGES: readonly ReplicaSetStage[];
+/**
+ * A member mark is how a log line of mongod marks a stage: by its id, or by its message. The
+ * message is the fallback for a version whose ids differ. `at` tells where the line comes from:
+ * the old primary, or another member.
+ */
+interface MemberMark {
+  readonly ids: readonly number[];
+  readonly message: RegExp;
+  readonly at: "old" | "other";
+}
+/** These are the log lines of mongod that mark each stage of a member. */
+export declare const MEMBER_MARKS: Readonly<Record<Exclude<ReplicaSetStage, "discovered" | "written-off" | "first-write">, MemberMark>>;
+/**
+ * This function gives the time of each stage of a replica set from an instant, in a run. The stages
+ * of the members come from their log lines (`ev` "mongod"), whose times the lab puts on the
+ * monotonic clock. The stages of the client come from its lines:
+ * - `discovered`: a member other than the old primary becomes RSPrimary in the view of the client.
+ * - `written-off`: the first heartbeat of the client to the old primary that fails.
+ * - `first-write`: the first write that a primary accepts, of a write that the client sent after
+ *   the commit of the fault (`sentAfter`). It is on a new primary when `lands` is `new`.
+ */
+export declare const replicaSetStagesOf: (run: Run, from: number, sentAfter: number, lands: "new" | "same") => Partial<Record<Stage, Quantity>>;
 /** This function gives what became of each write of a run, from the lines of the client and the keys that the group kept. */
 export declare const writesOf: (run: Run, after: number, primary: string) => Writes;
 /** This function gives the errors that the client met after an instant: each kind, its first time, and its count. */
@@ -673,7 +958,10 @@ interface GroupReading {
  * - The reference is the start of the commit of the fault, as the injector stamped it. For a
  *   stall, it is the first write into the fault, as for the stall of a direct pair.
  * - The outcome is the first write that a primary accepted after the fault. The prediction says
- *   which primary: a new one, or the same one after the lift.
+ *   which primary: a new one, or the same one after the lift. In a replica set, it is the first
+ *   accepted write of a write that the client sent after the commit of the fault.
+ * - The stages come from the Sentinels, or from the logs of the members of a replica set and the
+ *   lines of its client (`replicaSetStagesOf`).
  * - Its time is the last segment with data that the client's socket to that primary received
  *   before the client saw the acceptance. The stamp of the client is the fallback.
  * - A run is void when the prober lost records, or when the client did not become ready. It is
@@ -697,7 +985,21 @@ type Outcome = {
   readonly kind: "silent";
 } | {
   readonly kind: "answered";
+} | {
+  readonly kind: "closed";
 };
+/**
+ * A request reading is what the second request of an HTTP client met, from the lines of the
+ * client. It holds the connection of its first try, a new try, and its outcome. `gap` is the time
+ * from the end of the first reply to the send of the second request.
+ */
+interface RequestReading {
+  readonly connection: "reused" | "new";
+  readonly retried: boolean;
+  readonly error: string | null;
+  readonly pending: boolean;
+  readonly gap: Quantity | null;
+}
 /** A measurement is what one run measured, against the prediction of its cell. */
 interface Measured {
   readonly environment: number;
@@ -746,6 +1048,12 @@ interface Measured {
   readonly crossings?: Crossings;
   /** This record is what a run of a group cell measured: the stages of its timeline, and what became of each write. */
   readonly group?: GroupMeasured;
+  /** This record is what the second request of an HTTP client met, for a cell whose prediction says it. */
+  readonly request?: RequestReading | null;
+  /** This record is the time of each later band of a driver cell (`DriverBand.then`). */
+  readonly driver?: {
+    readonly steps: readonly DriverStepTime[];
+  };
   /** This text is the reason that the run cannot count, or null. A void run is not a pass and not a finding. */
   readonly void: string | null;
   /** This text tells what in the run disagrees with the prediction, other than its timing, or null. */
@@ -818,10 +1126,28 @@ export declare const crossingsOf: (run: Run, hop: string, toClient: SocketPlace
  *   it, and before the first keepalive probe. A live peer answers that probe, and the answer
  *   starts the idle time again. A run where the fault was late is void. A run whose records the
  *   prober lost is also void.
+ * - For an HTTP client, it also reads what the second request met (`requestOf`). An outcome other
+ *   than the one that the prediction gives is a finding.
  */
 export declare const measure: (cell: Cell, run: Run) => Measured;
+/**
+ * This function reads the second request of an HTTP client from its lines. Its first try is its
+ * first `request` line, and each `retry` line is a new try. Its end is its `error` line, or its
+ * `replied` line. A request with neither is pending. It gives null when the client wrote no try.
+ *
+ * A new try that gets no connection writes no `retry` line. Its dial writes a `dial-error` line.
+ * Thus a `dial-error` line after the first try and before the end is also a new try.
+ */
+export declare const requestOf: (events: readonly ProbeEvent[]) => RequestReading | null;
 //#endregion
 //#region src/verdict.d.ts
+/** This function gives the words of the outcome of a second request: its connection, a retry, and its end. */
+export declare const requestWords: (outcome: {
+  readonly connection: "reused" | "new";
+  readonly retried: boolean;
+  readonly error: string | null;
+  readonly pending?: boolean | undefined;
+}) => string;
 /**
  * This amount is the noise of the pipeline itself, δ_abs, as the calibration measured it. It is
  * the 99.9th percentile of the whole latency of a reset, at the concurrency cap or less. Its
@@ -877,6 +1203,12 @@ interface Verdict {
   };
   /** This record is what the group of a group cell did over the counted runs: its stages, and its writes. */
   readonly group?: GroupVerdict;
+  /**
+   * These are the outcomes of the second request of an HTTP client over the counted runs, each
+   * with its count: for example "reused, a retry, a reply". An outcome other than the predicted
+   * one is a finding of its run.
+   */
+  readonly requests?: Readonly<Record<string, number>>;
   readonly why: string;
 }
 /**
@@ -961,6 +1293,12 @@ export declare const harnessImage: () => Promise<string>;
 export declare const nodeClientImage: () => Promise<string>;
 /** This function gives the image of the router. */
 export declare const routerImage: () => Promise<string>;
+/** This function gives the image of the Go HTTP client. */
+export declare const goHttpImage: () => Promise<string>;
+/** This function gives the image of the Node HTTP clients. */
+export declare const nodeHttpImage: () => Promise<string>;
+/** This function gives the image of the MongoDB client. */
+export declare const mongoClientImage: () => Promise<string>;
 /**
  * This time is the interval of the listing of a router, when its hop does not set one. A listing
  * evicts each expired flow that it walks past, and the DESTROY event of that flow follows.
@@ -1022,6 +1360,10 @@ interface Manifest {
     readonly settings: Readonly<Record<string, string>>;
     /** These are the sysctls of its namespace, as read back after the harness set them. */
     readonly sysctls: Readonly<Record<string, string>>;
+    /** These are the files that went into the target, by their path, each with the sha256 of its content. */
+    readonly files?: Readonly<Record<string, string>>;
+    /** This line is the first line of a target that writes them: a server that tells its settings as it reads them. */
+    readonly first?: ProbeEvent;
   };
   readonly harness: {
     readonly image: string;
@@ -1113,6 +1455,8 @@ interface Lab {
   readonly topology: RunTopology | null;
   /** These are the lines of the routers of a topology, in sequence, each with the name of its hop. A direct pair has none. */
   readonly middlebox: readonly MiddleboxEvent[];
+  /** These are the lines of a target that writes them (`Target.lines`), in sequence. Another target has none. */
+  readonly server: readonly ProbeEvent[];
   /** This method adds a toxic to the Toxiproxy hop of a topology, or removes it with null. The stamps of the injector bracket it. */
   toxic(toxic: Toxic | null): Promise<{
     t0: number;
@@ -1190,6 +1534,15 @@ interface Lab {
   /** This method freezes the target with the cgroup freezer: its process stops, and its kernel continues to answer. */
   freeze(): Promise<void>;
   thaw(): Promise<void>;
+  /**
+   * This method sends a signal to the server process of the target (`Target.program`), at once or
+   * at an instant of the VM. A sidecar in the network and process namespaces of the target sends
+   * it, and the injector stamps it. The lab starts that sidecar at the first signal.
+   */
+  signal(signal: "STOP" | "CONT" | "KILL", at?: number): Promise<{
+    t0: number;
+    t1: number;
+  }>;
   probe(args: readonly string[]): Promise<Probe>;
   /**
    * This method waits for the next clock line of the prober, which gives the count of lost
@@ -1377,6 +1730,157 @@ export declare const subscriptionsOf: (timeline: readonly SentinelEvent[]) => Se
  */
 export declare const groupOnce: (lab: GroupLab, cell: Cell, environment: number, index: number) => Promise<Run>;
 //#endregion
+//#region src/replset.d.ts
+/**
+ * This module is the lab of a replica set: a replicated group with no external authority, whose
+ * members elect their own primary. It opens one environment for a replica set cell, for the runs
+ * of the cell. Each member is a container from the stock image of the set, in its own network
+ * namespace. An injector sidecar shares its network and process namespaces.
+ *
+ * The client of each run is the MongoDB driver, which finds the primary with its own monitors. The
+ * timeline comes from the log of each member and from the lines of the client. The prober watches
+ * the namespace of the client.
+ */
+/**
+ * A member line is one log line of a member, as mongod writes it in JSON. `real` is its time on
+ * the wall clock in ms, and `t` is its time on the monotonic clock. The run gives `t` from the
+ * clock lines of its client.
+ */
+interface MemberLine extends SentinelEvent {
+  readonly ev: "mongod";
+  readonly member: string;
+  readonly real: number;
+  readonly id: number;
+  readonly c: string;
+  readonly msg: string;
+  readonly attr: Readonly<Record<string, unknown>>;
+}
+/** These are the components of the log of mongod that a run keeps: replication, its heartbeats, the elections, the rollbacks and the commands. */
+export declare const KEPT: ReadonlySet<string>;
+/** This function reads one log line of a member, or gives null for a line of another component or a line that is not JSON. */
+export declare const memberLineOf: (member: string, text: string) => MemberLine | null;
+/**
+ * This function gives the offset of the wall clock from the monotonic clock, in ns, from the clock
+ * lines of a client. Each line pairs its stamp with the wall clock in whole ms, which truncates.
+ * Thus each pair is the offset or less, and the largest pair is the nearest. It is null without a clock line.
+ */
+export declare const clockOffsetOf: (events: readonly ProbeEvent[]) => number | null;
+/** This function puts the log lines of the members on the monotonic clock, with the offset of a client. */
+export declare const onMonotonic: (log: readonly MemberLine[], offset: number | null) => MemberLine[];
+/** A cut is the peers that a network fault on a part cuts it from, by their addresses, and the direction of the rules. */
+interface Cut {
+  readonly peers: readonly string[];
+  readonly direction?: "out";
+}
+/**
+ * This function gives the peers that a fault on a member of a replica set cuts it from. `all` is
+ * each other member and the client, `group` is the other members, and `clients` is the client.
+ * `one` is the first other member in the sequence of the set, and `one-way` is the same member for
+ * the segments from the part only.
+ */
+export declare const cutOf: (set: ReplicaSet, members: Readonly<Record<string, RunPart>>, client: string, part: string, cuts: NonNullable<GroupPlace["cuts"]>) => Cut;
+/** This function gives the command of the server of each member. The server is the child of a shell, thus a fin can stop it and the container stays up. */
+export declare const memberCommand: (set: ReplicaSet) => string;
+/** This function gives the script of the shell that starts the set: each member by its address, and the settings of the set. */
+export declare const initiateScript: (set: ReplicaSet, ips: Readonly<Record<string, string>>) => string;
+/** This script prints the view of a member: itself, if it accepts writes, and the state of each member as it sees them. */
+export declare const STATUS_SCRIPT: string;
+/** A status is the view of one member, as `STATUS_SCRIPT` prints it. */
+interface Status {
+  readonly me: string;
+  readonly writable: boolean;
+  readonly members: ReadonlyArray<{
+    readonly name: string;
+    readonly state: string;
+    readonly health: number;
+    readonly optime: number | null;
+  }>;
+}
+/**
+ * This function judges the view of a member: the primary of the set, and if the set is healthy.
+ * The set is healthy when the member is the one primary and accepts writes. Each other member must
+ * be a healthy secondary at the optime of the primary.
+ */
+export declare const statusViewOf: (set: ReplicaSet, ips: Readonly<Record<string, string>>, status: Status) => GroupView;
+/** A replica set lab is one environment of a replica set cell. */
+interface ReplicaSetLab {
+  readonly set: ReplicaSet;
+  readonly clientIp: string;
+  readonly members: Readonly<Record<string, RunPart>>;
+  readonly netns: {
+    readonly client: number;
+  };
+  /** These are all the records that the prober wrote, in sequence. */
+  readonly kernel: readonly KernelRecord[];
+  /** These are the log lines of the members that a run keeps, in the sequence of their arrival, with `t` at 0. */
+  readonly log: readonly MemberLine[];
+  /** This method sets the sysctls of the client namespace through its privileged sidecar, and reads them back. */
+  sysctl(values: Readonly<Record<string, string>>): Promise<Record<string, string>>;
+  /** This method waits until the set is whole and calm, and gives the name of its primary. */
+  converge(within?: number): Promise<string>;
+  /** This method removes each write from the set, and waits until each member has the removal. */
+  flush(primary: string): Promise<void>;
+  /**
+   * This method lets each secondary stand for election again (`rs.freeze(0)`). A step-down holds its
+   * old primary out of the elections for `STEP_DOWN.seconds`. Without it, a run inherits that hold.
+   */
+  unfreeze(primary: string): Promise<void>;
+  /** This method gives the sequence numbers that the set holds, from its primary. */
+  holds(primary: string): Promise<number[]>;
+  /** This method gives the member that has a role in the set, when `primary` is the primary. */
+  partOf(role: GroupRole, primary: string): string;
+  /** This method starts the client of a run, with the arguments of `client-mongo.mjs`. */
+  client(args: readonly string[]): Promise<GroupClient>;
+  /** This method applies a network fault between a member and some of its peers, at once or at an instant of the VM. */
+  fault(kind: GroupNetworkFault, part: string, cuts: NonNullable<GroupPlace["cuts"]>, at?: number): Promise<Stamp>;
+  /** This method removes a network fault, at once or at an instant of the VM. */
+  lift(kind: GroupNetworkFault, part: string, cuts: NonNullable<GroupPlace["cuts"]>, at?: number): Promise<Stamp>;
+  /** This method sends a signal to the server process of a member, at once or at an instant of the VM. */
+  signal(part: string, signal: "KILL" | "STOP" | "CONT", at?: number): Promise<Stamp>;
+  /** This method starts the server of a member again, after a fin. */
+  restart(member: string): Promise<void>;
+  /** This method sends replSetStepDown to the primary. */
+  stepDown(primary: string): Promise<Stamp>;
+  /** This method removes the TCP metrics of the client namespace: each run starts from a kernel that keeps nothing. */
+  forget(): Promise<void>;
+  /** This method waits for the next clock line of the prober, which gives the count of lost records. */
+  settle(): Promise<number | null>;
+  manifest(sysctls: Readonly<Record<string, string>>): Promise<Manifest>;
+  close(): Promise<void>;
+}
+/** These are the arguments of replSetStepDown: the time that the old primary cannot stand again, and the wait for a secondary to catch up. */
+export declare const STEP_DOWN: {
+  readonly seconds: 60;
+  readonly catchUp: 10;
+};
+/**
+ * This function opens one environment of a replica set cell:
+ * - one new network, with each member, the client and the sender of a step-down on it.
+ * - each member: the stock image, its server as the child of a shell. A log consumer reads its
+ *   JSON log. The first member starts the set with the address of each member, and the lab
+ *   waits for a primary.
+ * - an injector sidecar for each member, in its network and process namespaces.
+ * - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
+ * - the prober, which watches the namespace of the client.
+ */
+export declare const openReplicaSetLab: (set: ReplicaSet) => Promise<ReplicaSetLab>;
+/** This function gives the arguments of the client of a replica set cell: each member, and the stream of writes. */
+export declare const replicaSetArgs: (cell: Cell, members: Readonly<Record<string, RunPart>>) => string[];
+/**
+ * This function does one run of a replica set cell:
+ * - The set converges, and its writes go. The client finds the primary with its monitors, and its
+ *   stream of writes gets steady.
+ * - The fault occurs on its member, as the primary of this run makes the roles. A fault with a
+ *   hold lifts at its instant in the VM. A stall lifts at its length after the first write into
+ *   it. A step-down is replSetStepDown to the primary.
+ * - The run waits for its outcome: the first accepted write of a write sent after the commit of
+ *   the fault. It is on a new primary, or on any primary. Then the run waits for the end of the
+ *   hold, thus the record shows what the set did in the whole hold.
+ * - Each fault goes, the client stops, and the set converges again. The writes that it then holds
+ *   tell which writes it kept. The log lines of the members go onto the monotonic clock.
+ */
+export declare const replicaSetOnce: (lab: ReplicaSetLab, cell: Cell, environment: number, index: number) => Promise<Run>;
+//#endregion
 //#region src/run.d.ts
 /** A cell result is a cell with its manifests, its runs, their measurements and its verdict. */
 interface CellResult {
@@ -1402,6 +1906,16 @@ export declare const judge: (cell: Cell, runs: readonly Run[]) => {
   measured: Measured[];
   verdict: Verdict;
 };
+/**
+ * This time is the pause between the client's first reply and the fault. It is more than Linux's
+ * TCP_DELACK_MAX of 200 ms. Thus the delayed ACK of the reply is out before the fault.
+ */
+export declare const QUIET_MS = 250;
+/**
+ * This time is how long an HTTP client waits after its second request, before it exits. The exit
+ * closes its sockets, thus that close comes after the window of the cell.
+ */
+export declare const LINGER_MS = 1e3;
 /** This function divides a count of runs across environments, as equally as possible. */
 export declare const spread: (runs: number, environments: number) => number[];
 /**
@@ -1415,5 +1929,5 @@ export declare const spread: (runs: number, environments: number) => number[];
  */
 export declare const runCell: (cell: Cell, say?: (line: string) => void) => Promise<CellResult>;
 //#endregion
-export type { Agent, Cell, CellResult, Client, ClientImage, Crossings, Decision, ErrorOnTheWay, Fault, FaultPlace, FaultSide, Group, GroupClient, GroupLab, GroupMeasured, GroupNetworkFault, GroupPlace, GroupPrediction, GroupReading, GroupRole, GroupVerdict, GroupView, Hop, HopPlace, Interval, KernelRecord, Lab, Manifest, ManifestHop, Measured, MiddleboxEvent, MiddleboxExpectation, MiddleboxForm, NetworkFault, Observer, Outcome, Prediction, Probe, ProbeEvent, ProxyHop, ProxyTime, Recovers, Reference, Reordering, Replies, RouterHop, RttRange, Run, RunGroup, RunPart, RunTopology, SentinelEvent, SocketPlace, Spread, Stage, StageBand, StageVerdict, Stamp, Target, Topology, Toxic, Verdict, Writes };
+export type { Agent, Cell, CellResult, Client, ClientImage, Crossings, Cut, Decision, DriverBand, DriverSocket, DriverStep, DriverStepTime, ErrorOnTheWay, Fault, FaultPlace, FaultSide, Group, GroupClient, GroupLab, GroupMeasured, GroupNetworkFault, GroupPlace, GroupPrediction, GroupReading, GroupRole, GroupVerdict, GroupView, Hop, HopPlace, HttpProbe, Interval, KernelRecord, Lab, LineMatch, Manifest, ManifestHop, Measured, MemberLine, MiddleboxEvent, MiddleboxExpectation, MiddleboxForm, NetworkFault, Observer, Outcome, Prediction, Probe, ProbeEvent, ProxyHop, ProxyTime, Recovers, Reference, Reordering, ReplicaSet, ReplicaSetLab, ReplicaSetStage, Replies, RequestPrediction, RequestReading, RouterHop, RttRange, Run, RunGroup, RunPart, RunTopology, SentinelEvent, SentinelStage, SocketPlace, Spread, Stage, StageBand, StageVerdict, Stamp, Status, Target, Topology, Toxic, Verdict, Writes };
 //# sourceMappingURL=index.d.ts.map