@mark1russell7/harness 0.4.1 → 0.6.0

package/build/index.d.ts

@@ -35,8 +35,20 @@ import "testcontainers";
  * A fault at a router of the topology (`Cell.faultAt`) acts on what the router forwards toward
  * one side. It is a black hole, a reset or an ICMP unreachable. It can also be a simplex drop, an
  * MTU drop or netem.
+ *
+ * These faults are of a replicated group (`Cell.group`), on the part that `Cell.faultAt` names:
+ * - `blackhole` drops each segment between the part and each of its peers. `rst` answers each
+ *   segment from a peer with a reset, thus the part refuses each connection.
+ * - `partition` is a black hole between the part and some of its peers only (`GroupPlace.cuts`).
+ * - `freeze-process` stops the server process of the part with SIGSTOP, and SIGCONT thaws it. Its
+ *   kernel stays up. `fin` stops the process with SIGKILL.
+ * - `planned-failover` asks a Sentinel for a failover: `SENTINEL FAILOVER <group>`.
+ *
+ * This fault is of an HTTP server:
+ * - `app-stall` injects nothing. The server process holds the reply to the request, because the
+ *   request asks for that (`HttpProbe.hold`), while its kernel stays up.
  */
-type Fault = "blackhole" | "freeze-process" | "rst" | "stall" | "fin" | "unreachable" | "simplex" | "mtu-blackhole" | "netem" | "middlebox-forget" | "proxy-idle" | "toxic" | "server-close" | "none";
+type Fault = "blackhole" | "freeze-process" | "rst" | "stall" | "fin" | "unreachable" | "simplex" | "mtu-blackhole" | "netem" | "middlebox-forget" | "proxy-idle" | "toxic" | "server-close" | "partition" | "planned-failover" | "app-stall" | "none";
 /**
  * A middlebox form tells what a router does with a flow that it forgot, when a late packet comes:
  * - `reset-both` sends a reset to the two ends when it forgets the flow.
@@ -108,6 +120,57 @@ interface Topology {
   readonly name: string;
   readonly hops: readonly Hop[];
 }
+/**
+ * A group is a replicated group under an external failover authority, as data. It is a primary and
+ * its replicas, and the Sentinels that decide which member is the primary. Each member and each
+ * Sentinel is a container from the same stock image, pinned by digest. Each has its own network
+ * namespace, thus a fault can act on one part. The client finds the primary through the
+ * Sentinels, and the parts and the client share one network.
+ */
+interface Group {
+  /** This name is the name of the group that the Sentinels monitor, for example `mymaster`. */
+  readonly name: string;
+  /** This image is the stock image of each member and each Sentinel, pinned by digest. */
+  readonly image: string;
+  /** These are the programs of the image: the server, the Sentinel and the command line client. */
+  readonly programs: {
+    readonly server: string;
+    readonly sentinel: string;
+    readonly cli: string;
+  };
+  readonly port: number;
+  readonly sentinelPort: number;
+  /** These are the names of the members. The first one is the primary when the environment starts. */
+  readonly members: readonly string[];
+  readonly sentinels: readonly string[];
+  /** This count is how many Sentinels must agree that the primary is down. */
+  readonly quorum: number;
+  /** These are the settings of each member, as its configuration file spells them. The harness adds `replicaof` for each replica. */
+  readonly memberSettings: Readonly<Record<string, string>>;
+  /**
+   * These are the settings of each Sentinel for the group, as `sentinel <setting> <group> <value>`
+   * spells them: `down-after-milliseconds`, `failover-timeout` and `parallel-syncs`.
+   */
+  readonly sentinelSettings: Readonly<Record<string, string>>;
+  /** These lines tell that a member and a Sentinel serve. */
+  readonly ready: {
+    readonly member: string;
+    readonly sentinel: string;
+  };
+}
+/** A group role is a part of a group by its role at the start of a run: the primary, a replica, or a Sentinel. */
+type GroupRole = "primary" | "replica" | "sentinel";
+/**
+ * A group place is the part of a group that a fault acts on. A network fault cuts the part from
+ * some of its peers:
+ * - `all`: each other part of the group, and each client. This is the default.
+ * - `group`: the other members and the Sentinels, but not the clients.
+ * - `clients`: the clients, but not the group.
+ */
+interface GroupPlace {
+  readonly role: GroupRole;
+  readonly cuts?: "all" | "group" | "clients";
+}
 /** A toxic is one toxic of Toxiproxy, as its API takes it. Its toxicity is 1. */
 interface Toxic {
   readonly type: "latency" | "timeout" | "reset_peer" | "slow_close" | "bandwidth" | "limit_data" | "slicer";
@@ -176,12 +239,71 @@ interface ProxyTime {
   readonly quantile: number;
   readonly time: Quantity;
 }
+/**
+ * A stage is one step of the timeline of a group after a fault, by the event that a Sentinel
+ * publishes for it. The stages are the subjective down, the objective down, the start of a
+ * failover and the election. Then the selection of a replica, the promotion, the switch to the new
+ * primary, and the conversion of the old primary into a replica.
+ */
+type Stage = "+sdown" | "+odown" | "+try-failover" | "+elected-leader" | "+selected-slave" | "+promoted-slave" | "+switch-master" | "+convert-to-slave";
+/** A stage band is when the first Sentinel announces a stage, from the reference of the cell. */
+interface StageBand {
+  readonly stage: Stage;
+  readonly low: Quantity;
+  readonly high: Quantity;
+}
+/**
+ * A group prediction tells what a group does after the fault, beside the band of its cell:
+ * - `lands` is the primary that the first accepted write after the fault lands on: a new one, or the same one.
+ * - `failover` tells if a Sentinel announces a switch to a new primary in the window.
+ * - `stages` are the bands of the stages of the timeline. Each band includes the error of the
+ *   stamps of the observer (`stamps`).
+ * - `errors` are the errors that the client must meet on the way, each with the band of its
+ *   first occurrence. An example is the error of the offline queue when it gives up.
+ */
+interface GroupPrediction {
+  readonly lands: "new" | "same";
+  readonly failover: boolean;
+  readonly stages: readonly StageBand[];
+  readonly stamps?: Quantity;
+  readonly errors?: ReadonlyArray<{
+    readonly name: string;
+    readonly low: Quantity;
+    readonly high: Quantity;
+  }>;
+}
+/**
+ * A middlebox expectation tells what the router of a topology does with the flow in the window of
+ * a silent prediction. Without it, the router keeps the flow. With `forgets`, the router forgets
+ * the flow in that band after the last segment of the client, and nothing detects it.
+ */
+interface MiddleboxExpectation {
+  readonly forgets: {
+    readonly low: Quantity;
+    readonly high: Quantity;
+  };
+}
+/**
+ * A request prediction tells what the second request of an HTTP client meets, beside the band of
+ * its cell. The client writes it in its lines: the connection of each try, and the outcome.
+ * - `connection` is the connection of its first try: the one that the first request used
+ *   (`reused`), or a new one.
+ * - `retried` tells that the client sent it again on a new connection by itself.
+ * - `error` is the error that reaches the caller, or null for a reply.
+ * - `pending` tells that nothing ends the request in the window of the cell.
+ */
+interface RequestPrediction {
+  readonly connection: "reused" | "new";
+  readonly retried: boolean;
+  readonly error: string | null;
+  readonly pending?: true;
+}
 /**
  * A prediction is what a cell predicts. Either a detection occurs in an interval [T_lo, T_hi]
  * after the reference event, or nothing detects the fault in a window. Nothing detects a stall
  * that is shorter than the give-up, and the connection recovers from it. The prediction then also
  * tells when. A silent prediction can also tell when the reply to the request comes back
- * (`replies`).
+ * (`replies`), and what a router does with the flow (`middlebox`).
  *
  * A band can name the first round trip that it assumes (`rtt`), and the proxy times that it
  * includes (`through`).
@@ -190,12 +312,21 @@ interface ProxyTime {
  * prediction is `detects: null` with the reason, and its runs are measurements only. A
  * measurement-only cell that names a reference (`from`) also records the time from it to the end
  * of the connection, or to the last data.
+ *
+ * For a group cell, the band is the first write that a primary accepts after the fault, and its
+ * errno is `OK`. `group` tells which primary, and the stages of the group on the way.
+ *
+ * For an HTTP cell, `request` tells what the second request meets. The errno `CLOSED` is a close
+ * of the observer itself: its pool closes an idle connection, a FIN that it sends.
  */
 type Prediction = {
   readonly detects: true;
   /** This name is the timer of the machine that fires first, or the event that detects the fault, with its errno. */
   readonly timer: string;
-  /** This name is the errno of the detection, or EOF for an end of file without an error. */
+  /**
+   * This name is the errno of the detection, or EOF for an end of file without an error. It is
+   * OK for a write that a primary accepts, and CLOSED for a close of the observer itself.
+   */
   readonly errno: string;
   readonly from: Reference;
   /** This time is the closed form, less the most that the timer can fire early. */
@@ -204,18 +335,25 @@ type Prediction = {
   readonly high: Quantity;
   readonly rtt?: RttRange;
   readonly through?: readonly ProxyTime[];
+  readonly group?: GroupPrediction;
+  readonly request?: RequestPrediction;
 } | {
   readonly detects: false;
   readonly window: Quantity;
   readonly recovers?: Recovers;
   readonly replies?: Replies;
+  readonly middlebox?: MiddleboxExpectation;
   readonly rtt?: RttRange;
   readonly through?: readonly ProxyTime[];
+  readonly group?: GroupPrediction;
+  readonly request?: RequestPrediction;
 } | {
   readonly detects: null;
   readonly window: Quantity;
   readonly why: string;
   readonly from?: Reference;
+  readonly group?: GroupPrediction;
+  readonly request?: RequestPrediction;
 };
 /** A target is the server that a cell connects to: a stock image. */
 interface Target {
@@ -242,9 +380,40 @@ interface Target {
   readonly kill?: readonly string[];
   /** This command makes the server close the connection of each client, as argv, for a `server-close` fault. */
   readonly close?: readonly string[];
+  /**
+   * These files go into the target before it starts, by their path, for example the script of a
+   * server on a stock Node image. The image stays stock.
+   */
+  readonly files?: Readonly<Record<string, string>>;
+  /**
+   * This flag tells that the target writes JSON lines with an `ev` and a `t` on its standard
+   * output, on CLOCK_MONOTONIC. Each run then keeps its lines (`Run.server`).
+   */
+  readonly lines?: true;
+}
+/**
+ * A client is what the runs of a cell use: the TCP client of the harness, or a driver on its
+ * socket. The HTTP clients are the `http.Transport` of Go (`go-http`), the `http.Agent` of Node
+ * (`http-agent`) and an HTTP/1.1 `Client` of undici (`undici`).
+ */
+type Client = "tcp" | "ioredis" | "go-http" | "http-agent" | "undici";
+/**
+ * An HTTP probe is the workload of an HTTP client. The first request is a GET of the root. The
+ * second request follows the mode of the probe: after the signal (`inflight`), or after a gap
+ * (`gap`).
+ */
+interface HttpProbe {
+  /** This method is the method of the second request. */
+  readonly method: "GET" | "POST";
+  /** This time is the gap of a `gap` workload: from the end of the first reply to the second request. */
+  readonly gap?: Quantity;
+  /** This time is how long the server holds the reply to the second request (`/hold?ms=`). */
+  readonly hold?: Quantity;
+  /** This length is the length of the body of the reply to the second request, in bytes (`/big?bytes=`). */
+  readonly bytes?: number;
+  /** These are settings of the client over its defaults, by the names of its flags: for example `idle-conn-timeout-ms`. */
+  readonly options?: Readonly<Record<string, number>>;
 }
-/** A client is what the runs of a cell use: the TCP client of the harness, or a driver on its socket. */
-type Client = "tcp" | "ioredis";
 /**
  * A client image is a process that a lab starts as the client of each run, in the network
  * namespace of the client. An example is an agent of an application on Node. Its command gets the
@@ -279,8 +448,9 @@ interface Cell {
     /**
      * This mode tells if the client is idle after its first reply, or sends a request into the
      * fault. In a stream, it sends `count` requests into it, each `interval` after the reply before.
+     * An HTTP client in a `gap` sends its second request a gap after its first reply, by itself.
      */
-    readonly mode: "idle" | "inflight" | "stream";
+    readonly mode: "idle" | "inflight" | "stream" | "gap";
     /** This count is the number of requests of a stream. */
     readonly count?: number;
     /** This time is the pause after each reply of a stream, before the next request. */
@@ -304,6 +474,8 @@ interface Cell {
       readonly name: "ioredis";
       readonly keepAlive: Quantity;
     };
+    /** This workload is the workload of an HTTP client (`Cell.client` is an HTTP client). The client then ignores `keepalive` and `userTimeout`. */
+    readonly http?: HttpProbe;
   };
   readonly fault: Fault;
   /** This time is the length of a stall, from the send of the request. */
@@ -341,14 +513,29 @@ interface Cell {
   readonly idleBefore?: Quantity;
   /** This toxic is the toxic of a `toxic` fault, which the Toxiproxy hop gets. */
   readonly toxic?: Toxic;
-  /** This image replaces the TCP client of the harness for each run. */
-  readonly client?: ClientImage;
+  /** This image, or this client of the harness, replaces the TCP client of the harness for each run. An HTTP client is one. */
+  readonly client?: ClientImage | Client;
   /**
    * This name is the proxy hop whose own time a calibration cell measures. The time of a request
    * (`up`) starts at its arrival on the socket of the proxy toward the client. It stops at the
    * write on the socket toward the target. The time of its reply (`down`) goes the other way.
+   *
+   * For a group cell, `sentinels` makes the prober watch the namespace of each Sentinel too. Each
+   * stamp of the observer then gets the write of its Sentinel that it read (`lags`).
    */
   readonly calibrates?: string;
+  /**
+   * This group replaces the target: a replicated group under its Sentinels. The client is then
+   * ioredis in Sentinel mode, and it writes a stream of sequence numbers. `target` stays for the
+   * record, and the group does not read it.
+   */
+  readonly group?: Group;
+  /** This time is how long a fault on a group holds after its commit. The injector then lifts it at that instant, on the clock of the VM. */
+  readonly hold?: Quantity;
+  /** This flag makes a planned failover coordinated: `SENTINEL FAILOVER <group> COORDINATED`. */
+  readonly coordinated?: boolean;
+  /** These are options of ioredis over its defaults, for the client of a group. A cell at the defaults has none. */
+  readonly driverOptions?: Readonly<Record<string, unknown>>;
   /** This count is the number of runs of the cell (11), across new environments (two at least). */
   readonly runs: number;
   readonly environments: number;
@@ -359,16 +546,22 @@ interface Cell {
    */
   readonly concurrency?: number;
 }
+/** A hop place is a hop of a topology, with the side that the rules of a fault face. */
+interface HopPlace {
+  readonly hop: string;
+  readonly toward: "client" | "target";
+}
 /**
  * A fault place is the namespace that gets the rules of a fault. "target" and "client" are the two
  * ends. A hop place is a hop, with the side that the rules face. That side is the next hop toward
  * the client, or the next hop toward the target. The rules of a proxy act on its sockets. The
  * rules of a router act on what it forwards, by its device toward that side: the FORWARD chain.
+ *
+ * A group place is a part of a group, by its role.
  */
-type FaultPlace = "target" | "client" | {
-  readonly hop: string;
-  readonly toward: "client" | "target";
-};
+type FaultPlace = "target" | "client" | HopPlace | GroupPlace;
+/** This function tells if a fault place is a hop of a topology. */
+export declare const isHopPlace: (place: FaultPlace | undefined) => place is HopPlace;
 /** A middlebox event is one line of the router: a change of a conntrack entry, or a reset that it sent. `t` is CLOCK_MONOTONIC in nanoseconds. */
 interface MiddleboxEvent {
   readonly ev: string;
@@ -416,6 +609,42 @@ interface Run {
   readonly topology?: RunTopology;
   /** These are the lines of the routers of a topology from the start to the end of the run. */
   readonly middlebox?: readonly MiddleboxEvent[];
+  /** This record is the group of a group cell: its parts, and what it did in the run. */
+  readonly group?: RunGroup;
+  /** These are the lines of a target that writes them (`Target.lines`), from the start to the end of the run. */
+  readonly server?: readonly ProbeEvent[];
+}
+/** A Sentinel event is one line of the observer of a group: an event that a Sentinel published, on the clock of the VM. */
+interface SentinelEvent {
+  readonly ev: string;
+  readonly t: number;
+  /** This text is the address and port of the Sentinel. */
+  readonly sentinel?: string;
+  /** This name is the event, for example `+switch-master`. */
+  readonly event?: string;
+  /** This text is the detail of the event, as the Sentinel wrote it. */
+  readonly detail?: string;
+  readonly [field: string]: unknown;
+}
+/** A part is one member or Sentinel of a group, as a run found it: its address and its network namespace. */
+interface RunPart {
+  readonly ip: string;
+  readonly netns: number;
+}
+/** A run group is the group of a run: its parts, the primary before and after, the timeline, and the writes that it kept. */
+interface RunGroup {
+  readonly members: Readonly<Record<string, RunPart>>;
+  readonly sentinels: Readonly<Record<string, RunPart>>;
+  /** This name is the member that was the primary at the start of the run. */
+  readonly primary: string;
+  /** This name is the part that got the fault, or null for a fault on the whole group. */
+  readonly at: string | null;
+  /** These are the lines of the observer of the Sentinels from the start to the end of the run. */
+  readonly timeline: readonly SentinelEvent[];
+  /** This name is the member that was the primary when the group converged after the run, or null when it did not converge. */
+  readonly after: string | null;
+  /** These are the sequence numbers that the group holds after the run, when it converged, or null. */
+  readonly holds: readonly number[] | null;
 }
 /**
  * A run topology tells where each socket of a run is, in a topology:
@@ -433,6 +662,91 @@ interface RunTopology {
   }>>;
 }
 //#endregion
+//#region src/timeline.d.ts
+/**
+ * This module measures a run of a group cell. It gives the first write that a primary accepted
+ * after the fault, and the timeline of the group as its Sentinels announced it. It also gives what
+ * became of each write.
+ */
+/** These are the stages of a timeline, in their sequence after a fault. */
+export declare const STAGES: readonly Stage[];
+/** An error on the way is one kind of error that the client met after the fault. It has its first time from the fault, and its count. */
+interface ErrorOnTheWay {
+  readonly name: string;
+  readonly first: Quantity;
+  readonly count: number;
+}
+/**
+ * A writes record tells what became of the writes of a run, by their sequence numbers:
+ * - `old` counts the writes that the primary of the start of the run accepted after the fault.
+ * - `lost` are the writes that a primary accepted and the group did not keep. They are the cost
+ *   of a split brain. It is null when the group did not converge after the run.
+ * - `unknown` are the writes that the client sent and did not see accepted: an error, or no answer.
+ *   `held` tells if the group kept each one, or null when it did not converge.
+ */
+interface Writes {
+  readonly sent: number;
+  readonly acked: number;
+  readonly old: number;
+  readonly lost: readonly number[] | null;
+  readonly unknown: ReadonlyArray<{
+    readonly n: number;
+    readonly held: boolean | null;
+  }>;
+}
+/** A group measurement is what one run of a group cell measured, beside its time. */
+interface GroupMeasured {
+  /** These are the member that was the primary at the start, and the member of the first accepted write after the fault. */
+  readonly from: string;
+  readonly to: string | null;
+  /** These are the times of each stage from the fault: the first announcement by any Sentinel. A stage that no Sentinel announced is absent. */
+  readonly stages: Partial<Record<Stage, Quantity>>;
+  readonly writes: Writes;
+  readonly errors: readonly ErrorOnTheWay[];
+  /** These are the delays of the stamps of the observer after the writes of the Sentinels, for a calibration. */
+  readonly lags?: readonly Quantity[];
+}
+/** This function gives the time of each stage of a timeline from an instant: the first announcement of each, after the instant. */
+export declare const stagesOf: (timeline: readonly SentinelEvent[], from: number) => Partial<Record<Stage, Quantity>>;
+/** This function gives what became of each write of a run, from the lines of the client and the keys that the group kept. */
+export declare const writesOf: (run: Run, after: number, primary: string) => Writes;
+/** This function gives the errors that the client met after an instant: each kind, its first time, and its count. */
+export declare const errorsOf: (events: readonly ProbeEvent[], after: number) => ErrorOnTheWay[];
+/**
+ * This function gives the delay of each stamp of the observer after the write of its Sentinel that
+ * the observer read. That write is the latest write of the Sentinel's socket toward the observer,
+ * at the stamp or before it. The prober records each write (`tx`) on the clock of the stamps.
+ */
+export declare const lagsOf: (run: Run, kernel: readonly KernelRecord[]) => Quantity[];
+/** This type is the measurement of observe.ts, which a group run fills. */
+interface GroupReading {
+  readonly outcome: {
+    readonly kind: "answered";
+  } | {
+    readonly kind: "silent";
+  };
+  readonly elapsed: Quantity | null;
+  readonly socket: Quantity | null;
+  readonly noticed: Quantity | null;
+  readonly void: string | null;
+  readonly finding: string | null;
+  readonly group: GroupMeasured;
+}
+/**
+ * This function reads one run of a group cell.
+ * - The reference is the start of the commit of the fault, as the injector stamped it. For a
+ *   stall, it is the first write into the fault, as for the stall of a direct pair.
+ * - The outcome is the first write that a primary accepted after the fault. The prediction says
+ *   which primary: a new one, or the same one after the lift.
+ * - Its time is the last segment with data that the client's socket to that primary received
+ *   before the client saw the acceptance. The stamp of the client is the fallback.
+ * - A run is void when the prober lost records, or when the client did not become ready. It is
+ *   also void when a Sentinel entered its TILT mode, or when the election split its votes. A band
+ *   from the fault whose commit took longer than δ_abs is void too. So is a band whose hold lifted
+ *   more than δ_abs late.
+ */
+export declare const measureGroup: (cell: Cell, run: Run) => GroupReading;
+//#endregion
 //#region src/observe.d.ts
 /**
  * An outcome tells how the wait of a run ended. The socket failed with an errno, or it read the
@@ -447,7 +761,21 @@ type Outcome = {
   readonly kind: "silent";
 } | {
   readonly kind: "answered";
+} | {
+  readonly kind: "closed";
 };
+/**
+ * A request reading is what the second request of an HTTP client met, from the lines of the
+ * client. It holds the connection of its first try, a new try, and its outcome. `gap` is the time
+ * from the end of the first reply to the send of the second request.
+ */
+interface RequestReading {
+  readonly connection: "reused" | "new";
+  readonly retried: boolean;
+  readonly error: string | null;
+  readonly pending: boolean;
+  readonly gap: Quantity | null;
+}
 /** A measurement is what one run measured, against the prediction of its cell. */
 interface Measured {
   readonly environment: number;
@@ -494,6 +822,10 @@ interface Measured {
   readonly reordering?: Reordering;
   /** These are the times of the proxy of a calibration cell, one for each request and its reply. */
   readonly crossings?: Crossings;
+  /** This record is what a run of a group cell measured: the stages of its timeline, and what became of each write. */
+  readonly group?: GroupMeasured;
+  /** This record is what the second request of an HTTP client met, for a cell whose prediction says it. */
+  readonly request?: RequestReading | null;
   /** This text is the reason that the run cannot count, or null. A void run is not a pass and not a finding. */
   readonly void: string | null;
   /** This text tells what in the run disagrees with the prediction, other than its timing, or null. */
@@ -566,10 +898,28 @@ export declare const crossingsOf: (run: Run, hop: string, toClient: SocketPlace
  *   it, and before the first keepalive probe. A live peer answers that probe, and the answer
  *   starts the idle time again. A run where the fault was late is void. A run whose records the
  *   prober lost is also void.
+ * - For an HTTP client, it also reads what the second request met (`requestOf`). An outcome other
+ *   than the one that the prediction gives is a finding.
  */
 export declare const measure: (cell: Cell, run: Run) => Measured;
+/**
+ * This function reads the second request of an HTTP client from its lines. Its first try is its
+ * first `request` line, and each `retry` line is a new try. Its end is its `error` line, or its
+ * `replied` line. A request with neither is pending. It gives null when the client wrote no try.
+ *
+ * A new try that gets no connection writes no `retry` line. Its dial writes a `dial-error` line.
+ * Thus a `dial-error` line after the first try and before the end is also a new try.
+ */
+export declare const requestOf: (events: readonly ProbeEvent[]) => RequestReading | null;
 //#endregion
 //#region src/verdict.d.ts
+/** This function gives the words of the outcome of a second request: its connection, a retry, and its end. */
+export declare const requestWords: (outcome: {
+  readonly connection: "reused" | "new";
+  readonly retried: boolean;
+  readonly error: string | null;
+  readonly pending?: boolean | undefined;
+}) => string;
 /**
  * This amount is the noise of the pipeline itself, δ_abs, as the calibration measured it. It is
  * the 99.9th percentile of the whole latency of a reset, at the concurrency cap or less. Its
@@ -623,8 +973,65 @@ interface Verdict {
     readonly up: Spread | null;
     readonly down: Spread | null;
   };
+  /** This record is what the group of a group cell did over the counted runs: its stages, and its writes. */
+  readonly group?: GroupVerdict;
+  /**
+   * These are the outcomes of the second request of an HTTP client over the counted runs, each
+   * with its count: for example "reused, a retry, a reply". An outcome other than the predicted
+   * one is a finding of its run.
+   */
+  readonly requests?: Readonly<Record<string, number>>;
   readonly why: string;
 }
+/**
+ * A stage verdict is the test of one stage of the timeline of a group against its band. The band
+ * is [T_lo − δ_abs, T_hi + the error of the stamps + δ_abs]. A run that did not announce the stage
+ * is outside too.
+ */
+interface StageVerdict {
+  readonly stage: Stage;
+  readonly band: Interval;
+  readonly median: Quantity | null;
+  readonly outside: ReadonlyArray<{
+    readonly run: string;
+    readonly why: string;
+  }>;
+  readonly decision: "holds" | "fails";
+}
+/**
+ * A group verdict sums what the group of a group cell did in its counted runs:
+ * - `stages` tests each stage that the prediction gives a band.
+ * - `medians` is the median of each stage that a run announced, from the fault.
+ * - `lost` counts the writes that a primary accepted and the group did not keep, in each run.
+ * - `old` counts the writes that the old primary accepted after the fault, in each run.
+ * - `unknown` counts the writes whose outcome the client did not see, and how many the group kept.
+ * - `errors` are the errors on the way: the runs that met each, and the median of its first time.
+ * - `lags` is the spread of the delays of the stamps, for a calibration.
+ */
+interface GroupVerdict {
+  readonly stages: readonly StageVerdict[];
+  readonly medians: Partial<Record<Stage, Quantity>>;
+  readonly lost: {
+    readonly runs: number;
+    readonly total: number;
+    readonly median: number | null;
+    readonly max: number | null;
+  };
+  readonly old: {
+    readonly median: number | null;
+    readonly max: number | null;
+  };
+  readonly unknown: {
+    readonly total: number;
+    readonly held: number;
+  };
+  readonly errors: ReadonlyArray<{
+    readonly name: string;
+    readonly runs: number;
+    readonly first: Quantity;
+  }>;
+  readonly lags?: Spread | null;
+}
 /** This function gives the value at quantile p of sorted values, by nearest rank: x_(⌈p·n⌉). */
 export declare const quantileOf: (sorted: readonly number[], p: number) => number;
 /** This function gives the spread of a set of times, or null for none. */
@@ -648,6 +1055,8 @@ export declare const medianInterval: (sorted: readonly number[]) => [number, num
  * which does not exist at this time.
  */
 export declare const verdictOf: (prediction: Prediction, runs: readonly Measured[]) => Verdict;
+/** This function sums what the group of a group cell did in its counted runs, and tests each stage that has a band. */
+export declare const groupVerdictOf: (prediction: Prediction, counted: readonly Measured[]) => GroupVerdict;
 //#endregion
 //#region src/lab.d.ts
 /** This function gives the image of the harness. */
@@ -656,6 +1065,10 @@ export declare const harnessImage: () => Promise<string>;
 export declare const nodeClientImage: () => Promise<string>;
 /** This function gives the image of the router. */
 export declare const routerImage: () => Promise<string>;
+/** This function gives the image of the Go HTTP client. */
+export declare const goHttpImage: () => Promise<string>;
+/** This function gives the image of the Node HTTP clients. */
+export declare const nodeHttpImage: () => Promise<string>;
 /**
  * This time is the interval of the listing of a router, when its hop does not set one. A listing
  * evicts each expired flow that it walks past, and the DESTROY event of that flow follows.
@@ -717,6 +1130,10 @@ interface Manifest {
     readonly settings: Readonly<Record<string, string>>;
     /** These are the sysctls of its namespace, as read back after the harness set them. */
     readonly sysctls: Readonly<Record<string, string>>;
+    /** These are the files that went into the target, by their path, each with the sha256 of its content. */
+    readonly files?: Readonly<Record<string, string>>;
+    /** This line is the first line of a target that writes them: a server that tells its settings as it reads them. */
+    readonly first?: ProbeEvent;
   };
   readonly harness: {
     readonly image: string;
@@ -738,6 +1155,24 @@ interface Manifest {
   readonly sysctls: Readonly<Record<string, string>>;
   /** These are the hops of a topology, in sequence from the client. */
   readonly hops?: readonly ManifestHop[];
+  /** These are the parts of a group, with their settings as each part read them back. */
+  readonly group?: {
+    readonly members: ReadonlyArray<{
+      readonly name: string;
+      readonly ip: string;
+      readonly id: string;
+      readonly settings: Readonly<Record<string, string>>;
+    }>;
+    readonly sentinels: ReadonlyArray<{
+      readonly name: string;
+      readonly ip: string;
+      readonly id: string;
+      readonly settings: Readonly<Record<string, string>>;
+    }>;
+    readonly observer: {
+      readonly image: string;
+    };
+  };
 }
 /** These are the JSON lines of a program, as they arrive. */
 interface Lines<T extends {
@@ -790,6 +1225,8 @@ interface Lab {
   readonly topology: RunTopology | null;
   /** These are the lines of the routers of a topology, in sequence, each with the name of its hop. A direct pair has none. */
   readonly middlebox: readonly MiddleboxEvent[];
+  /** These are the lines of a target that writes them (`Target.lines`), in sequence. Another target has none. */
+  readonly server: readonly ProbeEvent[];
   /** This method adds a toxic to the Toxiproxy hop of a topology, or removes it with null. The stamps of the injector bracket it. */
   toxic(toxic: Toxic | null): Promise<{
     t0: number;
@@ -919,6 +1356,141 @@ export declare const deviceOf: (route: string) => string;
  */
 export declare const openLab: (target: Target, kind?: Client | ClientImage, topology?: Topology) => Promise<Lab>;
 //#endregion
+//#region src/group.d.ts
+/**
+ * This module is the lab of a replicated group: a primary and its replicas, and the Sentinels
+ * that decide which member is the primary. It opens one environment for a group cell, and it does
+ * the runs of the cell in it. Each member and each Sentinel is a container from the stock image of
+ * the group, in its own network namespace. An injector sidecar shares its network and process
+ * namespaces. The client of each run is ioredis in Sentinel mode. An observer subscribes to the
+ * events of each Sentinel, and the prober watches the namespace of the client.
+ */
+/** A stamp is the time of the injector around one commit, on CLOCK_MONOTONIC in nanoseconds. */
+interface Stamp {
+  readonly t0: number;
+  readonly t1: number;
+}
+/** A group network fault is a black hole between a part and its peers, or a reset of each segment from them. */
+type GroupNetworkFault = "blackhole" | "rst";
+/** A group client is the client container of one run of a group cell. */
+interface GroupClient {
+  readonly events: readonly ProbeEvent[];
+  until(kinds: readonly string[], within: number): Promise<ProbeEvent>;
+  stop(): Promise<void>;
+}
+/** A group lab is one environment of a group cell. */
+interface GroupLab {
+  readonly group: Group;
+  readonly clientIp: string;
+  readonly observerIp: string;
+  readonly members: Readonly<Record<string, RunPart>>;
+  readonly sentinels: Readonly<Record<string, RunPart>>;
+  /** These are the namespaces that the prober watches: the client, and the Sentinels for a calibration. */
+  readonly netns: {
+    readonly client: number;
+    readonly watched: readonly number[];
+  };
+  /** These are all the records that the prober wrote, in sequence. */
+  readonly kernel: readonly KernelRecord[];
+  /** These are all the lines of the observer of the Sentinels, in sequence. */
+  readonly timeline: readonly SentinelEvent[];
+  /** This method sets the sysctls of the client namespace through its privileged sidecar, and reads them back. */
+  sysctl(values: Readonly<Record<string, string>>): Promise<Record<string, string>>;
+  /** This method waits until the group is whole and calm, and gives the name of its primary. */
+  converge(within?: number): Promise<string>;
+  /** This method removes each key from the group, and waits until each replica has the removal. */
+  flush(primary: string): Promise<void>;
+  /** This method makes each Sentinel forget the state of the group (`SENTINEL RESET`): an aborted failover then holds off no later failover. */
+  reset(): Promise<void>;
+  /** This method gives the sequence numbers that the group holds, from its primary. */
+  holds(primary: string): Promise<number[]>;
+  /** This method gives the part that has a role in the group, when `primary` is the primary. */
+  partOf(role: GroupRole, primary: string): string;
+  /** This method starts the client of a run, with the arguments of `client-sentinel.mjs`. */
+  client(args: readonly string[]): Promise<GroupClient>;
+  /** This method applies a network fault between a part and some of its peers, at once or at an instant of the VM. */
+  fault(kind: GroupNetworkFault, part: string, cuts: NonNullable<GroupPlace["cuts"]>, at?: number): Promise<Stamp>;
+  /** This method removes a network fault, at once or at an instant of the VM. */
+  lift(kind: GroupNetworkFault, part: string, cuts: NonNullable<GroupPlace["cuts"]>, at?: number): Promise<Stamp>;
+  /** This method sends a signal to the server process of a part, at once or at an instant of the VM. */
+  signal(part: string, signal: "KILL" | "STOP" | "CONT", at?: number): Promise<Stamp>;
+  /** This method starts the server of a member again, after a fin. */
+  restart(member: string): Promise<void>;
+  /** This method asks the first Sentinel for a failover of the group. */
+  failover(coordinated: boolean): Promise<Stamp>;
+  /** This method removes the TCP metrics of the client namespace: each run starts from a kernel that keeps nothing. */
+  forget(): Promise<void>;
+  /** This method waits for the next clock line of the prober, which gives the count of lost records. */
+  settle(): Promise<number | null>;
+  manifest(sysctls: Readonly<Record<string, string>>): Promise<Manifest>;
+  close(): Promise<void>;
+}
+/** This function gives the configuration file of a member: its port, its settings, and the primary that a replica follows. */
+export declare const memberConfig: (group: Group, primary: string | null) => string;
+/** This function gives the configuration file of a Sentinel: its port, the group that it monitors, and the settings of the group. */
+export declare const sentinelConfig: (group: Group, primary: string) => string;
+/**
+ * This function gives the peers that a fault on a part cuts it from, by their addresses. `all` is
+ * each other part and the client, `group` is the other parts, and `clients` is the client. The
+ * observer is not a peer, thus the timeline of the group stays whole.
+ */
+export declare const peersOf: (parts: {
+  readonly members: Readonly<Record<string, RunPart>>;
+  readonly sentinels: Readonly<Record<string, RunPart>>;
+}, client: string, part: string, cuts: NonNullable<GroupPlace["cuts"]>) => string[];
+/** This function reads the flat reply of a command line client into its fields: name, value, name, value. */
+export declare const fieldsOf: (text: string) => Array<Record<string, string>>;
+/** This function reads the INFO of a server into its fields. */
+export declare const infoOf: (text: string) => Record<string, string>;
+/**
+ * A view of the group is what its parts tell at one time. `healthy` tells if each Sentinel agrees
+ * on one primary with no flag but `master`, and knows each replica and each other Sentinel. Each
+ * replica must follow that primary with its link up, and the primary must have each replica online,
+ * with no failover in progress.
+ */
+interface GroupView {
+  readonly primary: string | null;
+  readonly healthy: boolean;
+  readonly why: string;
+}
+/** This function judges the views of the Sentinels and of the members: the primary that they agree on, if the group is healthy. */
+export declare const viewOf: (group: Group, ips: Readonly<Record<string, string>>, sentinels: ReadonlyArray<{
+  readonly master: Record<string, string>;
+  readonly replicas: ReadonlyArray<Record<string, string>>;
+}>, members: Readonly<Record<string, Record<string, string>>>) => GroupView;
+/** This time is the longest that a group takes to converge before a run, or after it. */
+export declare const CONVERGE_MS = 24e4;
+/**
+ * This function opens one environment of a group cell:
+ * - one new network, with each member, each Sentinel, the client and the observer on it.
+ * - each member: the stock image, its server as the child of a shell, thus a fin can stop the
+ *   server and the container stays up. The first member is the primary, and the others follow it.
+ * - each Sentinel: the stock image, its Sentinel as the child of a shell, which monitors the first member.
+ * - an injector sidecar for each part, in its network and process namespaces.
+ * - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
+ * - the observer of the Sentinels, in a namespace of its own.
+ * - the prober, which watches the namespace of the client, and of each Sentinel for a calibration.
+ */
+export declare const openGroupLab: (group: Group, options?: {
+  readonly watchSentinels?: boolean;
+}) => Promise<GroupLab>;
+/** This function gives the latest time that a group cell waits for its outcome, in ms from the fault. */
+export declare const deadlineOf: (cell: Cell) => number;
+/** This function gives the latest subscription of the observer to each Sentinel, in some lines of the observer. */
+export declare const subscriptionsOf: (timeline: readonly SentinelEvent[]) => SentinelEvent[];
+/**
+ * This function does one run of a group cell:
+ * - The group converges, and its keys go. The client connects through the Sentinels, and its
+ *   stream of writes gets steady.
+ * - The fault occurs on its part, as the primary of this run makes the roles. A fault with a
+ *   hold lifts at its instant in the VM. A stall lifts at its length after the first write into it.
+ * - The run waits for its outcome: the first write that a new primary accepts, or the first one
+ *   after the lift on the same primary. Then it continues for a moment.
+ * - Each fault goes, the client stops, and the group converges again. The keys that it then holds
+ *   tell which writes it kept.
+ */
+export declare const groupOnce: (lab: GroupLab, cell: Cell, environment: number, index: number) => Promise<Run>;
+//#endregion
 //#region src/run.d.ts
 /** A cell result is a cell with its manifests, its runs, their measurements and its verdict. */
 interface CellResult {
@@ -944,6 +1516,16 @@ export declare const judge: (cell: Cell, runs: readonly Run[]) => {
   measured: Measured[];
   verdict: Verdict;
 };
+/**
+ * This time is the pause between the client's first reply and the fault. It is more than Linux's
+ * TCP_DELACK_MAX of 200 ms. Thus the delayed ACK of the reply is out before the fault.
+ */
+export declare const QUIET_MS = 250;
+/**
+ * This time is how long an HTTP client waits after its second request, before it exits. The exit
+ * closes its sockets, thus that close comes after the window of the cell.
+ */
+export declare const LINGER_MS = 1e3;
 /** This function divides a count of runs across environments, as equally as possible. */
 export declare const spread: (runs: number, environments: number) => number[];
 /**
@@ -957,5 +1539,5 @@ export declare const spread: (runs: number, environments: number) => number[];
  */
 export declare const runCell: (cell: Cell, say?: (line: string) => void) => Promise<CellResult>;
 //#endregion
-export type { Agent, Cell, CellResult, Client, ClientImage, Crossings, Decision, Fault, FaultPlace, FaultSide, Hop, Interval, KernelRecord, Lab, Manifest, ManifestHop, Measured, MiddleboxEvent, MiddleboxForm, NetworkFault, Observer, Outcome, Prediction, Probe, ProbeEvent, ProxyHop, ProxyTime, Recovers, Reference, Reordering, Replies, RouterHop, RttRange, Run, RunTopology, SocketPlace, Spread, Target, Topology, Toxic, Verdict };
+export type { Agent, Cell, CellResult, Client, ClientImage, Crossings, Decision, ErrorOnTheWay, Fault, FaultPlace, FaultSide, Group, GroupClient, GroupLab, GroupMeasured, GroupNetworkFault, GroupPlace, GroupPrediction, GroupReading, GroupRole, GroupVerdict, GroupView, Hop, HopPlace, HttpProbe, Interval, KernelRecord, Lab, Manifest, ManifestHop, Measured, MiddleboxEvent, MiddleboxExpectation, MiddleboxForm, NetworkFault, Observer, Outcome, Prediction, Probe, ProbeEvent, ProxyHop, ProxyTime, Recovers, Reference, Reordering, Replies, RequestPrediction, RequestReading, RouterHop, RttRange, Run, RunGroup, RunPart, RunTopology, SentinelEvent, SocketPlace, Spread, Stage, StageBand, StageVerdict, Stamp, Target, Topology, Toxic, Verdict, Writes };
 //# sourceMappingURL=index.d.ts.map