@mark1russell7/harness 0.2.0 → 0.4.0

package/build/index.d.ts

@@ -1,5 +1,6 @@
 import { Quantity } from "@mark1russell7/quantity";
 import { KernelRecord } from "@mark1russell7/prober";
+import "testcontainers";
 //#region src/cell.d.ts
 /**
  * A fault is a failure that the harness injects, by the name of its manifestation. Each name is
@@ -19,17 +20,110 @@ import { KernelRecord } from "@mark1russell7/prober";
  * - `mtu-blackhole` drops each segment to the observer that is longer than `Cell.mtu` bytes.
  * - `netem` is tc netem on the egress of the target (`Cell.netem`): loss, duplication,
  *   reordering or corruption. Its outcome is a distribution.
+ *
+ * These faults are of a topology (`Cell.topology`):
+ * - `middlebox-forget` injects nothing. The router of the topology forgets an idle flow at its own
+ *   idle timeout, and its form tells what a late packet then meets.
+ * - `proxy-idle` injects nothing. The proxy of the topology closes an idle connection at its own
+ *   timeout.
+ * - `toxic` adds a toxic of Toxiproxy (`Cell.toxic`) through its HTTP API.
+ * - `server-close` makes the server close the connection of its peer, with the command of the
+ *   target (`Target.close`): `CLIENT KILL` in Redis.
+ * - `none` injects nothing. A calibration cell measures the path itself: for example the time of a
+ *   proxy between its two sockets (`Cell.calibrates`).
+ *
+ * A fault at a router of the topology (`Cell.faultAt`) acts on what the router forwards toward
+ * one side. It is a black hole, a reset or an ICMP unreachable. It can also be a simplex drop, an
+ * MTU drop or netem.
+ */
+type Fault = "blackhole" | "freeze-process" | "rst" | "stall" | "fin" | "unreachable" | "simplex" | "mtu-blackhole" | "netem" | "middlebox-forget" | "proxy-idle" | "toxic" | "server-close" | "none";
+/**
+ * A middlebox form tells what a router does with a flow that it forgot, when a late packet comes:
+ * - `reset-both` sends a reset to the two ends when it forgets the flow.
+ * - `reset-next` answers the next packet from inside with a reset to its sender, and drops a late
+ *   packet from outside.
+ * - `silent-drop` drops each late packet, in each direction.
+ * - `recreate` lets a late packet from inside through, and the flow starts again. It drops a late
+ *   packet from outside.
+ * - `none` is a plain NAT router. A late packet from inside starts the flow again, and the stack of
+ *   the router answers a late packet from outside with a reset.
+ */
+type MiddleboxForm = "reset-both" | "reset-next" | "silent-drop" | "recreate" | "none";
+/**
+ * A router is a hop that forwards and translates the addresses of the side of the client (NAT),
+ * with conntrack. It is the router image of the harness.
+ */
+interface RouterHop {
+  readonly name: string;
+  readonly kind: "router";
+  readonly form: MiddleboxForm;
+  /** This time is the idle timeout of an established flow: nf_conntrack_tcp_timeout_established, in whole seconds. */
+  readonly idle: Quantity;
+  /**
+   * This time is the interval of the listing that evicts each expired flow. Without it, the
+   * interval is 0.1 s. The DESTROY event of an expired flow waits for that listing.
+   */
+  readonly pump?: Quantity;
+  /**
+   * These are the sysctls of the namespace of the router, which the router sets itself: it is
+   * privileged. An example is `net.ipv4.icmp_ratelimit` for the ICMP errors of a fault at the router.
+   */
+  readonly sysctls?: Readonly<Record<string, string>>;
+}
+/**
+ * A proxy is a hop that ends each TCP connection from the side of the client and opens its own to
+ * the next hop. It is a stock image, pinned by digest.
  */
-type Fault = "blackhole" | "freeze-process" | "rst" | "stall" | "fin" | "unreachable" | "simplex" | "mtu-blackhole" | "netem";
+interface ProxyHop {
+  readonly name: string;
+  readonly kind: "proxy";
+  readonly image: string;
+  /** This port is the port that the proxy listens on for the side of the client. */
+  readonly port: number;
+  readonly command?: readonly string[];
+  /** These files go into the proxy before it starts, by their path. In their content, {{next}} is the address and port of the next hop. */
+  readonly files?: Readonly<Record<string, string>>;
+  /** This text is the log line that tells that the proxy serves. */
+  readonly ready?: string;
+  /** This record is the HTTP API of Toxiproxy: its port, and the name of the proxy that gets the toxics. */
+  readonly api?: {
+    readonly port: number;
+    readonly proxy: string;
+  };
+  /** These are the sysctls of the namespace of the proxy. A privileged sidecar sets them before each run. */
+  readonly sysctls?: Readonly<Record<string, string>>;
+  /** These are the settings of the proxy, for the manifest: for example its idle timeouts. */
+  readonly settings?: Readonly<Record<string, string>>;
+}
+/**
+ * A hop is one box between the client and the target. Each hop is on two networks: the network
+ * of the side of the client, and the network of the side of the target.
+ */
+type Hop = RouterHop | ProxyHop;
+/**
+ * A topology is the chain of hops between the client and the target, in sequence from the client.
+ * Without hops, the client and the target are a direct pair.
+ */
+interface Topology {
+  readonly name: string;
+  readonly hops: readonly Hop[];
+}
+/** A toxic is one toxic of Toxiproxy, as its API takes it. Its toxicity is 1. */
+interface Toxic {
+  readonly type: "latency" | "timeout" | "reset_peer" | "slow_close" | "bandwidth" | "limit_data" | "slicer";
+  readonly stream: "downstream" | "upstream";
+  readonly attributes: Readonly<Record<string, number>>;
+}
 /**
  * A reference is the event that the time of a detector counts from: the event that arms the
  * timer in its machine document. Keepalive counts from the last received segment. The give-up of
  * the retransmissions counts from the first retransmission of the RTO timer, not from the tail
  * loss probe one RTO before it. The request itself meets a reset, thus a reset counts from the
  * send of the client. A FIN counts from the change of state of the socket of the peer when its
- * process dies (`peer-closed`).
+ * process dies (`peer-closed`). A toxic of Toxiproxy counts from the start of its commit, as the
+ * injector stamps it (`fault`).
  */
-type Reference = "last-received" | "first-retransmission" | "request-sent" | "peer-closed";
+type Reference = "last-received" | "first-retransmission" | "request-sent" | "peer-closed" | "fault";
 /**
  * This prediction tells when a connection moves again after a stall. That occurs at the first
  * firing of its retransmission timer after the stall stops, and its reply comes back one round
@@ -52,14 +146,50 @@ interface Recovers {
     readonly high: Quantity;
   };
 }
+/**
+ * This prediction tells when the reply to the request comes back, from the send of the request.
+ * For example, the reply comes after the latency of a toxic, or after a forgotten flow starts again.
+ */
+interface Replies {
+  readonly low: Quantity;
+  readonly high: Quantity;
+}
+/**
+ * An RTT range is the range of the first round trip of the observer that a band assumes. The
+ * models read a round trip, for example 1 ms. The band at that value holds within the noise for
+ * each round trip in the range. A run whose first round trip is above the range is void.
+ */
+interface RttRange {
+  readonly low: Quantity;
+  readonly high: Quantity;
+}
+/**
+ * A proxy time is the time of one proxy in one direction that a band includes. It is an upper
+ * quantile of the times that a calibration cell of the proxy measured. Each time goes from an
+ * arrival on one socket of the proxy to its write on the other. `up` goes toward the target, and
+ * `down` toward the client.
+ */
+interface ProxyTime {
+  readonly hop: string;
+  readonly direction: "up" | "down";
+  /** This fraction is the quantile of the calibration, for example 0.99. */
+  readonly quantile: number;
+  readonly time: Quantity;
+}
 /**
  * A prediction is what a cell predicts. Either a detection occurs in an interval [T_lo, T_hi]
  * after the reference event, or nothing detects the fault in a window. Nothing detects a stall
  * that is shorter than the give-up, and the connection recovers from it. The prediction then also
- * tells when.
+ * tells when. A silent prediction can also tell when the reply to the request comes back
+ * (`replies`).
+ *
+ * A band can name the first round trip that it assumes (`rtt`), and the proxy times that it
+ * includes (`through`).
  *
  * A cell whose outcome is a distribution, for example under netem loss, has no band. Its
- * prediction is `detects: null` with the reason, and its runs are measurements only.
+ * prediction is `detects: null` with the reason, and its runs are measurements only. A
+ * measurement-only cell that names a reference (`from`) also records the time from it to the end
+ * of the connection, or to the last data.
  */
 type Prediction = {
   readonly detects: true;
@@ -70,16 +200,22 @@ type Prediction = {
   readonly from: Reference;
   /** This time is the closed form, less the most that the timer can fire early. */
   readonly low: Quantity;
-  /** This time is the closed form, plus the most that the timer can fire late. */
+  /** This time is the closed form, plus the most that the timer can fire late, plus each proxy time. */
   readonly high: Quantity;
+  readonly rtt?: RttRange;
+  readonly through?: readonly ProxyTime[];
 } | {
   readonly detects: false;
   readonly window: Quantity;
   readonly recovers?: Recovers;
+  readonly replies?: Replies;
+  readonly rtt?: RttRange;
+  readonly through?: readonly ProxyTime[];
 } | {
   readonly detects: null;
   readonly window: Quantity;
   readonly why: string;
+  readonly from?: Reference;
 };
 /** A target is the server that a cell connects to: a stock image. */
 interface Target {
@@ -104,16 +240,35 @@ interface Target {
   readonly command?: readonly string[];
   /** This command stops the server process in the target with SIGKILL, as argv, for a `fin` fault. */
   readonly kill?: readonly string[];
+  /** This command makes the server close the connection of each client, as argv, for a `server-close` fault. */
+  readonly close?: readonly string[];
 }
 /** A client is what the runs of a cell use: the TCP client of the harness, or a driver on its socket. */
 type Client = "tcp" | "ioredis";
+/**
+ * A client image is a process that a lab starts as the client of each run, in the network
+ * namespace of the client. An example is an agent of an application on Node. Its command gets the
+ * arguments of the TCP client of the harness. These are the address and the port of the target, then
+ * the options of the cell. It writes JSON lines with an `ev` and a `t` (CLOCK_MONOTONIC in
+ * nanoseconds) on its standard output, as the client of the harness does.
+ */
+interface ClientImage {
+  readonly image: string;
+  readonly command: readonly string[];
+}
 /**
  * An observer is the side whose detection a cell measures. The client is one observer. The
- * server is the other observer. It is the hold of the target on the connection of the client, at
+ * server is another observer. It is the hold of the target on the connection of the client, at
  * its server socket in its namespace. The fault is on the peer of the observer. Thus
  * freeze-process freezes the target for the client, and the client for the server.
+ *
+ * A proxy of a topology is the third observer: its socket toward the client, or its socket toward
+ * the target.
  */
-type Observer = "client" | "server";
+type Observer = "client" | "server" | {
+  readonly hop: string;
+  readonly toward: "client" | "target";
+};
 /** A cell is one cell of the matrix: a topology, its settings, a workload, a fault and the prediction. */
 interface Cell {
   readonly name: string;
@@ -159,7 +314,10 @@ interface Cell {
   readonly drops?: "to-server" | "to-client";
   /** This length is the longest segment, in bytes, that an `mtu-blackhole` fault lets through to the observer. */
   readonly mtu?: number;
-  /** This specification is the tc netem of a `netem` fault, on the egress of the target: for example "loss 5%". */
+  /**
+   * This specification is the tc netem of a `netem` fault, for example "loss 5%". It is on the
+   * egress of the target, or at a hop (`faultAt`) on the egress of its device toward one side.
+   */
   readonly netem?: string;
   /**
    * This delay goes on the egress of the target before the client connects, with tc netem. Thus
@@ -175,6 +333,22 @@ interface Cell {
   readonly targetSettings?: Readonly<Record<string, string>>;
   /** These are the sysctls of the network namespace of the target, for a cell on the server side: its tcp_retries2. */
   readonly targetSysctls?: Readonly<Record<string, string>>;
+  /** This topology puts hops between the client and the target. Without it, the client and the target are a direct pair. */
+  readonly topology?: Topology;
+  /** This place is where the fault goes, when it is not the peer of the observer: for example one side of a hop. */
+  readonly faultAt?: FaultPlace;
+  /** This time is how long an in-flight client stays idle after the fault holds, before it sends its request. */
+  readonly idleBefore?: Quantity;
+  /** This toxic is the toxic of a `toxic` fault, which the Toxiproxy hop gets. */
+  readonly toxic?: Toxic;
+  /** This image replaces the TCP client of the harness for each run. */
+  readonly client?: ClientImage;
+  /**
+   * This name is the proxy hop whose own time a calibration cell measures. The time of a request
+   * (`up`) starts at its arrival on the socket of the proxy toward the client. It stops at the
+   * write on the socket toward the target. The time of its reply (`down`) goes the other way.
+   */
+  readonly calibrates?: string;
   /** This count is the number of runs of the cell (11), across new environments (two at least). */
   readonly runs: number;
   readonly environments: number;
@@ -185,6 +359,22 @@ interface Cell {
    */
   readonly concurrency?: number;
 }
+/**
+ * A fault place is the namespace that gets the rules of a fault. "target" and "client" are the two
+ * ends. A hop place is a hop, with the side that the rules face. That side is the next hop toward
+ * the client, or the next hop toward the target. The rules of a proxy act on its sockets. The
+ * rules of a router act on what it forwards, by its device toward that side: the FORWARD chain.
+ */
+type FaultPlace = "target" | "client" | {
+  readonly hop: string;
+  readonly toward: "client" | "target";
+};
+/** A middlebox event is one line of the router: a change of a conntrack entry, or a reset that it sent. `t` is CLOCK_MONOTONIC in nanoseconds. */
+interface MiddleboxEvent {
+  readonly ev: string;
+  readonly t: number;
+  readonly [field: string]: unknown;
+}
 /** A probe event is one line that the client wrote. `t` is CLOCK_MONOTONIC in nanoseconds: an instant, not an amount. */
 interface ProbeEvent {
   readonly ev: string;
@@ -195,6 +385,11 @@ interface ProbeEvent {
 interface Run {
   readonly environment: number;
   readonly index: number;
+  /**
+   * This flag marks the warm-up of an environment. It occurs before the counted runs, and the
+   * verdict does not count it. On a loaded machine, a new environment is sometimes slow at first.
+   */
+  readonly warmup?: true;
   readonly events: readonly ProbeEvent[];
   /** This bracket is the time of the injector around the commit of the fault, on the same clock. It is null for a freeze. */
   readonly fault: {
@@ -217,6 +412,25 @@ interface Run {
    * the run void. Null also makes the run void: no clock line came after the run to tell the count.
    */
   readonly lost: number | null;
+  /** These are the addresses and the namespaces of a topology, which the measurement of a run reads. */
+  readonly topology?: RunTopology;
+  /** These are the lines of the routers of a topology from the start to the end of the run. */
+  readonly middlebox?: readonly MiddleboxEvent[];
+}
+/**
+ * A run topology tells where each socket of a run is, in a topology:
+ * - `targetPeer` is the address of the peer of the target, as the target sees it: the address of
+ *   the last hop.
+ * - `hops` are the network namespaces of the hops, by their names, with the address of the peer of
+ *   each hop on each side.
+ */
+interface RunTopology {
+  readonly targetPeer: string;
+  readonly hops: Readonly<Record<string, {
+    readonly netns: number;
+    readonly client: string;
+    readonly target: string;
+  }>>;
 }
 //#endregion
 //#region src/observe.d.ts
@@ -264,14 +478,90 @@ interface Measured {
   readonly retransmissions: number;
   /** This count is the number of segments that the socket received after the request. */
   readonly acks: number;
+  /** This count is the number of bytes that the client read after the request. */
+  readonly bytes?: number;
+  /**
+   * This time is when a router of the topology forgot the flow of the client. It counts from the
+   * last segment that the client received before the fault. It is null when no router forgot the flow.
+   */
+  readonly forgot?: Quantity | null;
+  /**
+   * This time is the first round trip of the socket of the observer. It is the smoothed round trip
+   * in its first record, which is the sample of the handshake. It is null when no record has it.
+   */
+  readonly rtt?: Quantity | null;
+  /** This record tells what reordering did in a run of a measurement-only cell, from the records of its two ends. */
+  readonly reordering?: Reordering;
+  /** These are the times of the proxy of a calibration cell, one for each request and its reply. */
+  readonly crossings?: Crossings;
   /** This text is the reason that the run cannot count, or null. A void run is not a pass and not a finding. */
   readonly void: string | null;
   /** This text tells what in the run disagrees with the prediction, other than its timing, or null. */
   readonly finding: string | null;
 }
+/**
+ * A reordering record is what reordering, duplicates and losses did to the data of a run. The
+ * receiver is the client, and the sender is the socket of its peer.
+ * - `outOfOrder` counts the arrivals at the client beyond a hole, and `duplicates` counts the
+ *   arrivals whose data the client had. The prober reads the two from each arrival, against `rcv_nxt`.
+ * - An arrival can hold more than one segment. The kernel joins the segments that wait in the
+ *   backlog of a socket while its owner holds it.
+ * - `ooo` is the count of the kernel of the client in segments (`rcv_ooopack`).
+ * - `sacked` and `lost` are the most segments that the sender held as SACKed and as lost.
+ * - `dsacks` is the DSACK blocks that the sender received, and `fast` its fast retransmissions.
+ * - `degree` is the highest reordering degree of the sender (`tp->reordering`), and `seen` the
+ *   reorderings that it detected (`reord_seen`).
+ *
+ * The counts of the kernel are null for a record of the prober before version 0.2.
+ */
+interface Reordering {
+  readonly outOfOrder: number;
+  readonly duplicates: number;
+  readonly ooo: number | null;
+  readonly sacked: number | null;
+  readonly lost: number | null;
+  readonly dsacks: number | null;
+  readonly fast: number;
+  readonly degree: number | null;
+  readonly seen: number | null;
+}
+/**
+ * A crossings record is the time of a proxy for each request (`up`) and for each reply (`down`).
+ * The time of a request starts at its arrival on the socket toward the client. It stops at the
+ * write on the socket toward the target. The time of a reply goes the other way. A write is a
+ * `tx` record of the prober.
+ */
+interface Crossings {
+  readonly hop: string;
+  readonly up: readonly Quantity[];
+  readonly down: readonly Quantity[];
+}
+/** A socket place is a socket of a run: its namespace and its cookie. */
+interface SocketPlace {
+  readonly netns: number;
+  readonly cookie: number;
+}
+/**
+ * This function finds the socket of the observer of a cell in a run:
+ * - the client socket, by the address that it connected from, in its namespace.
+ * - the socket of the target for the same connection, in its namespace.
+ * - a socket of a proxy of the topology, toward the client or toward the target.
+ */
+export declare const socketOf: (cell: Cell, run: Run) => SocketPlace | undefined;
+/** This function gives what reordering did in a run, from the records of the receiver and of the sender. */
+export declare const reorderingOf: (receiver: readonly KernelRecord[], sender: readonly KernelRecord[]) => Reordering;
+/**
+ * This function gives the times of a proxy in a run, for each request after the first. The time
+ * of a request starts at its arrival on the socket of the proxy toward the client. It stops at the
+ * next write on the socket toward the target. The time of its reply starts at its arrival on the
+ * socket toward the target, and it stops at the next write toward the client. A request without
+ * each record gives no time.
+ */
+export declare const crossingsOf: (run: Run, hop: string, toClient: SocketPlace | undefined, toTarget: SocketPlace | undefined) => Crossings;
 /**
  * This function reads one run against its cell, from the records of the kernel first.
- * - The socket of the run is the socket that the client connected from, in its namespace.
+ * - The socket of the run is the socket of its observer. It is the socket that the client connected
+ *   from, in its namespace, or the socket of the target or of a proxy for that connection.
  * - The fault must hold before the event that it must meet. That is before the request goes into
  *   it, and before the first keepalive probe. A live peer answers that probe, and the answer
  *   starts the idle time again. A run where the fault was late is void. A run whose records the
@@ -302,6 +592,14 @@ interface Interval {
   readonly low: Quantity;
   readonly high: Quantity;
 }
+/** A spread is the distribution of a set of times: its count, its median and its upper quantiles by nearest rank. */
+interface Spread {
+  readonly count: number;
+  readonly median: Quantity;
+  readonly p99: Quantity;
+  readonly p999: Quantity;
+  readonly max: Quantity;
+}
 /** A verdict is the judgement of a cell on its runs. */
 interface Verdict {
   readonly decision: Decision;
@@ -317,8 +615,20 @@ interface Verdict {
   readonly median: Quantity | null;
   /** This interval is the order-statistic interval of the median at 95 %, or the whole range for fewer than six runs. */
   readonly medianInterval: Interval | null;
+  /** This record is the sum of what reordering did in each counted run of a measurement-only cell. */
+  readonly reordering?: Reordering;
+  /** This record is the time of the proxy of a calibration cell, in each direction. */
+  readonly proxy?: {
+    readonly hop: string;
+    readonly up: Spread | null;
+    readonly down: Spread | null;
+  };
   readonly why: string;
 }
+/** This function gives the value at quantile p of sorted values, by nearest rank: x_(⌈p·n⌉). */
+export declare const quantileOf: (sorted: readonly number[], p: number) => number;
+/** This function gives the spread of a set of times, or null for none. */
+export declare const spreadOf: (times: readonly Quantity[]) => Spread | null;
 /**
  * This function gives the distribution-free interval of a median: [x_(k), x_(n+1−k)], for the
  * largest k whose two tails each hold 2.5 % or less. For eleven runs, it gives [x_(2), x_(10)],
@@ -344,6 +654,20 @@ export declare const verdictOf: (prediction: Prediction, runs: readonly Measured
 export declare const harnessImage: () => Promise<string>;
 /** This function gives the image of the Node client. */
 export declare const nodeClientImage: () => Promise<string>;
+/** This function gives the image of the router. */
+export declare const routerImage: () => Promise<string>;
+/**
+ * This time is the interval of the listing of a router, when its hop does not set one. A listing
+ * evicts each expired flow that it walks past, and the DESTROY event of that flow follows.
+ */
+export declare const PUMP: Quantity;
+/**
+ * This time is the longest listing of the conntrack table of a router that a prediction assumes.
+ * A DESTROY follows an expiry within the interval of the pump and two listings. On 2026-10-15,
+ * the slowest of 500 listings in an idle router took 19.4 ms, and the median took 5.5 ms. A
+ * longer listing writes a `pump-slow` line, and a run with one near its detection is void.
+ */
+export declare const LISTING: Quantity;
 /**
  * A network fault is a fault that the injector applies as a filter rule. It is a black hole (both
  * directions, or one), or a reset on the next segment. It can also be an ICMP unreachable for each
@@ -367,6 +691,17 @@ interface FaultOptions {
  * because it fires only for a retransmission that went out.
  */
 type FaultSide = "target" | "client";
+/** A hop of a manifest is one hop of a topology, as the lab started it. */
+interface ManifestHop {
+  readonly name: string;
+  readonly kind: "router" | "proxy";
+  readonly image: string;
+  readonly id: string;
+  /** These are the settings of the hop: the form and the idle timeout of a router, the files and settings of a proxy. */
+  readonly settings: Readonly<Record<string, string>>;
+  /** These are the sysctls of the namespace of the hop, as read back after the harness set them. */
+  readonly sysctls: Readonly<Record<string, string>>;
+}
 /** A manifest is what a lab records about its machine, its images and its settings. */
 interface Manifest {
   readonly kernel: string;
@@ -388,18 +723,32 @@ interface Manifest {
     readonly id: string;
     readonly packages: string[];
   };
-  /** This record is the client of each run: the TCP client of the harness, or a driver from its own image. */
+  /** This record is the client of each run: the TCP client of the harness, a driver, or an image that the cell gives. */
   readonly client: {
-    readonly kind: Client;
+    readonly kind: Client | "image";
     readonly image: string;
   };
   /** These are the namespaces that the prober watched, by inode. */
   readonly netns: {
     readonly client: number;
     readonly target: number;
+    readonly hops?: Readonly<Record<string, number>>;
   };
   /** These are the sysctls of the client namespace, as read back after the harness set them. */
   readonly sysctls: Readonly<Record<string, string>>;
+  /** These are the hops of a topology, in sequence from the client. */
+  readonly hops?: readonly ManifestHop[];
+}
+/** These are the JSON lines of a program, as they arrive. */
+interface Lines<T extends {
+  readonly ev: string;
+}> {
+  readonly seen: T[];
+  /** These are the other lines that it wrote, for example an error. They explain a program that stopped its output. */
+  readonly other: string[];
+  consume: (stream: NodeJS.ReadableStream) => void;
+  /** This method gives the first line of one of these kinds, or it rejects after `within` ms. */
+  until(kinds: readonly string[], within: number, who: string): Promise<T>;
 }
 /** A probe is the client container of one run. */
 interface Probe {
@@ -415,14 +764,46 @@ interface Probe {
   readonly events: readonly ProbeEvent[];
   stop(): Promise<void>;
 }
-/** A lab is one environment: the target, the client, the injector and the prober. */
+/** An agent is a container that a lab starts in the namespaces of the target, for example a Node process that signals the server. */
+interface Agent {
+  /** These are the JSON lines that it wrote, as they arrive. */
+  readonly lines: Lines<{
+    readonly ev: string;
+  }>;
+  /** This method starts a command in the agent, and gives its standard output. */
+  exec(argv: readonly string[]): Promise<string>;
+  stop(): Promise<void>;
+}
+/** A lab is one environment: the target, the client, the hops of a topology, the injectors and the prober. */
 interface Lab {
+  /** This address is the address that the client connects to: the target, or the first proxy of a topology. */
   readonly targetIp: string;
+  /** This port is the port that the client connects to. */
+  readonly targetPort: number;
   readonly clientIp: string;
   readonly netns: {
     readonly client: number;
     readonly target: number;
+    readonly hops?: Readonly<Record<string, number>>;
   };
+  /** This record is where each socket of a topology is, or null for a direct pair. */
+  readonly topology: RunTopology | null;
+  /** These are the lines of the routers of a topology, in sequence, each with the name of its hop. A direct pair has none. */
+  readonly middlebox: readonly MiddleboxEvent[];
+  /** This method adds a toxic to the Toxiproxy hop of a topology, or removes it with null. The stamps of the injector bracket it. */
+  toxic(toxic: Toxic | null): Promise<{
+    t0: number;
+    t1: number;
+  }>;
+  /** This method makes the target close the connection of each client, with the command of the target. */
+  serverClose(): Promise<void>;
+  /**
+   * This method starts an agent in the network namespace of the target, and in its process
+   * namespace with `pid`. Its command keeps it up, for example `sleep infinity`.
+   */
+  agent(image: string, command: readonly string[], options?: {
+    readonly pid?: boolean;
+  }): Promise<Agent>;
   /** These are all the records that the prober wrote, in sequence. */
   readonly kernel: readonly KernelRecord[];
   /** This method sets the sysctls of the client namespace through its privileged sidecar, and reads them back. */
@@ -432,29 +813,31 @@ interface Lab {
   /** This method sets the sysctls of the target namespace through a privileged sidecar there, and reads them back. */
   targetSysctl(values: Readonly<Record<string, string>>): Promise<Record<string, string>>;
   /**
-   * This method applies a fault between the client and the target, from the namespace of the
-   * target. The fault is a black hole, or a reset that answers the next segment of the client. The stamps of
-   * the injector bracket it.
+   * This method applies a fault between the client and the target, by default from the namespace
+   * of the target. The fault is a black hole, or a reset that answers the next segment of the
+   * client. At a proxy of a topology, the rules face the peer of the proxy on one side. The stamps
+   * of the injector bracket it.
    */
-  fault(kind: NetworkFault, at?: FaultSide, options?: FaultOptions): Promise<{
+  fault(kind: NetworkFault, at?: FaultPlace, options?: FaultOptions): Promise<{
     t0: number;
     t1: number;
   }>;
   /** This method removes a fault. */
-  lift(kind: NetworkFault, at?: FaultSide, options?: FaultOptions): Promise<void>;
+  lift(kind: NetworkFault, at?: FaultPlace, options?: FaultOptions): Promise<void>;
   /**
    * This method puts tc netem on the egress of a namespace, for example "delay 20ms" or "loss 5%",
-   * or removes it with null. The stamps of the injector bracket it.
+   * or removes it with null. At a hop, it is the egress of the device of the hop toward one side.
+   * The stamps of the injector bracket it.
    */
-  netem(spec: string | null, at?: FaultSide): Promise<{
+  netem(spec: string | null, at?: FaultPlace): Promise<{
     t0: number;
     t1: number;
   }>;
   /**
    * This method sets the TSO, GSO and GRO of a namespace. With them off, netem and a length match
-   * act on segments, not on GSO super-packets of up to 64 KB.
+   * act on segments, not on GSO super-packets of up to 64 KB. At a hop, it sets the two devices of the hop.
    */
-  offload(on: boolean, at?: FaultSide): Promise<void>;
+  offload(on: boolean, at?: FaultPlace): Promise<void>;
   /** This method stops the server process in the target with SIGKILL, with the command of the target. */
   kill(): Promise<void>;
   /**
@@ -470,7 +853,7 @@ interface Lab {
    * waits for that instant in the VM, thus the length of a stall does not include the latency of
    * a docker exec. Stamps bracket it, as for a fault.
    */
-  heal(at: FaultSide, when: number): Promise<{
+  heal(at: FaultPlace, when: number): Promise<{
     t0: number;
     t1: number;
   }>;
@@ -493,20 +876,48 @@ interface Lab {
   manifest(sysctls: Readonly<Record<string, string>>, settings?: Readonly<Record<string, string>>, targetSysctls?: Readonly<Record<string, string>>): Promise<Manifest>;
   close(): Promise<void>;
 }
+/** This function gives the content of a file of a proxy, with the address of the next hop for each {{next}}. */
+export declare const templated: (content: string, next: string) => string;
+/** This function gives the JSON body of a toxic for the API of Toxiproxy. Each toxic of a cell has the name `fault`. */
+export declare const toxicBody: (toxic: Toxic) => string;
+/**
+ * This function gives the routes of a chain: for each router, the networks after it, which the
+ * element before it reaches through it. Element 0 is the client, and element k is hop k - 1. Network
+ * j is the network between element j and element j + 1.
+ */
+export declare const routesOf: (hops: readonly Pick<Hop, "kind">[]) => Array<{
+  readonly from: number;
+  readonly via: number;
+  readonly networks: readonly number[];
+}>;
+/**
+ * This function gives the endpoint that an element connects to: the first element after it that
+ * is not a router. Element 0 is the client, element k is hop k - 1, and the last element is the target.
+ */
+export declare const endpointAfter: (hops: readonly Pick<Hop, "kind">[], element: number) => number;
+/** This function gives the device of a route from its line in `ip -o route get`: "10.0.0.2 dev eth0 src …". */
+export declare const deviceOf: (route: string) => string;
 /**
  * This function opens one environment:
- * - a new network and the stock target.
+ * - a chain of new networks, and the stock target on the last one. A direct pair has one network.
  * - a client that holds its own network namespace, with a privileged sidecar in it for its
- *   sysctls. Docker Desktop keeps /proc/sys read-only in a container that operates.
+ *   sysctls and its routes. Docker Desktop keeps /proc/sys read-only in a container that operates
+ *   without privileges.
+ * - each hop of the topology, on the network of each of its sides. A hop is a router, or a stock
+ *   proxy with an injector sidecar in its namespace.
  * - an injector sidecar in the namespace of the target.
- * - the prober, a privileged container that loads the kernel program and watches the two
- *   namespaces.
+ * - the prober, a privileged container that loads the kernel program and watches the namespace of
+ *   the client, of the target and of each hop.
  *
  * The client of each run is its own container in the namespace of the client. It is the TCP
- * client of the harness, or a driver from its own image (ioredis on Node). The two print the same
- * lines.
+ * client of the harness, a driver from its own image (ioredis on Node), or an image that the
+ * caller gives. Each prints the same lines.
+ *
+ * The client reaches the networks after a router through that router, by a route that its
+ * privileged sidecar adds. A router translates the address of the side of the client, thus the
+ * target and each later hop see the router as the peer.
  */
-export declare const openLab: (target: Target, kind?: Client) => Promise<Lab>;
+export declare const openLab: (target: Target, kind?: Client | ClientImage, topology?: Topology) => Promise<Lab>;
 //#endregion
 //#region src/run.d.ts
 /** A cell result is a cell with its manifests, its runs, their measurements and its verdict. */
@@ -514,6 +925,8 @@ interface CellResult {
   readonly cell: Cell;
   readonly manifests: readonly Manifest[];
   readonly runs: readonly Run[];
+  /** These are the warm-up runs, one for each environment before its counted runs. The verdict does not count them. */
+  readonly warmups: readonly Run[];
   readonly measured: readonly Measured[];
   readonly verdict: Verdict;
   /** These times are when its first environment opened and its last environment closed, on the wall clock. */
@@ -539,8 +952,10 @@ export declare const spread: (runs: number, environments: number) => number[];
  * the cell from 11 runs to 22, and then to 44, each extension in new environments. With a
  * concurrency of more than one, that many environments operate at the same time, each with its
  * own prober. The runs then go across at least as many environments.
+ *
+ * Each environment does a warm-up first, and the result keeps it apart from the counted runs.
  */
 export declare const runCell: (cell: Cell, say?: (line: string) => void) => Promise<CellResult>;
 //#endregion
-export type { Cell, CellResult, Client, Decision, Fault, FaultSide, Interval, KernelRecord, Lab, Manifest, Measured, NetworkFault, Observer, Outcome, Prediction, Probe, ProbeEvent, Recovers, Reference, Run, Target, Verdict };
+export type { Agent, Cell, CellResult, Client, ClientImage, Crossings, Decision, Fault, FaultPlace, FaultSide, Hop, Interval, KernelRecord, Lab, Manifest, ManifestHop, Measured, MiddleboxEvent, MiddleboxForm, NetworkFault, Observer, Outcome, Prediction, Probe, ProbeEvent, ProxyHop, ProxyTime, Recovers, Reference, Reordering, Replies, RouterHop, RttRange, Run, RunTopology, SocketPlace, Spread, Target, Topology, Toxic, Verdict };
 //# sourceMappingURL=index.d.ts.map