@mark1russell7/harness 0.14.1 → 0.14.3

package/build/index.d.ts

@@ -1,6 +1,6 @@
-import { $ as StageBand, A as PinnedImage, B as Replies, C as Jitter, D as MiddleboxExpectation, E as MiddleboxEvent, F as Reconnects, G as RunGroup, H as RouterHop, I as Recovers, J as SentinelEvent, K as RunPart, L as Reference, M as ProbeEvent, N as ProxyHop, O as MiddleboxForm, P as ProxyTime, Q as Stage, R as ReplicaSet, S as ImageSource, T as LineMatch, U as RttRange, V as RequestPrediction, W as Run, X as Shipped, Y as SentinelStage, Z as ShippedImage, _ as HopEvent, a as Cluster, at as isHopPlace, b as HopWrite, c as DriverBand, d as FaultPlace, et as Subject, f as Group, g as Hop, h as GroupRole, i as ClientImage, it as WsProbe, j as Prediction, k as Observer, l as DriverStep, m as GroupPrediction, n as Cell, nt as Topology, o as ClusterClientAt, ot as latestEndOf, p as GroupPlace, q as RunTopology, r as Client, rt as Toxic, s as ClusterStage, t as BuildImage, tt as Target, u as Fault, v as HopPlace, w as KernelRecord, x as HttpProbe, y as HopSignal, z as ReplicaSetStage } from "./cell-BccWKiXh.js";
+import { $ as RunRoute, A as MiddleboxExpectation, B as Reference, C as HopWrite, D as KernelRecord, E as Jitter, F as ProbeEvent, G as RouterHop, H as ReplicaSetStage, I as ProxyHop, J as RunBackup, K as RttRange, L as ProxyTime, M as Observer, N as PinnedImage, O as LineMatch, P as Prediction, Q as RunPart, R as Reconnects, S as HopSignal, T as ImageSource, U as Replies, V as ReplicaSet, W as RequestPrediction, X as RunFabric, Y as RunDrill, Z as RunGroup, _ as GroupPrediction, a as Cluster, at as Stage, b as HopEvent, c as Drill, ct as Target, d as Fabric, dt as WsProbe, et as RunTopology, f as FabricManifest, ft as isHopPlace, g as GroupPlace, h as Group, i as ClientImage, it as ShippedImage, j as MiddleboxForm, k as MiddleboxEvent, l as DriverBand, lt as Topology, m as FaultPlace, n as Cell, nt as SentinelStage, o as ClusterClientAt, ot as StageBand, p as Fault, pt as latestEndOf, q as Run, r as Client, rt as Shipped, s as ClusterStage, st as Subject, t as BuildImage, tt as SentinelEvent, u as DriverStep, ut as Toxic, v as GroupRole, w as HttpProbe, x as HopPlace, y as Hop, z as Recovers } from "./cell-B3-BSTit.js";
 import { Quantity } from "@mark1russell7/quantity";
-import "testcontainers";
+import { StartedTestContainer } from "testcontainers";
 //#region src/application.d.ts
 /**
  * This module reads the acts of the program of the observer: its heartbeat, its drop of the
@@ -204,6 +204,74 @@ export declare const referenceOf: (run: Run, from: Reference, role: string, data
  */
 export declare const measureDriver: (cell: Cell, run: Run) => Measured;
 //#endregion
+//#region src/recovery.d.ts
+/**
+ * This module holds the pure parts of a recovery drill. These are the scripts of the shell, the reader
+ * of the output of `bsondump`, and the set check of a recovery. It has no container, thus unit tests
+ * run it. `drill.ts` does the drill on a lab.
+ *
+ * A dump with `--oplog` reads the oplog from the start of the dump to its end. A slice reads the
+ * oplog after the last entry of the backup before it. Thus the backups together hold each write up
+ * to the last entry of the last backup, and no write after it. One writer sends the writes one at a
+ * time, thus the highest sequence number is also the latest write.
+ */
+/** This size is one document of generated data. A tenant holds as many documents as its size needs. */
+export declare const SEED_DOCUMENT = 65536;
+/** This count is how far the lost writes can be from the writes after the last backup: the writes in flight at the fault. */
+export declare const LOSS_MARGIN = 2;
+/** This function gives the time of a drill from the start of the backups to the fault, in seconds. */
+export declare const drillSpan: (drill: Drill) => number;
+/**
+ * This script writes the generated data of each tenant into the collection `tenants`. Each write
+ * waits for `copies` members, thus each member holds the data when the script ends.
+ */
+export declare const seedScript: (drill: Drill, copies: number) => string;
+/** This script prints the hash of the collection `tenants`. It takes locks, thus a run reads it with no writer. */
+export declare const HASH_SCRIPT: string;
+/** This function gives the hash from the output of `HASH_SCRIPT`. */
+export declare const hashOf: (text: string) => string;
+/** This function gives the command of a server with no set, on its own port. A client of the set cannot reach that server, and a restore writes to it. */
+export declare const standaloneCommand: (set: ReplicaSet, port: number) => string;
+/**
+ * This script waits for the end of the server of a box, and it can stop the server first. `kill -0`
+ * also succeeds on a zombie, and the shell of the box can leave its child as a zombie for a long
+ * time. Thus the script reads the state of the process, and a zombie has ended.
+ */
+export declare const stopScript: (terminate: boolean) => string;
+/** An oplog time is the `ts` of an oplog entry: a time in seconds, and the count of the entry in that second. */
+interface OplogTime {
+  readonly t: number;
+  readonly i: number;
+}
+/** An oplog read is what an oplog file holds: its last entry, and the highest sequence number that it inserts. */
+interface OplogRead {
+  readonly last: OplogTime | null;
+  readonly highest: number | null;
+}
+/** This function reads an oplog file from the output of `bsondump`. Only an insert into `cells.w` is a write of a client. */
+export declare const readOplog: (text: string) => OplogRead;
+/** This function gives the highest sequence number of the file of the writes in a dump, from the output of `bsondump`, or null. */
+export declare const highestHeld: (text: string) => number | null;
+/** This function gives the query of a slice of the oplog: each entry after an oplog time, in Extended JSON. */
+export declare const sliceQuery: (after: OplogTime) => string;
+/** A recovery measurement is what a drill found, beside the writes of the run. */
+interface RecoveryMeasured {
+  /** This time is from the reply to the last accepted write that the store kept to the fault, or null with no such write. */
+  readonly rpo: Quantity | null;
+  /** This time is `rpo` less the age of the drill: what a schedule adds to its interval. It is null for a control and for an election. */
+  readonly excess: Quantity | null;
+  /** These are the accepted writes that the store lost, below a write that it kept. A restore must leave none. */
+  readonly holes: readonly number[];
+  /** This count is the number of accepted writes after the last write of the last backup, or null without a backup. */
+  readonly afterBackup: number | null;
+  /** This flag tells if the tenants have the same data after the recovery, or null when that is not known. */
+  readonly sameData: boolean | null;
+}
+/** This function gives the measurement of a drill, from its run. `t0` is the fault. */
+export declare const recoveryOf: (drill: Drill, run: Run, t0: number) => RecoveryMeasured;
+/** This function gives the findings of a drill: each rule of a recovery that its run broke. */
+export declare const recoveryFindings: (measured: RecoveryMeasured, writes: Writes) => string[];
+//#endregion
 //#region src/timeline.d.ts
 /**
  * This module measures a run of a group cell. It gives the first write that a primary accepted
@@ -249,6 +317,8 @@ interface GroupMeasured {
   readonly errors: readonly ErrorOnTheWay[];
   /** These are the delays of the stamps of the observer after the writes of the Sentinels, for a calibration. */
   readonly lags?: readonly Quantity[];
+  /** This is what a recovery drill measured, for a drill only. */
+  readonly recovery?: RecoveryMeasured;
 }
 /** This function gives the time of each stage of a timeline from an instant: the first announcement of each, after the instant. */
 export declare const stagesOf: (timeline: readonly SentinelEvent[], from: number) => Partial<Record<Stage, Quantity>>;
@@ -435,6 +505,8 @@ interface Verdict {
   readonly median: Quantity | null;
   /** This interval is the order-statistic interval of the median at 95 %, or the whole range for fewer than six runs. */
   readonly medianInterval: Interval | null;
+  /** This time is the slowest reply of a cell that measures a distribution. A closed loop hides the sends in a stall, thus a quantile is a lower bound, and this maximum is not. */
+  readonly slowest?: Quantity;
   /** This record is the sum of what reordering did in each counted run of a measurement-only cell. */
   readonly reordering?: Reordering;
   /** This record is the time of the proxy of a calibration cell, in each direction. */
@@ -761,6 +833,378 @@ export declare const jitterOf: (cell: Cell, run: Run) => Jitter;
  */
 export declare const requestOf: (events: readonly ProbeEvent[]) => RequestReading | null;
 //#endregion
+//#region src/fabric.d.ts
+/**
+ * A fabric is a routed network between the client and the target, on layer E3 (docs/S7.md). Four
+ * routers of FRRouting, two leaves and two spines, run real BGP over veth pairs that containerlab
+ * makes. The client hangs off leaf1 and the target off leaf2. The path between them goes through
+ * spine1 by a weight, and spine2 holds the other path. A cell silences spine1, and the routes of
+ * the leaves move to spine2 when a detector of the routers fires.
+ *
+ * This file holds the plan of the fabric as pure functions, and `openFabric` that starts it. A lab
+ * opens a fabric with `Topology.fabric`. Thus the client, the target, the prober and the verdict
+ * stay the same as on layer E1. The fabric needs a Linux host with containerlab and sudo.
+ */
+/** This image is FRRouting 10.7.1, pinned by digest. */
+export declare const FRR_IMAGE = "quay.io/frrouting/frr:10.7.1@sha256:e995beaa50fdc9edb35eadcfefa29b7f062cc06f2b812613789b68fa541554d2";
+/** This device is the side of the fabric in the client and in the target. Docker keeps eth0. */
+export declare const FABRIC_DEVICE = "eth1";
+/** This address is the client on its network behind leaf1. */
+export declare const FABRIC_CLIENT = "10.1.0.2";
+/** This address is the target on its network behind leaf2. */
+export declare const FABRIC_TARGET = "10.2.0.2";
+/** A router is one box of the fabric, as data. */
+interface Router {
+  readonly name: string;
+  readonly as: number;
+  readonly id: string;
+  readonly interfaces: Readonly<Record<string, string>>;
+  readonly neighbors: ReadonlyArray<{
+    readonly address: string;
+    readonly as: number;
+    readonly weight?: number;
+  }>;
+  /** This prefix is the network behind a leaf, which its BGP advertises. */
+  readonly network?: string;
+  /** This prefix is the network behind the other leaf. A move is a change of its route. */
+  readonly remote?: string;
+}
+/** These are the routers. Each leaf prefers spine1 by a weight of 100, thus the path of each run is known. */
+export declare const ROUTERS: readonly Router[];
+/** This text is the `daemons` file of each router: BGP and BFD, on top of what FRRouting always starts. */
+export declare const DAEMONS: string;
+/**
+ * This function gives the configuration of a router. Graceful restart is off: after a hold expiry,
+ * a helper can keep the stale routes, and the move that a cell measures does not come.
+ * `maximum-paths 1` keeps one path, thus a move is one change of one route.
+ */
+export declare const frrConf: (router: Router, fabric: Fabric) => string;
+/**
+ * This function gives the topology file of containerlab. The client and the target are containers
+ * of the lab, and the fabric adds a device to each. Each link has the MTU of a Docker network, thus
+ * a segment is as long as on layer E1. The default of containerlab is 9500. The management network
+ * belongs to the lab, and Docker chooses its subnet.
+ */
+export declare const topologyOf: (lab: string, dir: string, client: string, target: string) => string;
+/** A route line is one line of `ip -ts -o monitor route`, with its time on the wall clock in ms. */
+interface RouteLine {
+  readonly wall: number;
+  readonly deleted: boolean;
+  readonly prefix: string;
+  readonly via: string | null;
+  readonly dev: string | null;
+}
+/** This function reads a line of `ip -ts -o monitor route`, or it gives null for another line. The clock of a container is UTC. */
+export declare const routeLine: (text: string) => RouteLine | null;
+/** A clock pair is one instant on the monotonic clock in ns and on the wall clock in ms. */
+interface ClockPair {
+  readonly mono: number;
+  readonly wall: number;
+}
+/**
+ * This function takes a time on the wall clock to the monotonic clock. The harness runs on the host
+ * of the containers, thus `process.hrtime` reads the CLOCK_MONOTONIC of the injector and the
+ * prober. The error of the pair is approximately 1 ms, the step of `Date.now`.
+ */
+export declare const monotonicAt: (wall: number, pair: ClockPair) => number;
+/** This function gives the state of the BGP sessions from `show bgp neighbors json`. */
+export declare const sessionsOf: (json: string) => {
+  readonly up: number;
+  readonly established: number;
+};
+/** These are the views of a leaf that tell if a run can start on it. */
+interface LeafViews {
+  readonly neighbors: string;
+  /** This text is the kernel's route to the far network, from `ip -o route show`. */
+  readonly route: string;
+  /** This text is `show bgp ipv4 unicast PREFIX json`. */
+  readonly paths: string;
+  /** This text is `show bfd peers json`, or null when the fabric has no BFD. */
+  readonly bfd: string | null;
+}
+/**
+ * This function tells what a leaf still lacks before a run, or it gives null. The two sessions must
+ * be up and the route must be on spine1. Spine2 must hold the other path. Without these, the fault
+ * of a run acts on another path, or nothing can take over.
+ */
+export declare const leafLacks: (views: LeafViews) => string | null;
+/**
+ * This function gives the later move of the leaves after an instant: for each leaf, the first route
+ * to its far network that leaves spine1. It gives null when a leaf has no move. The later move ends
+ * the outage in the two directions.
+ */
+export declare const laterMove: (routes: readonly RunRoute[], after: number) => number | null;
+/** A fabric lab is the open fabric of a lab. */
+interface FabricLab {
+  /** These are the lines of the route monitors of the leaves, on the monotonic clock. */
+  readonly routes: readonly RunRoute[];
+  /** This method drops each packet of spine1 in one commit, and the links stay up. */
+  silence(): Promise<{
+    t0: number;
+    t1: number;
+  }>;
+  /** This method ends a silence. */
+  clear(): Promise<{
+    t0: number;
+    t1: number;
+  }>;
+  /** This method waits until each leaf is on spine1 with each session up. */
+  ready(): Promise<void>;
+  /** This method gives the count of established sessions of each leaf so far. */
+  sessions(): Promise<Readonly<Record<string, number>>>;
+  /** This method gives the record of the fabric for a manifest. */
+  manifest(): FabricManifest;
+  /** This method destroys the routers. */
+  close(): Promise<void>;
+}
+/** These are the parts of a lab that a fabric uses. */
+interface FabricParts {
+  readonly fabric: Fabric;
+  readonly image: string;
+  readonly client: StartedTestContainer;
+  readonly server: StartedTestContainer;
+  readonly admin: StartedTestContainer;
+  readonly injector: StartedTestContainer;
+  readonly opened: StartedTestContainer[];
+  readonly output: (container: StartedTestContainer, argv: string[]) => Promise<string>;
+}
+/**
+ * This function deploys the routers of a lab and joins the client and the target to them. It also
+ * starts the sidecars: the injector of spine1 and a route monitor on each leaf. The caller stops the
+ * sidecars with the other containers of the lab, and it calls `close` first.
+ */
+export declare const openFabric: (parts: FabricParts) => Promise<FabricLab>;
+//#endregion
+//#region src/image.d.ts
+/** These are the names of the images that the harness ships. */
+export declare const SHIPPED_NAMES: readonly [Shipped, ...Shipped[]];
+/** This function gives the source of a shipped image. */
+export declare const shippedSource: (name: Shipped) => ImageSource;
+/** This function gives the version of the package of the harness, from its package.json. */
+export declare const harnessVersion: () => string;
+/**
+ * These are the clients that the harness ships, by their names (`Client`): the image of each, and
+ * its command. A cell that names a client gets this image and this command.
+ */
+export declare const SHIPPED_CLIENTS: Readonly<Record<Client, ClientImage>>;
+/** These are the names of the clients that the harness ships. */
+export declare const CLIENT_NAMES: readonly [Client, ...Client[]];
+/** This function gives the command of a program of a subject by its role. It fails when the subject has no such program. */
+export declare const programOf: (subject: Subject, role: string) => readonly string[];
+/**
+ * This function gives the client of a subject for a cell (`Cell.client`): the image of the subject,
+ * with the command of a program. A client that the harness ships goes by its name, thus its records
+ * stay the same.
+ */
+export declare const subjectClient: (subject: Subject, role?: string) => Client | ClientImage;
+/** This pattern matches an image pinned by digest: a name, a tag or none, then `@sha256:` and 64 hex digits. */
+export declare const PINNED: RegExp;
+/** This pattern matches the repository of a tag that the harness builds. Its parts are lower case, with `/` between them. */
+export declare const REPOSITORY: RegExp;
+/** This function tells if a path stays in its context: relative, with `/` between its parts, and no `..`. */
+export declare const inContext: (path: string) => boolean;
+/**
+ * This function gives the problems of the source of an image, as data. A pinned image names a
+ * digest. A build names a repository, a context, and paths that stay in the context. A shipped image
+ * is one that the harness ships.
+ */
+export declare const imageProblems: (source: ImageSource) => string[];
+/**
+ * This function gives the problems of a subject, as data. A subject has a name and a base that a
+ * digest pins. Its image source has no problems, and each of its programs is a command.
+ */
+export declare const subjectProblems: (subject: Subject) => string[];
+/**
+ * This function gives each `FROM` of a Dockerfile that no digest pins. A `FROM` of an earlier stage
+ * by its name, or of `scratch`, needs no digest. A base from an `ARG` has no digest that the harness
+ * can read, thus it is in the list too.
+ */
+export declare const unpinnedBases: (dockerfile: string) => string[];
+/**
+ * This function gives each base of a Dockerfile, in the sequence of the file. A base is the image of
+ * a `FROM` that is not `scratch` or an earlier stage by its name.
+ */
+export declare const basesOf: (dockerfile: string) => string[];
+/**
+ * This function gives each manifest of packages in a context (`package.json`, `Cargo.toml`) that
+ * has no lockfile in its folder or in a folder above it. The paths are the paths in the context.
+ */
+export declare const unlockedManifests: (paths: readonly string[]) => string[];
+/**
+ * An image plan tells how the harness gets the image of a source. It is a pull of an image by its
+ * name, or a build of a context. A build has its tag, its Dockerfile, and each file with its path.
+ */
+type ImagePlan = {
+  readonly kind: "pull";
+  readonly image: string;
+} | {
+  readonly kind: "build";
+  readonly tag: string;
+  readonly dockerfile: string;
+  readonly files: ReadonlyArray<{
+    readonly file: string;
+    readonly at: string;
+  }>;
+  /** This record tells what the image holds, for the manifest of a run. */
+  readonly record: BuildRecord;
+};
+/**
+ * A build record tells what an image that the harness builds holds, for the manifest of a run. The
+ * tag is a sha256 of each file of the context. The bases are pinned by digest, and each lockfile
+ * has its sha256. Thus the record pins each program of the image. The versions are the part that a
+ * person reads.
+ */
+interface BuildRecord {
+  readonly tag: string;
+  /** These are the bases of its Dockerfile, each pinned by digest (`basesOf`). */
+  readonly bases: readonly string[];
+  /** These are the lockfiles of its context by their path, each with the sha256 of its content (`locksOf`). */
+  readonly locks: Readonly<Record<string, string>>;
+  /** These are the versions of the packages that its manifests name directly, by their names (`versionsOf`). */
+  readonly versions: Readonly<Record<string, string>>;
+}
+/**
+ * This function gives the plan of the image of a source. A stock image by its name, or a pinned
+ * image, is a pull. A build or a shipped image is a build. Its tag is its name, then 12 digits of a
+ * sha256. The sha256 reads the path of the Dockerfile, and the path and the content of each file.
+ * Thus a changed program cannot operate as an old one.
+ *
+ * A build fails before it starts on a problem of its source. It also fails on a `FROM` that no
+ * digest pins, and on a manifest of packages without a lockfile.
+ */
+export declare const planOf: (source: string | ImageSource) => ImagePlan;
+/**
+ * This function gives an image that a lab can start, from its source. It is the name of a stock
+ * image or of a pinned image, which the lab pulls, or the tag of a build. A build goes through the CLI
+ * of Docker.
+ */
+export declare const imageOf: (source: string | ImageSource) => Promise<string>;
+/**
+ * This function gives the build record of the image of a source, for the manifest of a run. A
+ * stock image by its name, or a pinned image, is a pull, and it has no build record.
+ */
+export declare const buildRecordOf: (source: string | ImageSource) => BuildRecord | undefined;
+/** This function gives the image of the harness: the prober, the TCP client, the injector and the namespace admin. */
+export declare const harnessImage: () => Promise<string>;
+/** This function gives the image of the Node client. */
+export declare const nodeClientImage: () => Promise<string>;
+/** This function gives the image of the router. */
+export declare const routerImage: () => Promise<string>;
+/** This function gives the image of the Go HTTP client. */
+export declare const goHttpImage: () => Promise<string>;
+/** This function gives the image of the Node HTTP clients. */
+export declare const nodeHttpImage: () => Promise<string>;
+/** This function gives the image of the MongoDB client. */
+export declare const mongoClientImage: () => Promise<string>;
+/** This function gives the image of the PostgreSQL client. */
+export declare const pgClientImage: () => Promise<string>;
+/** This function gives the image of the client of HikariCP over pgjdbc. */
+export declare const hikariClientImage: () => Promise<string>;
+/** This function gives the image of the RabbitMQ client: amqplib, with the client and the admin of a cluster cell. */
+export declare const amqpClientImage: () => Promise<string>;
+/** This function gives the image of the gRPC client of the harness: grpc-js 1.14.6 on Node 24.21.0, with its client. */
+export declare const grpcClientImage: () => Promise<string>;
+/**
+ * This function gives the source of the image that a target operates. It is the image of its
+ * subject (`Target.subject`), or its stock image.
+ */
+export declare const targetSourceOf: (target: Target) => string | ImageSource;
+/** This function gives the image that a target operates: the image of its subject, or its stock image. */
+export declare const targetImageOf: (target: Target) => Promise<string>;
+//#endregion
+//#region src/substrate.d.ts
+/**
+ * A layer is a rung of the laboratory (`docs/S7.md`). E1 is containers on one kernel, and E2 is a
+ * cluster of containers on one kernel. A lab with `Topology.fabric` writes E3: routers in containers.
+ * A lab in a guest of QEMU writes E4 (`substrateHere`). The other layers do not have a runner at this
+ * time.
+ */
+type Layer = "E0" | "E1" | "E2" | "E3" | "E4" | "E5" | "E6";
+/** A cloud is where the runner operates. GitHub can schedule a standard Linux job on Azure or on AWS. */
+type Cloud = "azure" | "aws" | "unknown";
+/** A host is what a process reads about its own machine. */
+interface Host {
+  /** This text is the kernel release. */
+  readonly kernel: string | null;
+  /** This number is `CONFIG_HZ` from the configuration of the kernel, or null without that file. */
+  readonly hz: number | null;
+  /** This text is the DMI system vendor, for example `Microsoft Corporation`, or null without that file. */
+  readonly vendor: string | null;
+  /** This value tells if `/dev/kvm` exists. A later VM layer needs it. */
+  readonly kvm: boolean;
+  /** This text is `$ImageOS $ImageVersion` of a hosted runner, or null elsewhere. */
+  readonly image: string | null;
+}
+/**
+ * A substrate tells where a record comes from. The rules of comparison in `docs/S7.md` read it: no
+ * statistic pools two layers, two kernels or two clouds. The digest of each image stays in the
+ * manifest. A value is null in a record from before this block.
+ */
+interface Substrate {
+  readonly layer: Layer;
+  readonly cloud: Cloud;
+  readonly vendor: string | null;
+  readonly kernel: string | null;
+  readonly hz: number | null;
+  readonly kvm: boolean | null;
+  readonly image: string | null;
+}
+/** GitHub announced Linux jobs on AWS on this date. Each earlier hosted job ran on Azure. */
+export declare const AWS_FROM = "2026-10-28";
+/** This function gives the cloud of a DMI vendor. A kernel name does not tell: one image can start on each cloud. */
+export declare const cloudOf: (vendor: string | null) => Cloud;
+/** This function reads the host. `root` and `env` are for tests. */
+export declare const readHost: (root?: string, env?: Readonly<Record<string, string | undefined>>, kernel?: string) => Host;
+/** This function gives the substrate of a layer on a host. */
+export declare const substrateOn: (layer: Layer, host?: Host) => Substrate;
+/**
+ * This function gives the substrate of a lab at this place. A guest cannot read its own runner: its
+ * DMI vendor is QEMU, and `ImageOS` is not set. Thus the runner writes its host to `LAB_RUNNER`, and
+ * the guest keeps only its own kernel and HZ. A value of that variable makes the layer E4.
+ */
+export declare const substrateHere: (layer: Layer, host?: Host, env?: Readonly<Record<string, string | undefined>>) => Substrate;
+/** A CPU reading is two counters of `/proc/stat`, in ticks of the clock. */
+interface CpuTime {
+  /** This count is the time that the host gave to another guest while this one had work. */
+  readonly steal: number;
+  /** This count is the sum of all states of each CPU. */
+  readonly total: number;
+}
+/** This function reads the CPU time of this machine, or gives null without `/proc/stat`. `root` is for tests. */
+export declare const readCpu: (root?: string) => CpuTime | null;
+/**
+ * This function gives the share of the CPU time between two readings that the host took. The share
+ * goes from zero to one. It is null if no tick passed. A guest sees only the host above it. On a hosted
+ * runner that host is itself a guest, and this number does not show the contention of the layer below it.
+ */
+export declare const stealShare: (before: CpuTime, after: CpuTime) => number | null;
+/** A part of a record that can hold a substrate: a manifest of the harness, or the fields of a kind manifest. */
+interface SubstrateSource {
+  readonly substrate?: Substrate;
+  readonly kernel?: string;
+  readonly hz?: string | null;
+}
+/**
+ * This function reads the substrate of a record, the upgrade of a record from before this block.
+ * Nothing rewrites such a record. It reads as the given layer. Its cloud is Azure if the record is
+ * from before `AWS_FROM`, and unknown after that date, because its runner did not record the cloud.
+ */
+export declare const substrateOf: (source: SubstrateSource, at: string, layer?: Layer) => Substrate;
+/**
+ * This function tells if a statistic can pool two substrates. They must have one layer, one kernel
+ * and one cloud. An unknown value does not refuse a pair, because an old record cannot tell. The
+ * reader of such a pair says so with `substrateKey`.
+ */
+export declare const comparable: (a: Substrate, b: Substrate) => boolean;
+/** This function gives the one-line name of a substrate, for a summary. */
+export declare const substrateKey: (substrate: Substrate) => string;
+/**
+ * This function gives the first two substrates that a statistic cannot pool, or undefined. It
+ * examines each pair. An unknown value matches Azure and AWS, thus one comparison with the first
+ * substrate can miss a pair.
+ */
+export declare const unpoolable: (substrates: readonly Substrate[]) => readonly [Substrate, Substrate] | undefined;
+//#endregion
 //#region src/lab.d.ts
 /**
  * This time is the interval of the listing of a router, when its hop does not set one. A listing
@@ -829,6 +1273,10 @@ interface Manifest {
   readonly hzFrom?: "config" | "measured";
   readonly boot_id: string;
   readonly docker: string;
+  /** This block tells the layer, the cloud and the kernel of the record. A record from before it reads through `substrateOf`. */
+  readonly substrate?: Substrate;
+  /** This record is the fabric of a cell with `Topology.fabric`. */
+  readonly fabric?: FabricManifest;
   /** This record is the target, and its settings as it read them back. */
   readonly target: {
     readonly image: string;
@@ -843,11 +1291,20 @@ interface Manifest {
     readonly files?: Readonly<Record<string, string>>;
     /** This line is the first line of a target that writes them: a server that tells its settings as it reads them. */
     readonly first?: ProbeEvent;
+    /** This record tells what the image of a target holds when the harness builds it: the image of a subject (`buildRecordOf`). */
+    readonly build?: BuildRecord;
   };
+  /**
+   * This record is the harness: its image, the id of that image and its Alpine packages. It also
+   * has the version of the package of the harness, and the build of its image. The record of a
+   * cell gives the commit of the code that operated.
+   */
   readonly harness: {
     readonly image: string;
     readonly id: string;
     readonly packages: string[];
+    readonly version?: string;
+    readonly build?: BuildRecord;
   };
   /**
    * This record is the client of each run: the TCP client of the harness, a driver, or an image that
@@ -857,6 +1314,7 @@ interface Manifest {
     readonly kind: Client | "image";
     readonly image: string;
     readonly source?: ImageSource;
+    readonly build?: BuildRecord;
   };
   /** These are the namespaces that the prober watched, by inode. */
   readonly netns: {
@@ -1014,6 +1472,8 @@ interface Lab {
    * independent of the run before it.
    */
   forget(): Promise<void>;
+  /** This part is the routed fabric of a lab with `Topology.fabric`. */
+  readonly fabric?: FabricLab;
   /** This method freezes the target with the cgroup freezer: its process stops, and its kernel continues to answer. */
   freeze(): Promise<void>;
   thaw(): Promise<void>;
@@ -1080,6 +1540,20 @@ export declare const hzOf: (first: {
   now: number;
   jiffies: number;
 }) => string | null;
+/** This function gives the build record of a source as a part of a manifest: `{ build }`, or nothing for an image that the harness pulls. */
+export declare const buildOf: (source: Parameters<typeof buildRecordOf>[0]) => {
+  readonly build?: BuildRecord;
+};
+/** This function gives the version of the harness and the build of its image, as a part of the harness record of a manifest. */
+export declare const harnessRecord: () => {
+  readonly version: string;
+  readonly build?: BuildRecord;
+};
+/**
+ * This function gives the record of the client of a run that the harness ships, for a manifest:
+ * its kind, its image, and its build.
+ */
+export declare const shippedClientRecord: (kind: Client, image: string) => Manifest["client"];
 /** This function gives the device of a route from its line in `ip -o route get`: "10.0.0.2 dev eth0 src …". */
 export declare const deviceOf: (route: string) => string;
 /**
@@ -1104,111 +1578,60 @@ export declare const deviceOf: (route: string) => string;
  */
 export declare const openLab: (target: Target, kind?: Client | ClientImage, topology?: Topology) => Promise<Lab>;
 //#endregion
-//#region src/image.d.ts
-/** These are the names of the images that the harness ships. */
-export declare const SHIPPED_NAMES: readonly [Shipped, ...Shipped[]];
+//#region src/locks.d.ts
 /**
- * These are the clients that the harness ships, by their names (`Client`): the image of each, and
- * its command. A cell that names a client gets this image and this command.
+ * This module reads the lockfiles of a build context for the manifest of a run. A lockfile pins each
+ * package that a build installs, thus its sha256 pins the packages that the image holds. The
+ * versions of the packages that each manifest names directly are the part that a person reads.
  */
-export declare const SHIPPED_CLIENTS: Readonly<Record<Client, ClientImage>>;
-/** These are the names of the clients that the harness ships. */
-export declare const CLIENT_NAMES: readonly [Client, ...Client[]];
-/** This function gives the command of a program of a subject by its role. It fails when the subject has no such program. */
-export declare const programOf: (subject: Subject, role: string) => readonly string[];
-/**
- * This function gives the client of a subject for a cell (`Cell.client`): the image of the subject,
- * with the command of a program. A client that the harness ships goes by its name, thus its records
- * stay the same.
- */
-export declare const subjectClient: (subject: Subject, role?: string) => Client | ClientImage;
-/** This pattern matches an image pinned by digest: a name, a tag or none, then `@sha256:` and 64 hex digits. */
-export declare const PINNED: RegExp;
-/** This pattern matches the repository of a tag that the harness builds. Its parts are lower case, with `/` between them. */
-export declare const REPOSITORY: RegExp;
-/** This function tells if a path stays in its context: relative, with `/` between its parts, and no `..`. */
-export declare const inContext: (path: string) => boolean;
-/**
- * This function gives the problems of the source of an image, as data. A pinned image names a
- * digest. A build names a repository, a context, and paths that stay in the context. A shipped image
- * is one that the harness ships.
- */
-export declare const imageProblems: (source: ImageSource) => string[];
-/**
- * This function gives the problems of a subject, as data. A subject has a name and a base that a
- * digest pins. Its image source has no problems, and each of its programs is a command.
- */
-export declare const subjectProblems: (subject: Subject) => string[];
-/**
- * This function gives each `FROM` of a Dockerfile that no digest pins. A `FROM` of an earlier stage
- * by its name, or of `scratch`, needs no digest. A base from an `ARG` has no digest that the harness
- * can read, thus it is in the list too.
- */
-export declare const unpinnedBases: (dockerfile: string) => string[];
-/**
- * This function gives each manifest of packages in a context (`package.json`, `Cargo.toml`) that
- * has no lockfile in its folder or in a folder above it. The paths are the paths in the context.
- */
-export declare const unlockedManifests: (paths: readonly string[]) => string[];
-/**
- * An image plan tells how the harness gets the image of a source. It is a pull of an image by its
- * name, or a build of a context. A build has its tag, its Dockerfile, and each file with its path.
- */
-type ImagePlan = {
-  readonly kind: "pull";
-  readonly image: string;
-} | {
-  readonly kind: "build";
-  readonly tag: string;
-  readonly dockerfile: string;
-  readonly files: ReadonlyArray<{
-    readonly file: string;
-    readonly at: string;
-  }>;
-};
-/**
- * This function gives the plan of the image of a source. A stock image by its name, or a pinned
- * image, is a pull. A build or a shipped image is a build. Its tag is its name, then 12 digits of a
- * sha256. The sha256 reads the path of the Dockerfile, and the path and the content of each file.
- * Thus a changed program cannot operate as an old one.
- *
- * A build fails before it starts on a problem of its source. It also fails on a `FROM` that no
- * digest pins, and on a manifest of packages without a lockfile.
- */
-export declare const planOf: (source: string | ImageSource) => ImagePlan;
-/**
- * This function gives an image that a lab can start, from its source. It is the name of a stock
- * image or of a pinned image, which the lab pulls, or the tag of a build. A build goes through the CLI
- * of Docker.
- */
-export declare const imageOf: (source: string | ImageSource) => Promise<string>;
-/** This function gives the image of the harness: the prober, the TCP client, the injector and the namespace admin. */
-export declare const harnessImage: () => Promise<string>;
-/** This function gives the image of the Node client. */
-export declare const nodeClientImage: () => Promise<string>;
-/** This function gives the image of the router. */
-export declare const routerImage: () => Promise<string>;
-/** This function gives the image of the Go HTTP client. */
-export declare const goHttpImage: () => Promise<string>;
-/** This function gives the image of the Node HTTP clients. */
-export declare const nodeHttpImage: () => Promise<string>;
-/** This function gives the image of the MongoDB client. */
-export declare const mongoClientImage: () => Promise<string>;
-/** This function gives the image of the PostgreSQL client. */
-export declare const pgClientImage: () => Promise<string>;
-/** This function gives the image of the client of HikariCP over pgjdbc. */
-export declare const hikariClientImage: () => Promise<string>;
-/** This function gives the image of the RabbitMQ client: amqplib, with the client and the admin of a cluster cell. */
-export declare const amqpClientImage: () => Promise<string>;
-/** This function gives the image of the gRPC client of the harness: grpc-js 1.14.6 on Node 24.21.0, with its client. */
-export declare const grpcClientImage: () => Promise<string>;
+/** A context file is a file of a build context: its path in the context, and its content. */
+interface ContextFile {
+  readonly at: string;
+  readonly content: Buffer | string;
+}
 /**
- * This function gives the source of the image that a target operates. It is the image of its
- * subject (`Target.subject`), or its stock image.
- */
-export declare const targetSourceOf: (target: Target) => string | ImageSource;
-/** This function gives the image that a target operates: the image of its subject, or its stock image. */
-export declare const targetImageOf: (target: Target) => Promise<string>;
+ * These are the names of the lockfiles that the harness records. A `go.mod` is the lockfile of a Go
+ * module, because its `require` lines give exact versions. For them, Go selects the same packages
+ * each time, and its checksum database examines each package.
+ */
+export declare const LOCK_NAMES: ReadonlySet<string>;
+/** This function gives each lockfile of a context by its path, with the sha256 of its content in hex. */
+export declare const locksOf: (files: readonly ContextFile[]) => Record<string, string>;
+/** This function gives the version of each direct dependency of an npm lockfile (lockfileVersion 2 or 3). */
+export declare const npmVersions: (lock: string) => Record<string, string>;
+/**
+ * This function gives the version of each direct dependency of the root package of a Cargo.lock: the
+ * package that the Cargo.toml beside it names. An entry of a dependency list is a name, or a name
+ * and a version when the lock holds two versions of that name.
+ */
+export declare const cargoVersions: (lock: string, manifest: string) => Record<string, string>;
+/** This function gives the version of each direct requirement of a go.mod. A requirement that is `// indirect` is not direct. */
+export declare const goVersions: (mod: string) => Record<string, string>;
+/** This function gives the version of each jar of a list of jars by their SHA-256: `<hex>  HikariCP-7.1.0.jar` gives HikariCP 7.1.0. */
+export declare const jarVersions: (list: string) => Record<string, string>;
+/**
+ * This function gives the version of each package that the manifests of a context name directly,
+ * by its name. It reads each npm lockfile, Cargo.lock, go.mod and list of jars. A package of a
+ * workspace in the context is a package of the image too, with the version of its package.json.
+ * Such a package has a name and a version, and it is not private. The record has the sha256 of
+ * the lockfile of a pnpm, yarn or bun workspace (`locksOf`). This function does not read versions
+ * from such a lockfile.
+ */
+export declare const versionsOf: (files: readonly ContextFile[]) => Record<string, string>;
+//#endregion
+//#region src/pins.d.ts
+/**
+ * This function gives each problem of a manifest that stops it from pinning the programs of its
+ * run. An empty list means that the manifest pins each program.
+ * - the target: an image by its digest, or the build of a subject with its build record and its base
+ *   by digest.
+ * - the harness: the version of its package, and the build record of its image.
+ * - the client: an image by its digest, or a build with its build record.
+ * - each hop: an image by its digest, or the tag of a build and the id of its image.
+ */
+export declare const manifestProblems: (manifest: Manifest) => string[];
+/** This function gives the problem of the commit of a record: no commit, or a text that is not a commit of git. */
+export declare const commitProblems: (commit: string | undefined) => string[];
 //#endregion
 //#region src/group.d.ts
 /**
@@ -1455,6 +1878,12 @@ interface ReplicaSetLab {
   signal(part: string, signal: "KILL" | "STOP" | "CONT", at?: number): Promise<Stamp>;
   /** This method starts the server of a member again, after a fin. */
   restart(member: string): Promise<void>;
+  /** This method runs a command in the box of a member, and gives its output. */
+  exec(member: string, argv: string[]): Promise<string>;
+  /** This method runs a script of the shell on a member, against its own server only, and gives its output. Another port reads a server with no set. */
+  script(member: string, script: string, port?: number): Promise<string>;
+  /** This method runs a command on the backup host of a drill, and gives its output. A lab without that host refuses it. */
+  vault(argv: string[]): Promise<string>;
   /** This method sends replSetStepDown to the primary. */
   stepDown(primary: string): Promise<Stamp>;
   /** This method removes the TCP metrics of the client namespace: each run starts from a kernel that keeps nothing. */
@@ -1478,8 +1907,9 @@ export declare const STEP_DOWN: {
  * - an injector sidecar for each member, in its network and process namespaces.
  * - a client that holds its own network namespace, with a privileged sidecar for its sysctls.
  * - the prober, which watches the namespace of the client.
+ * - a backup host for a recovery drill: the stock image, for its tools, with no server.
  */
-export declare const openReplicaSetLab: (set: ReplicaSet) => Promise<ReplicaSetLab>;
+export declare const openReplicaSetLab: (set: ReplicaSet, backups?: boolean) => Promise<ReplicaSetLab>;
 /** This function gives the arguments of the client of a replica set cell: each member, and the stream of writes. */
 export declare const replicaSetArgs: (cell: Cell, members: Readonly<Record<string, RunPart>>) => string[];
 /**
@@ -1497,6 +1927,18 @@ export declare const replicaSetArgs: (cell: Cell, members: Readonly<Record<strin
  */
 export declare const replicaSetOnce: (lab: ReplicaSetLab, cell: Cell, environment: number, index: number) => Promise<Run>;
 //#endregion
+//#region src/drill.d.ts
+/**
+ * This function does one run of a recovery drill:
+ * - The set converges, and it loses its writes. The tenants get their data, and a hash reads it.
+ * - The client writes until its stream is steady. A control stops it here.
+ * - The backups go, each at its interval. The fault comes at the age of the newest backup.
+ * - An election continues the same client. A restore stops the client, and it starts a second one.
+ * - The set converges again. The writes that it then holds tell which writes it kept, and a second
+ *   hash tells if the tenants have the same data.
+ */
+export declare const drillOnce: (lab: ReplicaSetLab, cell: Cell, environment: number, index: number) => Promise<Run>;
+//#endregion
 //#region src/cluster.d.ts
 /**
  * This module is the lab of a cluster of RabbitMQ with one quorum queue. The members of the queue
@@ -1715,5 +2157,5 @@ export declare const spread: (runs: number, environments: number) => number[];
  */
 export declare const runCell: (cell: Cell, say?: (line: string) => void) => Promise<CellResult>;
 //#endregion
-export { type Agent, type BuildImage, type CallReading, type Cell, type CellResult, type Client, type ClientImage, type Cluster, type ClusterClientAt, type ClusterLab, type ClusterStage, type Crossings, type Cut, type Decision, type DriverBand, type DriverSocket, type DriverStep, type DriverStepTime, type ErrorOnTheWay, type Fault, type FaultPlace, type FaultSide, type Group, type GroupClient, type GroupLab, type GroupMeasured, type GroupNetworkFault, type GroupPlace, type GroupPrediction, type GroupReading, type GroupRole, type GroupVerdict, type GroupView, type Hop, type HopEvent, type HopPlace, type HopSignal, type HopWrite, type HttpProbe, type ImagePlan, type ImageSource, type Interval, type Jitter, type KernelRecord, type Lab, type LineMatch, type Manifest, type ManifestHop, type Measured, type MemberLine, type MiddleboxEvent, type MiddleboxExpectation, type MiddleboxForm, type NetworkFault, type NodeLine, type Observer, type Outcome, type Pairing, type PinnedImage, type Prediction, type Probe, type ProbeEvent, type ProgramEvent, type ProxyHop, type ProxyTime, type ReconnectReading, type ReconnectVerdict, type Reconnects, type Recovers, type Reference, type Reordering, type ReplicaSet, type ReplicaSetLab, type ReplicaSetStage, type Replies, type RequestPrediction, type RequestReading, type RouterHop, type RttRange, type Run, type RunGroup, type RunPart, type RunTopology, type SentinelEvent, type SentinelStage, type Shipped, type ShippedImage, type SocketPlace, type Spread, type Stage, type StageBand, type StageVerdict, type Stamp, type Status, type StatusRow, type Subject, type Target, type Topology, type Toxic, type Verdict, type WallStamp, type Writes, type WsProbe, isHopPlace, latestEndOf };
+export { type Agent, type BuildImage, type BuildRecord, type CallReading, type Cell, type CellResult, type Client, type ClientImage, type ClockPair, type Cloud, type Cluster, type ClusterClientAt, type ClusterLab, type ClusterStage, type ContextFile, type Crossings, type Cut, type Decision, type Drill, type DriverBand, type DriverSocket, type DriverStep, type DriverStepTime, type ErrorOnTheWay, type Fabric, type FabricLab, type FabricManifest, type FabricParts, type Fault, type FaultPlace, type FaultSide, type Group, type GroupClient, type GroupLab, type GroupMeasured, type GroupNetworkFault, type GroupPlace, type GroupPrediction, type GroupReading, type GroupRole, type GroupVerdict, type GroupView, type Hop, type HopEvent, type HopPlace, type HopSignal, type HopWrite, type Host, type HttpProbe, type ImagePlan, type ImageSource, type Interval, type Jitter, type KernelRecord, type Lab, type Layer, type LeafViews, type LineMatch, type Manifest, type ManifestHop, type Measured, type MemberLine, type MiddleboxEvent, type MiddleboxExpectation, type MiddleboxForm, type NetworkFault, type NodeLine, type Observer, type OplogRead, type OplogTime, type Outcome, type Pairing, type PinnedImage, type Prediction, type Probe, type ProbeEvent, type ProgramEvent, type ProxyHop, type ProxyTime, type ReconnectReading, type ReconnectVerdict, type Reconnects, type Recovers, type RecoveryMeasured, type Reference, type Reordering, type ReplicaSet, type ReplicaSetLab, type ReplicaSetStage, type Replies, type RequestPrediction, type RequestReading, type RouteLine, type RouterHop, type RttRange, type Run, type RunBackup, type RunDrill, type RunFabric, type RunGroup, type RunPart, type RunRoute, type RunTopology, type SentinelEvent, type SentinelStage, type Shipped, type ShippedImage, type SocketPlace, type Spread, type Stage, type StageBand, type StageVerdict, type Stamp, type Status, type StatusRow, type Subject, type Substrate, type SubstrateSource, type Target, type Topology, type Toxic, type Verdict, type WallStamp, type Writes, type WsProbe, isHopPlace, latestEndOf };
 //# sourceMappingURL=index.d.ts.map