@mark1russell7/harness 0.11.14 → 0.11.15

package/README.md

@@ -213,6 +213,10 @@ ioredis sets the keepalive of its socket itself, after it connects. Thus a cell
 - **A toxic** (`toxic`, `Cell.toxic`) is one toxic of Toxiproxy, added through its HTTP API by `inject http`, with monotonic stamps around the request. Its name is `fault`, and the harness removes it after the run. A cell whose fault is another one adds its toxic first: a slow close, then the close of the server.
 - **A close of the server** (`server-close`) makes the server close the connection of each client, with `Target.close`: `CLIENT KILL` for Redis.
 - **A stall of the application** (`app-stall`) injects nothing. The server process holds the reply, because the request asks for it (`HttpProbe.hold`), and its kernel stays up.
+- **An event at a proxy** (`hop-event`, `Cell.event`) is a signal to the program of the proxy, or a query on its console. Examples are a reload of the configuration and PAUSE on the console of PgBouncer.
+  - A signal goes to each process of `ProxyHop.program`. A sidecar in the network and process namespaces of the proxy sends it (`inject signal NAME HUP`). The lab starts that sidecar at the first signal.
+  - A query goes from the injector of the proxy to its console on 127.0.0.1, as a user that the console trusts (`inject query`). The injector speaks the simple query of the PostgreSQL protocol. Its stamps bracket the query and its ReadyForQuery. After the run, the harness sends the `undo` of the query, for example RESUME after PAUSE.
+  - Without `inFlight`, the event comes first, and the request meets it. With `inFlight`, the request goes first, and the event comes while it is in flight. A run whose order is not that order is void.
 - **A fault on a group** acts on the part that `Cell.faultAt` names by its role:
   - A black hole (`blackhole`) drops each segment between the part and each peer. A reset (`rst`) answers each segment from a peer with a reset, thus the part refuses each connection.
   - A partial partition (`partition`) is a black hole between the part and some of its peers (`GroupPlace.cuts`). `group` cuts the part from the other members and the Sentinels, and `clients` cuts it from the client.

package/build/{cell-COlAmUZ-.d.ts → cell-DOoU1LRf.d.ts}

@@ -30,6 +30,9 @@ import { KernelRecord } from "@mark1russell7/prober";
  *   target (`Target.close`): `CLIENT KILL` in Redis.
  * - `none` injects nothing. A calibration cell measures the path itself: for example the time of a
  *   proxy between its two sockets (`Cell.calibrates`).
+ * - `hop-event` is an event at a proxy of the topology (`Cell.event`): a signal to its program, or a
+ *   query on its console. Examples are a reload of the configuration (SIGHUP) and PAUSE on the
+ *   console of PgBouncer.
  *
  * A fault at a router of the topology (`Cell.faultAt`) acts on what the router forwards toward
  * one side. It is a black hole, a reset or an ICMP unreachable. It can also be a simplex drop, an
@@ -51,7 +54,7 @@ import { KernelRecord } from "@mark1russell7/prober";
  * This fault is of a replica set (`Cell.replicaSet`):
  * - `step-down` sends `replSetStepDown` to the primary, as a client sends a command.
  */
-type Fault = "blackhole" | "freeze-process" | "rst" | "stall" | "fin" | "unreachable" | "simplex" | "mtu-blackhole" | "netem" | "middlebox-forget" | "proxy-idle" | "toxic" | "server-close" | "partition" | "planned-failover" | "app-stall" | "step-down" | "none";
+type Fault = "blackhole" | "freeze-process" | "rst" | "stall" | "fin" | "unreachable" | "simplex" | "mtu-blackhole" | "netem" | "middlebox-forget" | "proxy-idle" | "toxic" | "server-close" | "partition" | "planned-failover" | "app-stall" | "step-down" | "hop-event" | "none";
 /**
  * A middlebox form tells what a router does with a flow that it forgot, when a late packet comes:
  * - `reset-both` sends a reset to the two ends when it forgets the flow.
@@ -115,7 +118,31 @@ interface ProxyHop {
   readonly sysctls?: Readonly<Record<string, string>>;
   /** These are the settings of the proxy, for the manifest: for example its idle timeouts. */
   readonly settings?: Readonly<Record<string, string>>;
+  /** This name is the process of the proxy, as /proc/PID/comm holds it, for a signal of a `hop-event`. */
+  readonly program?: string;
 }
+/**
+ * An event at a proxy hop (`hop-event`). A signal goes to each process of the program of the hop
+ * (`ProxyHop.program`). A query goes to the console of the hop from its own namespace, as a user
+ * that the console trusts. Its `undo` lifts it after the run. Without `inFlight`, the event
+ * comes first, and the request of the client meets it. With `inFlight`, the request goes first,
+ * and the event comes while it is in flight.
+ */
+type HopEvent = {
+  readonly hop: string;
+  readonly signal: "HUP";
+  readonly inFlight?: true;
+} | {
+  readonly hop: string;
+  readonly console: {
+    readonly port: number;
+    readonly user: string;
+    readonly database: string;
+    readonly query: string;
+    readonly undo?: string;
+  };
+  readonly inFlight?: true;
+};
 /**
  * A hop is one box between the client and the target. Each hop is on two networks: the network
  * of the side of the client, and the network of the side of the target.
@@ -828,6 +855,8 @@ interface Cell {
   readonly idleBefore?: Quantity;
   /** This toxic is the toxic of a `toxic` fault, which the Toxiproxy hop gets. */
   readonly toxic?: Toxic;
+  /** This event is the event of a `hop-event` fault, at a proxy of the topology. */
+  readonly event?: HopEvent;
   /** This image, or this client of the harness, replaces the TCP client of the harness for each run. An HTTP client is one. */
   readonly client?: ClientImage | Client;
   /**
@@ -1022,5 +1051,5 @@ interface RunTopology {
   }>>;
 }
 //#endregion
-export { isHopPlace as $, Recovers as A, RunPart as B, Observer as C, ProxyHop as D, ProbeEvent as E, RequestPrediction as F, ShippedImage as G, SentinelEvent as H, RouterHop as I, Subject as J, Stage as K, RttRange as L, ReplicaSet as M, ReplicaSetStage as N, ProxyTime as O, Replies as P, WsProbe as Q, Run as R, MiddleboxForm as S, Prediction as T, SentinelStage as U, RunTopology as V, Shipped as W, Topology as X, Target as Y, Toxic as Z, ImageSource as _, DriverBand as a, MiddleboxEvent as b, FaultPlace as c, GroupPrediction as d, latestEndOf as et, GroupRole as f, HttpProbe as g, HopPlace as h, ClientImage as i, Reference as j, Reconnects as k, Group as l, Hop as m, Cell as n, DriverStep as o, HarnessImage as p, StageBand as q, Client as r, Fault as s, BuildImage as t, GroupPlace as u, KernelRecord as v, PinnedImage as w, MiddleboxExpectation as x, LineMatch as y, RunGroup as z };
-//# sourceMappingURL=cell-COlAmUZ-.d.ts.map
+export { WsProbe as $, Reconnects as A, RunGroup as B, MiddleboxForm as C, ProbeEvent as D, Prediction as E, Replies as F, Shipped as G, RunTopology as H, RequestPrediction as I, StageBand as J, ShippedImage as K, RouterHop as L, Reference as M, ReplicaSet as N, ProxyHop as O, ReplicaSetStage as P, Toxic as Q, RttRange as R, MiddleboxExpectation as S, PinnedImage as T, SentinelEvent as U, RunPart as V, SentinelStage as W, Target as X, Subject as Y, Topology as Z, HttpProbe as _, DriverBand as a, LineMatch as b, FaultPlace as c, GroupPrediction as d, isHopPlace as et, GroupRole as f, HopPlace as g, HopEvent as h, ClientImage as i, Recovers as j, ProxyTime as k, Group as l, Hop as m, Cell as n, DriverStep as o, HarnessImage as p, Stage as q, Client as r, Fault as s, BuildImage as t, latestEndOf as tt, GroupPlace as u, ImageSource as v, Observer as w, MiddleboxEvent as x, KernelRecord as y, Run as z };
+//# sourceMappingURL=cell-DOoU1LRf.d.ts.map

package/build/index.d.ts

@@ -1,4 +1,4 @@
-import { $ as isHopPlace, A as Recovers, B as RunPart, C as Observer, D as ProxyHop, E as ProbeEvent, F as RequestPrediction, G as ShippedImage, H as SentinelEvent, I as RouterHop, J as Subject, K as Stage, L as RttRange, M as ReplicaSet, N as ReplicaSetStage, O as ProxyTime, P as Replies, Q as WsProbe, R as Run, S as MiddleboxForm, T as Prediction, U as SentinelStage, V as RunTopology, W as Shipped, X as Topology, Y as Target, Z as Toxic, _ as ImageSource, a as DriverBand, b as MiddleboxEvent, c as FaultPlace, d as GroupPrediction, et as latestEndOf, f as GroupRole, g as HttpProbe, h as HopPlace, i as ClientImage, j as Reference, k as Reconnects, l as Group, m as Hop, n as Cell, o as DriverStep, p as HarnessImage, q as StageBand, r as Client, s as Fault, t as BuildImage, u as GroupPlace, v as KernelRecord, w as PinnedImage, x as MiddleboxExpectation, y as LineMatch, z as RunGroup } from "./cell-COlAmUZ-.js";
+import { $ as WsProbe, A as Reconnects, B as RunGroup, C as MiddleboxForm, D as ProbeEvent, E as Prediction, F as Replies, G as Shipped, H as RunTopology, I as RequestPrediction, J as StageBand, K as ShippedImage, L as RouterHop, M as Reference, N as ReplicaSet, O as ProxyHop, P as ReplicaSetStage, Q as Toxic, R as RttRange, S as MiddleboxExpectation, T as PinnedImage, U as SentinelEvent, V as RunPart, W as SentinelStage, X as Target, Y as Subject, Z as Topology, _ as HttpProbe, a as DriverBand, b as LineMatch, c as FaultPlace, d as GroupPrediction, et as isHopPlace, f as GroupRole, g as HopPlace, h as HopEvent, i as ClientImage, j as Recovers, k as ProxyTime, l as Group, m as Hop, n as Cell, o as DriverStep, p as HarnessImage, q as Stage, r as Client, s as Fault, t as BuildImage, tt as latestEndOf, u as GroupPlace, v as ImageSource, w as Observer, x as MiddleboxEvent, y as KernelRecord, z as Run } from "./cell-DOoU1LRf.js";
 import { Quantity } from "@mark1russell7/quantity";
 import "testcontainers";
 //#region src/application.d.ts
@@ -911,6 +911,16 @@ interface Lab {
     t0: number;
     t1: number;
   }>;
+  /**
+   * This method applies an event at a proxy hop (`HopEvent`), or its undo. A signal goes from a
+   * sidecar in the network and process namespaces of the hop. The lab starts that sidecar at the
+   * first signal of the hop. A query goes from the injector of the hop to its console on 127.0.0.1. The
+   * injector stamps each one. An event with no undo gives null for its undo.
+   */
+  event(event: HopEvent, undo?: boolean): Promise<{
+    t0: number;
+    t1: number;
+  } | null>;
   probe(args: readonly string[]): Promise<Probe>;
   /**
    * This method waits for the next clock line of the prober, which gives the count of lost
@@ -1422,5 +1432,5 @@ export declare const spread: (runs: number, environments: number) => number[];
  */
 export declare const runCell: (cell: Cell, say?: (line: string) => void) => Promise<CellResult>;
 //#endregion
-export { type Agent, type BuildImage, type CallReading, type Cell, type CellResult, type Client, type ClientImage, type Crossings, type Cut, type Decision, type DriverBand, type DriverSocket, type DriverStep, type DriverStepTime, type ErrorOnTheWay, type Fault, type FaultPlace, type FaultSide, type Group, type GroupClient, type GroupLab, type GroupMeasured, type GroupNetworkFault, type GroupPlace, type GroupPrediction, type GroupReading, type GroupRole, type GroupVerdict, type GroupView, type HarnessImage, type Hop, type HopPlace, type HttpProbe, type ImagePlan, type ImageSource, type Interval, type KernelRecord, type Lab, type LineMatch, type Manifest, type ManifestHop, type Measured, type MemberLine, type MiddleboxEvent, type MiddleboxExpectation, type MiddleboxForm, type NetworkFault, type Observer, type Outcome, type PinnedImage, type Prediction, type Probe, type ProbeEvent, type ProgramEvent, type ProxyHop, type ProxyTime, type ReconnectReading, type ReconnectVerdict, type Reconnects, type Recovers, type Reference, type Reordering, type ReplicaSet, type ReplicaSetLab, type ReplicaSetStage, type Replies, type RequestPrediction, type RequestReading, type RouterHop, type RttRange, type Run, type RunGroup, type RunPart, type RunTopology, type SentinelEvent, type SentinelStage, type Shipped, type ShippedImage, type SocketPlace, type Spread, type Stage, type StageBand, type StageVerdict, type Stamp, type Status, type Subject, type Target, type Topology, type Toxic, type Verdict, type Writes, type WsProbe, isHopPlace, latestEndOf };
+export { type Agent, type BuildImage, type CallReading, type Cell, type CellResult, type Client, type ClientImage, type Crossings, type Cut, type Decision, type DriverBand, type DriverSocket, type DriverStep, type DriverStepTime, type ErrorOnTheWay, type Fault, type FaultPlace, type FaultSide, type Group, type GroupClient, type GroupLab, type GroupMeasured, type GroupNetworkFault, type GroupPlace, type GroupPrediction, type GroupReading, type GroupRole, type GroupVerdict, type GroupView, type HarnessImage, type Hop, type HopEvent, type HopPlace, type HttpProbe, type ImagePlan, type ImageSource, type Interval, type KernelRecord, type Lab, type LineMatch, type Manifest, type ManifestHop, type Measured, type MemberLine, type MiddleboxEvent, type MiddleboxExpectation, type MiddleboxForm, type NetworkFault, type Observer, type Outcome, type PinnedImage, type Prediction, type Probe, type ProbeEvent, type ProgramEvent, type ProxyHop, type ProxyTime, type ReconnectReading, type ReconnectVerdict, type Reconnects, type Recovers, type Reference, type Reordering, type ReplicaSet, type ReplicaSetLab, type ReplicaSetStage, type Replies, type RequestPrediction, type RequestReading, type RouterHop, type RttRange, type Run, type RunGroup, type RunPart, type RunTopology, type SentinelEvent, type SentinelStage, type Shipped, type ShippedImage, type SocketPlace, type Spread, type Stage, type StageBand, type StageVerdict, type Stamp, type Status, type Subject, type Target, type Topology, type Toxic, type Verdict, type Writes, type WsProbe, isHopPlace, latestEndOf };
 //# sourceMappingURL=index.d.ts.map

package/build/index.js

@@ -5,6 +5,11 @@ import { createHash } from "node:crypto";
 /** This function tells if a fault place is a hop of a topology. */
 const isHopPlace = (place) => typeof place === "object" && "hop" in place;
 /**
+* This function tells if the request of a client goes before the fault of its cell. That is so for a
+* hop event that comes while the request is in flight (`HopEvent.inFlight`). Then a request after the fault is void.
+*/
+const requestFirst = (cell) => cell.fault === "hop-event" && cell.event?.inFlight === true;
+/**
 * This function gives the latest end of a band in seconds, from the reply of the client. It is
 * the high bound, after the latest reference of the band (`startsBy`).
 */
@@ -898,7 +903,9 @@ const measureDriver = (cell, run) => {
 	let voided = null;
 	if (cell.probe.mode !== "idle" && cell.before !== true) {
 		if (request === void 0) voided = "the operation was never sent into the fault";
-		else if (run.fault !== null && run.fault.t1 > request.t) voided = "the operation went out before the fault held";
+		else if (requestFirst(cell)) {
+			if (run.fault !== null && run.fault.t0 < request.t) voided = "the event came before the operation";
+		} else if (run.fault !== null && run.fault.t1 > request.t) voided = "the operation went out before the fault held";
 	}
 	const keepalive = records.find((record) => record.ev === "probe" && record.what === "keepalive");
 	if (voided === null && run.fault !== null && keepalive !== void 0 && keepalive.mono < run.fault.t1) voided = "a keepalive probe went out before the fault held";
@@ -1705,7 +1712,7 @@ const measureSocket = (cell, run) => {
 	};
 	let voided = null;
 	if (run.fault !== null) {
-		const replyFirst = server && inflight && cell.fault !== "mtu-blackhole";
+		const replyFirst = server && inflight && cell.fault !== "mtu-blackhole" || requestFirst(cell);
 		const replied = events.some((event) => event.ev === "data" && event.t > request.t && event.t < run.fault.t1);
 		if (inflight && !replyFirst && run.fault.t1 > request.t) voided = "the request went out before the fault held";
 		else if (replyFirst && run.fault.t1 < request.t) voided = "the request went out into the fault, so the server never had it";
@@ -2429,6 +2436,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 		const paused = runtime.container.getById(server.getId());
 		let targetAdmin;
 		let signaller;
+		const hopSignallers = /* @__PURE__ */ new Map();
 		const hopNamed = (name) => {
 			const found = inChain.find((hop) => hop.hop.name === name);
 			if (found === void 0) throw new Error(`the topology has no hop ${name}`);
@@ -2704,6 +2712,38 @@ const openLab = async (target, kind = "tcp", topology) => {
 					...when
 				]);
 			},
+			async event(event, undo = false) {
+				const hop = hopNamed(event.hop);
+				if (hop.hop.kind !== "proxy") throw new Error(`the hop ${event.hop} is a router: an event goes to a proxy`);
+				if ("signal" in event) {
+					if (undo) return null;
+					const program = hop.hop.program;
+					if (program === void 0) throw new Error(`the hop ${event.hop} names no program to signal`);
+					let sidecar = hopSignallers.get(event.hop);
+					if (sidecar === void 0) {
+						sidecar = await withPid(new GenericContainer(image).withNetworkMode(`container:${hop.box.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW"), `container:${hop.box.getId()}`).start();
+						hopSignallers.set(event.hop, sidecar);
+						opened.push(sidecar);
+					}
+					return stamped(sidecar, [
+						"/opt/harness/inject",
+						"signal",
+						program,
+						event.signal
+					]);
+				}
+				const query = undo ? event.console.undo : event.console.query;
+				if (query === void 0) return null;
+				const { port, user, database } = event.console;
+				return stamped(hop.injector, [
+					"/opt/harness/inject",
+					"query",
+					`127.0.0.1:${port}`,
+					user,
+					database,
+					query
+				]);
+			},
 			async probe(args) {
 				const events = lines();
 				const container = await new GenericContainer(clientImage).withNetworkMode(`container:${client.getId()}`).withCommand([...command, ...args]).withLogConsumer(events.consume).start();
@@ -4396,7 +4436,7 @@ const once = async (lab, cell, environment, index, dry = false) => {
 			if (cell.probe.mode !== "idle") await go();
 			await sleep(500);
 		} else {
-			const replyInFlight = cell.probe.mode === "inflight" && (server && cell.fault !== "mtu-blackhole" || cell.fault === "fin");
+			const replyInFlight = cell.probe.mode === "inflight" && (server && cell.fault !== "mtu-blackhole" || cell.fault === "fin" || requestFirst(cell));
 			if (replyInFlight) {
 				await go();
 				await sleep(200);
@@ -4447,6 +4487,9 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					case "server-close":
 						await lab.serverClose();
 						return null;
+					case "hop-event":
+						if (cell.event === void 0) throw new Error(`${cell.name} names no event`);
+						return lab.event(cell.event);
 					default: return lab.fault("blackhole", faultAt);
 				}
 			};
@@ -4465,6 +4508,9 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					case "server-close":
 					case "app-stall":
 					case "none": return;
+					case "hop-event":
+						if (cell.event !== void 0) await lab.event(cell.event, true);
+						return;
 					case "unreachable": return lab.lift("reject", faultAt, { code: cell.icmp ?? "host" });
 					case "simplex": return lab.lift("blackhole", faultAt, options);
 					case "mtu-blackhole": return lab.lift("mtu", cell.faultAt ?? "client", { above: cell.mtu ?? 1200 });