@mark1russell7/harness 0.11.23 → 0.11.25

package/README.md

@@ -250,7 +250,10 @@ ioredis sets the keepalive of its socket itself, after it connects. Thus a cell
 - **A close of the server** (`server-close`) makes the server close the connection of each client, with `Target.close`: `CLIENT KILL` for Redis.
 - **A stall of the application** (`app-stall`) injects nothing. The server process holds the reply, because the request asks for it (`HttpProbe.hold`), and its kernel stays up.
 - **An event at a proxy** (`hop-event`, `Cell.event`) is a signal to the program of the proxy, or a query on its console. Examples are a reload of the configuration, a stop of a proxy and PAUSE on the console of PgBouncer.
-  - A signal goes to each process of `ProxyHop.program`: HUP, TERM or INT (`HopSignal`). A sidecar in the network and process namespaces of the proxy sends it (`inject signal NAME HUP`). The lab starts that sidecar at the first signal.
+  - A signal goes to each process of `ProxyHop.program`: HUP, TERM, INT, STOP or CONT (`HopSignal`). A sidecar in the network and process namespaces of the proxy sends it (`inject signal NAME HUP`). The lab starts that sidecar at the first signal.
+  - STOP blocks the event loop of the proxy: its processes stop, and its kernel keeps each socket. With a hold (`Cell.hold`), CONT thaws it at that time after the stop, from inside the VM (`--at`). The lift thaws a proxy that its hold did not thaw.
+  - A write (`HopWrite`) changes a file of the proxy, as an edit of its configuration does. The sidecar writes it through the root of the program (`inject write NAME PATH CONTENT`): first a file beside it, then a rename in one step. Thus a program that watches the folder reads the whole file.
+  - A write can name a line of the output of the proxy that the change gives (`confirm`), for example the log line of a reload. The event waits 10 s at most for a new such line, else the run fails. Its `undo` writes the first content back after the run.
   - A query goes from the injector of the proxy to its console on 127.0.0.1, as a user that the console trusts (`inject query`). The injector speaks the simple query of the PostgreSQL protocol. Its stamps bracket the query and its ReadyForQuery. After the run, the harness sends the `undo` of the query, for example RESUME after PAUSE.
   - Without `inFlight`, the event comes first, and the request meets it. With `inFlight`, the request goes first, and the event comes while it is in flight. A run whose order is not that order is void.
 - **A fault on a group** acts on the part that `Cell.faultAt` names by its role:

package/build/{cell-Dn1TvJcA.d.ts → cell-D4HZ1off.d.ts}

@@ -130,14 +130,20 @@ interface ProxyHop {
 /**
  * A signal of a hop event. HUP reloads the configuration of a pooler. TERM stops a proxy, as a stop of
  * its container does. INT stops docker-proxy, as Docker does when it removes a port binding.
+ *
+ * STOP blocks the event loop of a proxy: its processes stop, and its kernel keeps each socket. Thus
+ * ACKs go on, and the data of each peer waits in its receive queue. With the hold of the cell
+ * (`Cell.hold`), CONT thaws it at the end of the hold, from inside the VM.
  */
-type HopSignal = "HUP" | "TERM" | "INT";
+type HopSignal = "HUP" | "TERM" | "INT" | "STOP" | "CONT";
 /**
  * An event at a proxy hop (`hop-event`). A signal goes to each process of the program of the hop
  * (`ProxyHop.program`). A query goes to the console of the hop from its own namespace, as a user
  * that the console trusts. Its `undo` lifts it after the run. Without `inFlight`, the event
  * comes first, and the request of the client meets it. With `inFlight`, the request goes first,
  * and the event comes while it is in flight.
+ *
+ * A write changes a file of the hop, as an edit of its configuration does (`HopWrite`).
  */
 type HopEvent = {
   readonly hop: string;
@@ -153,7 +159,27 @@ type HopEvent = {
     readonly undo?: string;
   };
   readonly inFlight?: true;
+} | {
+  readonly hop: string;
+  readonly write: HopWrite;
+  readonly inFlight?: true;
 };
+/**
+ * A hop write puts `content` at `path` in the root of the program of the hop. The content goes to a
+ * file beside it first, and a rename puts it in place in one step. Thus a program that watches the
+ * folder reads the whole file, not half of it. Its `undo` writes the file of the start again
+ * after the run, thus each run makes the same change.
+ *
+ * `confirm` is a line of the output of the hop that the change gives, for example the log line of
+ * a reload. The event waits for a new such line, and a write that the hop does not confirm in 10 s
+ * fails the run. Thus a cell that predicts no effect knows that the change took effect.
+ */
+interface HopWrite {
+  readonly path: string;
+  readonly content: string;
+  readonly undo?: string;
+  readonly confirm?: string;
+}
 /**
  * A hop is one box between the client and the target. Each hop is on two networks: the network
  * of the side of the client, and the network of the side of the target.
@@ -1127,5 +1153,5 @@ interface RunTopology {
   }>>;
 }
 //#endregion
-export { Subject as $, Prediction as A, RequestPrediction as B, KernelRecord as C, MiddleboxForm as D, MiddleboxExpectation as E, Recovers as F, RunPart as G, RttRange as H, Reference as I, SentinelStage as J, RunTopology as K, ReplicaSet as L, ProxyHop as M, ProxyTime as N, Observer as O, Reconnects as P, StageBand as Q, ReplicaSetStage as R, ImageSource as S, MiddleboxEvent as T, Run as U, RouterHop as V, RunGroup as W, ShippedImage as X, Shipped as Y, Stage as Z, Hop as _, Cluster as a, latestEndOf as at, HopSignal as b, DriverBand as c, FaultPlace as d, Target as et, Group as f, HarnessImage as g, GroupRole as h, ClientImage as i, isHopPlace as it, ProbeEvent as j, PinnedImage as k, DriverStep as l, GroupPrediction as m, Cell as n, Toxic as nt, ClusterClientAt as o, GroupPlace as p, SentinelEvent as q, Client as r, WsProbe as rt, ClusterStage as s, BuildImage as t, Topology as tt, Fault as u, HopEvent as v, LineMatch as w, HttpProbe as x, HopPlace as y, Replies as z };
-//# sourceMappingURL=cell-Dn1TvJcA.d.ts.map
+export { StageBand as $, PinnedImage as A, Replies as B, ImageSource as C, MiddleboxExpectation as D, MiddleboxEvent as E, Reconnects as F, RunGroup as G, RouterHop as H, Recovers as I, SentinelEvent as J, RunPart as K, Reference as L, ProbeEvent as M, ProxyHop as N, MiddleboxForm as O, ProxyTime as P, Stage as Q, ReplicaSet as R, HttpProbe as S, LineMatch as T, RttRange as U, RequestPrediction as V, Run as W, Shipped as X, SentinelStage as Y, ShippedImage as Z, Hop as _, Cluster as a, isHopPlace as at, HopSignal as b, DriverBand as c, FaultPlace as d, Subject as et, Group as f, HarnessImage as g, GroupRole as h, ClientImage as i, WsProbe as it, Prediction as j, Observer as k, DriverStep as l, GroupPrediction as m, Cell as n, Topology as nt, ClusterClientAt as o, latestEndOf as ot, GroupPlace as p, RunTopology as q, Client as r, Toxic as rt, ClusterStage as s, BuildImage as t, Target as tt, Fault as u, HopEvent as v, KernelRecord as w, HopWrite as x, HopPlace as y, ReplicaSetStage as z };
+//# sourceMappingURL=cell-D4HZ1off.d.ts.map

package/build/index.d.ts

@@ -1,4 +1,4 @@
-import { $ as Subject, A as Prediction, B as RequestPrediction, C as KernelRecord, D as MiddleboxForm, E as MiddleboxExpectation, F as Recovers, G as RunPart, H as RttRange, I as Reference, J as SentinelStage, K as RunTopology, L as ReplicaSet, M as ProxyHop, N as ProxyTime, O as Observer, P as Reconnects, Q as StageBand, R as ReplicaSetStage, S as ImageSource, T as MiddleboxEvent, U as Run, V as RouterHop, W as RunGroup, X as ShippedImage, Y as Shipped, Z as Stage, _ as Hop, a as Cluster, at as latestEndOf, b as HopSignal, c as DriverBand, d as FaultPlace, et as Target, f as Group, g as HarnessImage, h as GroupRole, i as ClientImage, it as isHopPlace, j as ProbeEvent, k as PinnedImage, l as DriverStep, m as GroupPrediction, n as Cell, nt as Toxic, o as ClusterClientAt, p as GroupPlace, q as SentinelEvent, r as Client, rt as WsProbe, s as ClusterStage, t as BuildImage, tt as Topology, u as Fault, v as HopEvent, w as LineMatch, x as HttpProbe, y as HopPlace, z as Replies } from "./cell-Dn1TvJcA.js";
+import { $ as StageBand, A as PinnedImage, B as Replies, C as ImageSource, D as MiddleboxExpectation, E as MiddleboxEvent, F as Reconnects, G as RunGroup, H as RouterHop, I as Recovers, J as SentinelEvent, K as RunPart, L as Reference, M as ProbeEvent, N as ProxyHop, O as MiddleboxForm, P as ProxyTime, Q as Stage, R as ReplicaSet, S as HttpProbe, T as LineMatch, U as RttRange, V as RequestPrediction, W as Run, X as Shipped, Y as SentinelStage, Z as ShippedImage, _ as Hop, a as Cluster, at as isHopPlace, b as HopSignal, c as DriverBand, d as FaultPlace, et as Subject, f as Group, g as HarnessImage, h as GroupRole, i as ClientImage, it as WsProbe, j as Prediction, k as Observer, l as DriverStep, m as GroupPrediction, n as Cell, nt as Topology, o as ClusterClientAt, ot as latestEndOf, p as GroupPlace, q as RunTopology, r as Client, rt as Toxic, s as ClusterStage, t as BuildImage, tt as Target, u as Fault, v as HopEvent, w as KernelRecord, x as HopWrite, y as HopPlace, z as ReplicaSetStage } from "./cell-D4HZ1off.js";
 import { Quantity } from "@mark1russell7/quantity";
 import "testcontainers";
 //#region src/application.d.ts
@@ -941,9 +941,10 @@ interface Lab {
    * This method applies an event at a proxy hop (`HopEvent`), or its undo. A signal goes from a
    * sidecar in the network and process namespaces of the hop. The lab starts that sidecar at the
    * first signal of the hop. A query goes from the injector of the hop to its console on 127.0.0.1. The
-   * injector stamps each one. An event with no undo gives null for its undo.
+   * injector stamps each one. An event with no undo gives null for its undo. A signal can wait for an
+   * instant on CLOCK_MONOTONIC (`at`), as the thaw at the end of a hold does.
    */
-  event(event: HopEvent, undo?: boolean): Promise<{
+  event(event: HopEvent, undo?: boolean, at?: number): Promise<{
     t0: number;
     t1: number;
   } | null>;
@@ -1632,5 +1633,5 @@ export declare const spread: (runs: number, environments: number) => number[];
  */
 export declare const runCell: (cell: Cell, say?: (line: string) => void) => Promise<CellResult>;
 //#endregion
-export { type Agent, type BuildImage, type CallReading, type Cell, type CellResult, type Client, type ClientImage, type Cluster, type ClusterClientAt, type ClusterLab, type ClusterStage, type Crossings, type Cut, type Decision, type DriverBand, type DriverSocket, type DriverStep, type DriverStepTime, type ErrorOnTheWay, type Fault, type FaultPlace, type FaultSide, type Group, type GroupClient, type GroupLab, type GroupMeasured, type GroupNetworkFault, type GroupPlace, type GroupPrediction, type GroupReading, type GroupRole, type GroupVerdict, type GroupView, type HarnessImage, type Hop, type HopEvent, type HopPlace, type HopSignal, type HttpProbe, type ImagePlan, type ImageSource, type Interval, type KernelRecord, type Lab, type LineMatch, type Manifest, type ManifestHop, type Measured, type MemberLine, type MiddleboxEvent, type MiddleboxExpectation, type MiddleboxForm, type NetworkFault, type NodeLine, type Observer, type Outcome, type PinnedImage, type Prediction, type Probe, type ProbeEvent, type ProgramEvent, type ProxyHop, type ProxyTime, type ReconnectReading, type ReconnectVerdict, type Reconnects, type Recovers, type Reference, type Reordering, type ReplicaSet, type ReplicaSetLab, type ReplicaSetStage, type Replies, type RequestPrediction, type RequestReading, type RouterHop, type RttRange, type Run, type RunGroup, type RunPart, type RunTopology, type SentinelEvent, type SentinelStage, type Shipped, type ShippedImage, type SocketPlace, type Spread, type Stage, type StageBand, type StageVerdict, type Stamp, type Status, type StatusRow, type Subject, type Target, type Topology, type Toxic, type Verdict, type WallStamp, type Writes, type WsProbe, isHopPlace, latestEndOf };
+export { type Agent, type BuildImage, type CallReading, type Cell, type CellResult, type Client, type ClientImage, type Cluster, type ClusterClientAt, type ClusterLab, type ClusterStage, type Crossings, type Cut, type Decision, type DriverBand, type DriverSocket, type DriverStep, type DriverStepTime, type ErrorOnTheWay, type Fault, type FaultPlace, type FaultSide, type Group, type GroupClient, type GroupLab, type GroupMeasured, type GroupNetworkFault, type GroupPlace, type GroupPrediction, type GroupReading, type GroupRole, type GroupVerdict, type GroupView, type HarnessImage, type Hop, type HopEvent, type HopPlace, type HopSignal, type HopWrite, type HttpProbe, type ImagePlan, type ImageSource, type Interval, type KernelRecord, type Lab, type LineMatch, type Manifest, type ManifestHop, type Measured, type MemberLine, type MiddleboxEvent, type MiddleboxExpectation, type MiddleboxForm, type NetworkFault, type NodeLine, type Observer, type Outcome, type PinnedImage, type Prediction, type Probe, type ProbeEvent, type ProgramEvent, type ProxyHop, type ProxyTime, type ReconnectReading, type ReconnectVerdict, type Reconnects, type Recovers, type Reference, type Reordering, type ReplicaSet, type ReplicaSetLab, type ReplicaSetStage, type Replies, type RequestPrediction, type RequestReading, type RouterHop, type RttRange, type Run, type RunGroup, type RunPart, type RunTopology, type SentinelEvent, type SentinelStage, type Shipped, type ShippedImage, type SocketPlace, type Spread, type Stage, type StageBand, type StageVerdict, type Stamp, type Status, type StatusRow, type Subject, type Target, type Topology, type Toxic, type Verdict, type WallStamp, type Writes, type WsProbe, isHopPlace, latestEndOf };
 //# sourceMappingURL=index.d.ts.map

package/build/index.js

@@ -2092,6 +2092,11 @@ const PUMP_SLACK = {
 };
 /** This count is how many of the last lines of a proxy hop the lab keeps. The error of a hop that does not start gives them. */
 const TAIL_LINES = 60;
+/** This count is how many lines of the output of a proxy hop the lab keeps, for a write that the proxy confirms. */
+const OUTPUT_LINES = 2e3;
+/** This time is how long a write of a hop waits for the line that confirms it, in milliseconds. */
+const CONFIRM_MS = 1e4;
+const sleep$4 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** This function gives the arguments of the injector for the options of a fault. */
 const optionArgs = (options) => [
 	...options.direction === void 0 ? [] : ["--direction", options.direction],
@@ -2328,6 +2333,7 @@ const openLab = async (target, kind = "tcp", topology) => {
 		opened.push(server);
 		const serverIp = await ipOn(server, last);
 		const middlebox = [];
+		const hopOutput = /* @__PURE__ */ new Map();
 		const inChain = [];
 		const endpointOf = (element) => {
 			const at = endpointAfter(hops, element);
@@ -2386,6 +2392,11 @@ const openLab = async (target, kind = "tcp", topology) => {
 			})));
 			if (hop.ready !== void 0) proxy = proxy.withWaitStrategy(Wait.forLogMessage(hop.ready));
 			const tail = [];
+			const output = {
+				lines: [],
+				total: 0
+			};
+			hopOutput.set(hop.name, output);
 			proxy = proxy.withLogConsumer((stream) => {
 				let rest = "";
 				stream.on("data", (chunk) => {
@@ -2394,6 +2405,9 @@ const openLab = async (target, kind = "tcp", topology) => {
 					for (const line of lines) {
 						tail.push(line);
 						if (tail.length > TAIL_LINES) tail.shift();
+						output.lines.push(line);
+						output.total += 1;
+						if (output.lines.length > OUTPUT_LINES) output.lines.shift();
 					}
 				});
 			});
@@ -2803,25 +2817,58 @@ const openLab = async (target, kind = "tcp", topology) => {
 					...when
 				]);
 			},
-			async event(event, undo = false) {
+			async event(event, undo = false, at) {
 				const hop = hopNamed(event.hop);
 				if (hop.hop.kind !== "proxy") throw new Error(`the hop ${event.hop} is a router: an event goes to a proxy`);
-				if ("signal" in event) {
-					if (undo) return null;
-					const program = hop.hop.program;
-					if (program === void 0) throw new Error(`the hop ${event.hop} names no program to signal`);
+				const program = hop.hop.program;
+				const sidecarOf = async () => {
 					let sidecar = hopSignallers.get(event.hop);
 					if (sidecar === void 0) {
-						sidecar = await withPid(new GenericContainer(image).withNetworkMode(`container:${hop.box.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW"), `container:${hop.box.getId()}`).start();
+						sidecar = await withPid(new GenericContainer(image).withNetworkMode(`container:${hop.box.getId()}`).withAddedCapabilities("NET_ADMIN", "NET_RAW", "SYS_PTRACE"), `container:${hop.box.getId()}`).start();
 						hopSignallers.set(event.hop, sidecar);
 						opened.push(sidecar);
 					}
-					return stamped(sidecar, [
+					return sidecar;
+				};
+				if ("signal" in event) {
+					if (undo) return null;
+					if (program === void 0) throw new Error(`the hop ${event.hop} names no program to signal`);
+					const when = at === void 0 ? [] : ["--at", String(Math.round(at))];
+					return stamped(await sidecarOf(), [
 						"/opt/harness/inject",
 						"signal",
 						program,
-						event.signal
+						event.signal,
+						...when
+					]);
+				}
+				if ("write" in event) {
+					const content = undo ? event.write.undo : event.write.content;
+					if (content === void 0) return null;
+					if (program === void 0) throw new Error(`the hop ${event.hop} names no program whose root a write reaches`);
+					const output = hopOutput.get(event.hop);
+					const before = output?.total ?? 0;
+					const stamp = await stamped(await sidecarOf(), [
+						"/opt/harness/inject",
+						"write",
+						program,
+						event.write.path,
+						content
 					]);
+					const confirm = event.write.confirm;
+					if (confirm !== void 0) {
+						const started = Date.now();
+						const confirmed = () => {
+							if (output === void 0) return false;
+							const fresh = Math.min(output.total - before, output.lines.length);
+							return output.lines.slice(output.lines.length - fresh).some((line) => line.includes(confirm));
+						};
+						while (!confirmed()) {
+							if (Date.now() - started > CONFIRM_MS) throw new Error(`the hop ${event.hop} did not confirm the write of ${event.write.path}: no line "${confirm}" in ${CONFIRM_MS / 1e3} s`);
+							await sleep$4(50);
+						}
+					}
+					return stamp;
 				}
 				const query = undo ? event.console.undo : event.console.query;
 				if (query === void 0) return null;
@@ -5324,9 +5371,23 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					case "server-close":
 						await lab.serverClose();
 						return null;
-					case "hop-event":
+					case "hop-event": {
 						if (cell.event === void 0) throw new Error(`${cell.name} names no event`);
-						return lab.event(cell.event);
+						const stamp = await lab.event(cell.event);
+						if ("signal" in cell.event && cell.event.signal === "STOP" && cell.hold !== void 0 && stamp !== null) {
+							const thaw = {
+								hop: cell.event.hop,
+								signal: "CONT"
+							};
+							thawing = lab.event(thaw, false, stamp.t0 + valueIn(cell.hold, "s") * 1e9).then((thawed) => {
+								if (thawed === null) throw new Error(`the hop ${thaw.hop} gave no stamp for CONT`);
+								heal = thawed;
+								return thawed;
+							});
+							thawing.catch(() => void 0);
+						}
+						return stamp;
+					}
 					default: return lab.fault("blackhole", faultAt);
 				}
 			};
@@ -5346,6 +5407,15 @@ const once = async (lab, cell, environment, index, dry = false) => {
 					case "app-stall":
 					case "none": return;
 					case "hop-event":
+						if (cell.event !== void 0 && "signal" in cell.event && cell.event.signal === "STOP") {
+							const thaw = {
+								hop: cell.event.hop,
+								signal: "CONT"
+							};
+							if (thawing !== null) await thawing.catch(() => lab.event(thaw));
+							else await lab.event(thaw);
+							return;
+						}
 						if (cell.event !== void 0) await lab.event(cell.event, true);
 						return;
 					case "unreachable": return lab.lift("reject", faultAt, { code: cell.icmp ?? "host" });