@mark1russell7/harness 0.4.0 → 0.5.0

package/image/native/src/bin/inject.rs

@@ -27,6 +27,17 @@
 //!   toxic of Toxiproxy. The stamps bracket the request and its answer, after the connect, so the
 //!   commit of the toxic is known on the same clock as the kernel's records.
 //!
+//! - `signal NAME SIGNAL`: SIGNAL (KILL, STOP, CONT or TERM) to each process whose name is NAME,
+//!   in the process namespace this sidecar shares with a member of a group. STOP and CONT freeze
+//!   a server and thaw it while its kernel stays up; KILL stops it, and its kernel closes its
+//!   sockets. The stamps bracket the kill(2) calls.
+//! - `resp ADDR:PORT ARG...`: one command to a Redis-protocol server, for example
+//!   `SENTINEL FAILOVER mymaster` to a Sentinel. The stamps bracket the write of the command and
+//!   the first line of its reply, so a planned failover starts at a known instant.
+//!
+//! A filter rule's PEER can be a list, `A,B,C`: the rules for each peer go in the same commit, so
+//! a part of a group loses all of its peers at one instant.
+//!
 //! `--at NS` holds the commit until that instant on CLOCK_MONOTONIC, the clock every stamp is on.
 //! A stall's end is timed that way, from inside the VM: the exec only has to arrive before it, and
 //! its 190 ms never reach the stall's length.
@@ -234,12 +245,139 @@ fn http(args: &[String]) -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
+/// the number of a signal that `signal` sends, by its name
+fn signal_number(name: &str) -> Result<libc::c_int, Box<dyn Error>> {
+    Ok(match name {
+        "KILL" => libc::SIGKILL,
+        "STOP" => libc::SIGSTOP,
+        "CONT" => libc::SIGCONT,
+        "TERM" => libc::SIGTERM,
+        other => return Err(format!("not a signal this injector sends: {other}").into()),
+    })
+}
+
+/// a process name: letters, digits, dots, dashes and underscores, as /proc/PID/comm holds it
+fn process_name(name: &str) -> Result<&str, Box<dyn Error>> {
+    if !name.is_empty()
+        && name.len() <= 15
+        && name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_'))
+    {
+        Ok(name)
+    } else {
+        Err(format!("not a process name: {name}").into())
+    }
+}
+
+/// tells if a line of /proc/PID/stat is a zombie's: its state, after the name in parentheses, is Z
+fn zombie(stat: &str) -> bool {
+    stat.rfind(')').and_then(|end| stat[end + 1..].split_whitespace().next()) == Some("Z")
+}
+
+/// the processes of this process namespace whose name is `name`
+fn processes(name: &str) -> Result<Vec<libc::pid_t>, Box<dyn Error>> {
+    let mut found = Vec::new();
+    for entry in std::fs::read_dir("/proc")? {
+        let entry = entry?;
+        let Some(pid) = entry.file_name().to_str().and_then(|text| text.parse::<libc::pid_t>().ok()) else {
+            continue;
+        };
+        // a process can end between the listing and the read. A zombie keeps its name until its
+        // parent reaps it: a server that a fin killed is one, and no signal reaches it
+        let Ok(comm) = std::fs::read_to_string(entry.path().join("comm")) else { continue };
+        let Ok(stat) = std::fs::read_to_string(entry.path().join("stat")) else { continue };
+        if comm.trim_end() == name && !zombie(&stat) {
+            found.push(pid);
+        }
+    }
+    found.sort_unstable();
+    Ok(found)
+}
+
+/// `signal NAME SIGNAL [--at NS]`: the signal to each process of that name, bracketed by stamps
+fn signal(args: &[String]) -> Result<(), Box<dyn Error>> {
+    let name = process_name(args.get(1).ok_or("signal NAME SIGNAL [--at NS]")?)?;
+    let sig = signal_number(args.get(2).ok_or("and the signal: KILL, STOP, CONT or TERM")?)?;
+    let opts = options(&args[3..])?;
+    let pids = processes(name)?;
+    if pids.is_empty() {
+        return Err(format!("no process {name} in this process namespace").into());
+    }
+    if let Some(at) = opts.at {
+        until(at)?;
+    }
+    let t0 = monotonic_ns();
+    for pid in &pids {
+        // SAFETY: kill(2) with a pid that this namespace listed, and a signal from the table above
+        if unsafe { libc::kill(*pid, sig) } != 0 {
+            return Err(format!("kill {pid} failed: {}", std::io::Error::last_os_error()).into());
+        }
+    }
+    let t1 = monotonic_ns();
+    println!("{}", json!({ "action": "signal", "name": name, "signal": args[2], "pids": pids, "t0": t0, "t1": t1 }));
+    Ok(())
+}
+
+/// a command in the Redis protocol: an array of bulk strings
+fn resp_command(args: &[String]) -> Vec<u8> {
+    let mut out = format!("*{}\r\n", args.len()).into_bytes();
+    for arg in args {
+        out.extend_from_slice(format!("${}\r\n", arg.len()).as_bytes());
+        out.extend_from_slice(arg.as_bytes());
+        out.extend_from_slice(b"\r\n");
+    }
+    out
+}
+
+/// `resp ADDR:PORT ARG...`: one command, and the first line of its reply, bracketed by stamps
+fn resp(args: &[String]) -> Result<(), Box<dyn Error>> {
+    use std::io::Read;
+    use std::net::{SocketAddrV4, TcpStream};
+    let addr: SocketAddrV4 = args.get(1).ok_or("resp ADDR:PORT ARG...")?.parse()?;
+    let command = &args[2..];
+    if command.is_empty() {
+        return Err("and the command".into());
+    }
+    let mut stream = TcpStream::connect(addr)?;
+    let t0 = monotonic_ns();
+    stream.write_all(&resp_command(command))?;
+    let mut line = Vec::new();
+    let mut byte = [0u8; 1];
+    while !line.ends_with(b"\r\n") {
+        if stream.read(&mut byte)? == 0 {
+            return Err("the server closed before its reply".into());
+        }
+        line.push(byte[0]);
+    }
+    let t1 = monotonic_ns();
+    let reply = String::from_utf8_lossy(&line[..line.len() - 2]).to_string();
+    if reply.starts_with('-') {
+        return Err(format!("{} answered {reply}", command.join(" ")).into());
+    }
+    println!("{}", json!({ "action": "resp", "command": command, "reply": reply, "t0": t0, "t1": t1 }));
+    Ok(())
+}
+
+/// the peers of a filter rule: one address, or a list `A,B,C`
+fn peers(text: &str) -> Result<Vec<Ipv4Addr>, Box<dyn Error>> {
+    let list: Vec<Ipv4Addr> = text.split(',').map(str::parse).collect::<Result<_, _>>()?;
+    if list.is_empty() {
+        return Err("no peer".into());
+    }
+    Ok(list)
+}
+
 fn run() -> Result<(), Box<dyn Error>> {
     let args: Vec<String> = std::env::args().skip(1).collect();
     let action = args.first().ok_or("say an action: blackhole, clear, rst, reject, mtu, netem, offload, http …")?.as_str();
     if action == "http" {
         return http(&args);
     }
+    if action == "signal" {
+        return signal(&args);
+    }
+    if action == "resp" {
+        return resp(&args);
+    }
     match action {
         "netem" | "clear-netem" | "offload" => {
             let dev = device(args.get(1).ok_or("and a device")?)?;
@@ -279,10 +417,14 @@ fn run() -> Result<(), Box<dyn Error>> {
             Ok(())
         }
         _ => {
-            // parsed, never pasted: only an address reaches the rules
-            let peer: Ipv4Addr = args.get(1).ok_or("and the peer's address")?.parse()?;
+            // parsed, never pasted: only addresses reach the rules
+            let list = peers(args.get(1).ok_or("and the peer's address")?)?;
             let opts = options(&args[2..])?;
-            let script = format!("*filter\n{}COMMIT\n", rules(action, peer, &opts)?);
+            let mut body = String::new();
+            for peer in &list {
+                body.push_str(&rules(action, *peer, &opts)?);
+            }
+            let script = format!("*filter\n{body}COMMIT\n");
             if let Some(at) = opts.at {
                 until(at)?;
             }
@@ -295,7 +437,8 @@ fn run() -> Result<(), Box<dyn Error>> {
             if !status.success() {
                 return Err(format!("iptables-restore exited {status}").into());
             }
-            println!("{}", json!({ "action": action, "peer": peer.to_string(), "t0": t0, "t1": t1 }));
+            let named: Vec<String> = list.iter().map(ToString::to_string).collect();
+            println!("{}", json!({ "action": action, "peer": named.join(","), "t0": t0, "t1": t1 }));
             Ok(())
         }
     }
@@ -365,6 +508,41 @@ mod tests {
         assert_eq!(options(&["--via".into(), "eth1".into()]).unwrap().via.as_deref(), Some("eth1"));
     }
 
+    #[test]
+    fn a_list_of_peers_is_addresses_only() {
+        assert_eq!(peers("10.0.0.2").unwrap().len(), 1);
+        assert_eq!(peers("10.0.0.2,10.0.0.3,10.0.0.4").unwrap()[2].to_string(), "10.0.0.4");
+        for text in ["", "10.0.0.2,", "10.0.0.2;x", "a,b", "10.0.0.2 10.0.0.3"] {
+            assert!(peers(text).is_err(), "{text}");
+        }
+    }
+
+    #[test]
+    fn signals_and_process_names_come_from_a_short_list() {
+        assert_eq!(signal_number("STOP").unwrap(), libc::SIGSTOP);
+        assert_eq!(signal_number("CONT").unwrap(), libc::SIGCONT);
+        assert_eq!(signal_number("KILL").unwrap(), libc::SIGKILL);
+        assert!(signal_number("HUP").is_err());
+        assert!(process_name("valkey-server").is_ok());
+        assert!(process_name("valkey-sentinel").is_ok());
+        for name in ["", "a b", "x;y", "much-too-long-a-name"] {
+            assert!(process_name(name).is_err(), "{name}");
+        }
+        // the state is the field after the name, whose parentheses can hold spaces
+        assert!(zombie("812 (valkey-server) Z 1 812 812 0 -1"));
+        assert!(!zombie("813 (valkey-server) S 1 813 813 0 -1"));
+        assert!(!zombie("814 (a) Z) S 1"));
+    }
+
+    #[test]
+    fn a_resp_command_is_an_array_of_bulk_strings() {
+        let args: Vec<String> = ["SENTINEL", "FAILOVER", "mymaster"].iter().map(|a| a.to_string()).collect();
+        assert_eq!(
+            resp_command(&args),
+            b"*3\r\n$8\r\nSENTINEL\r\n$8\r\nFAILOVER\r\n$8\r\nmymaster\r\n".to_vec()
+        );
+    }
+
     #[test]
     fn http_requests_take_only_a_method_a_path_and_a_length() {
         assert!(http_path("/proxies/redis/toxics").is_ok());

package/image/node/Dockerfile

@@ -1,6 +1,7 @@
 # The harness's Node client: ioredis 6.0.0 on Node 24.21.0, the releases the corpus's claims about
 # ioredis and Node's keepalive name. It runs as a container in the client's
-# network namespace, as the harness's own TCP client does, and prints the same JSON lines.
+# network namespace, as the harness's own TCP client does, and prints the same JSON lines. The same
+# image holds a group's client in Sentinel mode and the observer of its Sentinels.
 # lab.ts stages the build context: this folder, under harness/image/node. The lockfile is
 # npm-lock.json here, because npm leaves each package-lock.json out of a published tarball.
 FROM node:24.21.0-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1
@@ -8,4 +9,4 @@ WORKDIR /opt/harness/node
 COPY harness/image/node/package.json ./
 COPY harness/image/node/npm-lock.json ./package-lock.json
 RUN npm ci --omit=dev --ignore-scripts --no-audit --no-fund
-COPY harness/image/node/client-ioredis.mjs ./
+COPY harness/image/node/client-ioredis.mjs harness/image/node/client-sentinel.mjs harness/image/node/observer.mjs ./

package/image/node/client-sentinel.mjs

@@ -0,0 +1,106 @@
+// client-sentinel --sentinels H:P,H:P,... --name NAME [--interval-ms N] [--max-s N] [--options JSON]
+//
+// A group cell's client: ioredis 6.0.0 in Sentinel mode, which asks the Sentinels for the primary
+// of NAME and connects to it. Every event is a JSON line stamped on CLOCK_MONOTONIC
+// (process.hrtime, which libuv reads from it), the clock the prober stamps the kernel's records
+// with, and the injector stamps each fault with.
+//
+// The workload is a stream of writes with a sequence number: SET w:<n> <n>, each sent when the
+// reply to the one before it came and --interval-ms passed. Each write is a sent line, then an
+// ack line with the address of the server that accepted it, or a fail line with its error. A
+// write whose line is neither had no answer when the client stopped. The group's keys after the
+// run tell which writes it kept. --options are ioredis options over its defaults: none at the
+// defaults of a cell. The client never stops on an error: ioredis reconnects, and the stream
+// goes on until --max-s or SIGTERM.
+
+import { readFileSync } from "node:fs";
+import Redis from "ioredis";
+
+const now = () => process.hrtime.bigint();
+const emit = (ev, fields = {}, at = now()) =>
+  process.stdout.write(`${JSON.stringify({ ev, t: Number(at), ...fields })}\n`);
+
+const argv = process.argv.slice(2);
+const flag = (name) => {
+  const at = argv.indexOf(name);
+  return at === -1 ? undefined : argv[at + 1];
+};
+const sentinels = (flag("--sentinels") ?? "")
+  .split(",")
+  .filter((each) => each !== "")
+  .map((each) => {
+    const [host, port] = each.split(":");
+    return { host, port: Number(port) };
+  });
+const name = flag("--name");
+if (sentinels.length === 0 || name === undefined) {
+  emit("fail", { why: "client-sentinel --sentinels H:P,... --name NAME" });
+  process.exit(1);
+}
+const interval = Number(flag("--interval-ms") ?? 50);
+const max = Number(flag("--max-s") ?? 120) * 1000;
+const options = JSON.parse(flag("--options") ?? "{}");
+
+emit("start", {
+  boot_id: readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(),
+  client: "ioredis",
+  mode: "sentinel",
+  options,
+});
+
+const redis = new Redis({ ...options, sentinels, name, lazyConnect: true });
+/** the server that the connection is on now, or null between connections */
+const remote = () => {
+  const stream = redis.stream;
+  return stream?.remoteAddress === undefined ? null : `${stream.remoteAddress}:${stream.remotePort}`;
+};
+for (const status of ["connect", "ready", "close", "reconnecting", "end"]) {
+  redis.on(status, (delay) => emit("status", { status, at: remote(), ...(status === "reconnecting" ? { delay } : {}) }));
+}
+redis.on("connect", () => {
+  const { localAddress, localPort } = redis.stream;
+  emit("connected", { local: `${localAddress}:${localPort}`, at: remote() });
+});
+// the errors on the way: a reset, a refused connect, a Sentinel that did not answer
+redis.on("error", (error) =>
+  emit("error", { name: error.name, errno: error.code ?? error.errno ?? null, message: error.message }),
+);
+redis.on("sentinelError", (error) => emit("sentinel-error", { message: error.message }));
+
+let stopped = false;
+const stop = () => {
+  if (stopped) return;
+  stopped = true;
+  emit("done");
+  redis.disconnect(false);
+  setTimeout(() => process.exit(0), 50);
+};
+process.on("SIGTERM", stop);
+setTimeout(() => {
+  emit("deadline");
+  stop();
+}, max);
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+try {
+  await redis.connect();
+  emit("ready", { at: remote() });
+} catch (error) {
+  emit("fail", { why: error.message });
+  process.exit(1);
+}
+
+for (let n = 1; !stopped; n += 1) {
+  const at = now();
+  // stamped before the write: the kernel can answer before the call returns
+  const reply = redis.set(`w:${n}`, String(n));
+  emit("sent", { n }, at);
+  try {
+    const answer = await reply;
+    emit("ack", { n, at: remote(), reply: answer });
+  } catch (error) {
+    emit("fail", { n, name: error.name, message: error.message });
+  }
+  await sleep(interval);
+}

package/image/node/observer.mjs

@@ -0,0 +1,39 @@
+// observer H:P,H:P,...
+//
+// The observer of a group's Sentinels. It subscribes to every channel of each Sentinel
+// (PSUBSCRIBE *): each Sentinel publishes each event of its own on a channel named for the event,
+// +sdown, +odown, +try-failover, +elected-leader, +selected-slave, +promoted-slave, +switch-master
+// and the others. Each message is a JSON line stamped on CLOCK_MONOTONIC when it arrives, the one
+// clock of the harness. The faults of a cell never cut the observer: it is in a namespace of its
+// own, and no rule names its address. A subscribed line gives the address of each connection, so
+// the prober's records of a Sentinel's writes to it can calibrate the stamps.
+
+import Redis from "ioredis";
+
+const now = () => process.hrtime.bigint();
+const emit = (ev, fields = {}, at = now()) =>
+  process.stdout.write(`${JSON.stringify({ ev, t: Number(at), ...fields })}\n`);
+
+const sentinels = (process.argv[2] ?? "").split(",").filter((each) => each !== "");
+if (sentinels.length === 0) {
+  emit("fail", { why: "observer H:P,H:P,..." });
+  process.exit(1);
+}
+
+for (const address of sentinels) {
+  const [host, port] = address.split(":");
+  // RESP2: a Sentinel's events are plain pmessage frames, and a lost connection comes back
+  const sub = new Redis({ host, port: Number(port), protocol: 2, enableReadyCheck: false, retryStrategy: () => 100 });
+  sub.on("ready", () => {
+    const { localAddress, localPort } = sub.stream;
+    emit("subscribed", { sentinel: address, local: `${localAddress}:${localPort}` });
+  });
+  sub.on("error", (error) => emit("observer-error", { sentinel: address, message: error.message }));
+  sub.on("pmessage", (_pattern, channel, message) => {
+    const at = now();
+    emit("sentinel", { sentinel: address, event: channel, detail: message }, at);
+  });
+  sub.psubscribe("*").catch((error) => emit("observer-error", { sentinel: address, message: error.message }));
+}
+
+process.on("SIGTERM", () => process.exit(0));

package/package.json

@@ -1,8 +1,8 @@
 {
   "$schema": "https://json.schemastore.org/package",
   "name": "@mark1russell7/harness",
-  "version": "0.4.0",
-  "description": "A fault-injection harness for connection timers: a cell runs a client against a stock target in Docker, through a topology of routers and proxies, injects a fault, measures the kernel's own records and judges a prediction.",
+  "version": "0.5.0",
+  "description": "A fault-injection harness for connection timers: a cell runs a client against a stock target in Docker, through a topology of routers and proxies or a replicated group under Sentinels, injects a fault, measures the kernel's own records and judges a prediction.",
   "keywords": [
     "fault-injection",
     "chaos-engineering",
@@ -43,6 +43,8 @@
     "image/native/src",
     "image/node/Dockerfile",
     "image/node/client-ioredis.mjs",
+    "image/node/client-sentinel.mjs",
+    "image/node/observer.mjs",
     "image/node/package.json",
     "image/node/npm-lock.json",
     "image/router/Dockerfile",
@@ -56,8 +58,8 @@
     "node": ">=22.19"
   },
   "dependencies": {
-    "@mark1russell7/prober": "^0.2.0",
-    "@mark1russell7/quantity": "^0.1.0",
+    "@mark1russell7/prober": "^0.3.1",
+    "@mark1russell7/quantity": "^0.2.0",
     "testcontainers": "12.2.0"
   },
   "peerDependencies": {