@mark1russell7/harness 0.2.0 → 0.4.0

package/image/native/src/bin/client.rs

@@ -15,8 +15,10 @@
 //! timeout makes the server answer only when the timeout runs out, so its reply can be the one in
 //! flight when a fault lands.
 //! stream: then wait for --go, and send the request --count times, each when the reply to the one
-//! before it has come and --interval-ms has passed: a distribution of reply times, for a fault
-//! whose outcome is one (loss, reordering). It ends with a done line.
+//! before it is whole and --interval-ms has passed: a distribution of reply times, for a fault
+//! whose outcome is one (loss, reordering). A reply is whole at its CRLF, or for a bulk string
+//! after its length: so `GET big` keeps a long reply's many segments in flight, and the replied
+//! line stamps its last byte. It ends with a done line.
 
 use prober::{errno_name, monotonic_ns, quantity};
 use serde_json::{json, Value};
@@ -87,6 +89,40 @@ fn readable(fd: i32, ms: i32) -> std::io::Result<bool> {
     }
 }
 
+/// The whole length of a Redis reply from its first bytes, or None while its first line is not
+/// all there: a simple string, an error or an integer ends at its CRLF, and a bulk string after
+/// its length and one more CRLF (a null one, of length -1, at its first CRLF).
+fn reply_length(head: &[u8]) -> Option<usize> {
+    let line = head.windows(2).position(|pair| pair == b"\r\n")? + 2;
+    if head.first() != Some(&b'$') {
+        return Some(line);
+    }
+    let length: i64 = std::str::from_utf8(&head[1..line - 2]).ok()?.parse().ok()?;
+    Some(if length < 0 { line } else { line + length as usize + 2 })
+}
+
+/// The reply a stream waits for: its first bytes until its length is known, and how much came.
+#[derive(Default)]
+struct Reply {
+    head: Vec<u8>,
+    length: Option<usize>,
+    got: usize,
+}
+
+impl Reply {
+    /// Takes the bytes of one read, and says whether the reply is whole.
+    fn take(&mut self, bytes: &[u8]) -> bool {
+        self.got += bytes.len();
+        if self.length.is_none() {
+            // a bulk string's first line is its length: 64 bytes hold any
+            let room = 64usize.saturating_sub(self.head.len());
+            self.head.extend_from_slice(&bytes[..bytes.len().min(room)]);
+            self.length = reply_length(&self.head);
+        }
+        matches!(self.length, Some(length) if self.got >= length)
+    }
+}
+
 /// what in TCP_INFO is worth a line when it changes: the timers' own state, never the ages
 fn watched(info: &libc::tcp_info) -> [u32; 7] {
     [
@@ -186,6 +222,7 @@ fn run() -> Result<(), Box<dyn Error>> {
     // stream mode: requests sent, whether one waits for its reply, and when the next may go
     let mut sent: u32 = 0;
     let mut awaiting = false;
+    let mut reply = Reply::default();
     let mut next_at: u64 = 0;
     loop {
         if readable(fd, tick)? {
@@ -196,8 +233,10 @@ fn run() -> Result<(), Box<dyn Error>> {
                 }
                 Ok(n) => {
                     emit("data", json!({ "n": n, "text": String::from_utf8_lossy(&buf[..n.min(40)]) }));
-                    if awaiting {
+                    if awaiting && reply.take(&buf[..n]) {
+                        emit("replied", json!({ "n": quantity(reply.got as u32, "B") }));
                         awaiting = false;
+                        reply = Reply::default();
                         next_at = monotonic_ns() + args.interval.as_nanos() as u64;
                     }
                 }
@@ -261,3 +300,24 @@ fn main() {
         std::process::exit(1);
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_reply_is_whole_at_its_crlf_or_after_its_bulk_length() {
+        assert_eq!(reply_length(b"+PONG\r\n"), Some(7));
+        assert_eq!(reply_length(b"+PON"), None);
+        assert_eq!(reply_length(b"$5\r\nhello\r\n"), Some(11));
+        assert_eq!(reply_length(b"$-1\r\n"), Some(5));
+        assert_eq!(reply_length(b"$1000000\r\nxx"), Some(1_000_012));
+        // a long reply in reads of any size: whole only at its last byte
+        let mut reply = Reply::default();
+        assert!(!reply.take(b"$10\r"));
+        assert!(!reply.take(b"\n01234"));
+        assert!(!reply.take(b"56789\r"));
+        assert!(reply.take(b"\n"));
+        assert_eq!(reply.got, 17);
+    }
+}

package/image/native/src/bin/inject.rs

@@ -23,10 +23,19 @@
 //!   `netem eth0 delay 100ms 10ms` or `netem eth0 loss 5%`.
 //! - `offload DEV on|off`: the device's TSO, GSO and GRO, so that netem and length matches act on
 //!   segments rather than on GSO super-packets of up to 64 KB.
+//! - `http METHOD ADDR:PORT PATH [BODY]`: one HTTP/1.1 request to an API in the VM, for example a
+//!   toxic of Toxiproxy. The stamps bracket the request and its answer, after the connect, so the
+//!   commit of the toxic is known on the same clock as the kernel's records.
 //!
 //! `--at NS` holds the commit until that instant on CLOCK_MONOTONIC, the clock every stamp is on.
 //! A stall's end is timed that way, from inside the VM: the exec only has to arrive before it, and
 //! its 190 ms never reach the stall's length.
+//!
+//! `--via DEV` puts a filter rule in the FORWARD chain, for a router: its INPUT and OUTPUT see only
+//! what the router itself sends and receives. The rule faces one side by the device toward it:
+//! what comes from the peer arrives on DEV (`-i DEV -s PEER`), and what goes to it leaves on DEV
+//! (`-o DEV -d PEER`). A REJECT then answers from the router: a reset spoofs the address the
+//! segment was going to, and an ICMP error comes from the router's own address on DEV.
 
 use prober::monotonic_ns;
 use serde_json::json;
@@ -60,13 +69,15 @@ fn until(at: u64) -> Result<(), Box<dyn Error>> {
     }
 }
 
-/// The options after the positional arguments: `--at NS`, `--direction D`, `--code C`, `--above N`.
+/// The options after the positional arguments: `--at NS`, `--direction D`, `--code C`, `--above N`,
+/// `--via DEV`.
 #[derive(Default)]
 struct Options {
     at: Option<u64>,
     direction: Option<String>,
     code: Option<String>,
     above: Option<u32>,
+    via: Option<String>,
 }
 
 fn options(args: &[String]) -> Result<Options, Box<dyn Error>> {
@@ -79,6 +90,7 @@ fn options(args: &[String]) -> Result<Options, Box<dyn Error>> {
             "--direction" => out.direction = Some(value.clone()),
             "--code" => out.code = Some(value.clone()),
             "--above" => out.above = Some(value.parse()?),
+            "--via" => out.via = Some(device(value)?.to_string()),
             other => return Err(format!("unknown option {other}").into()),
         }
     }
@@ -88,12 +100,18 @@ fn options(args: &[String]) -> Result<Options, Box<dyn Error>> {
 /// The iptables rules of an action that is a filter rule, or None for an action that is not.
 fn rules(action: &str, peer: Ipv4Addr, opts: &Options) -> Result<String, Box<dyn Error>> {
     let op = |set: &str| if action == set { "-I" } else { "-D" };
+    // what comes from the peer, and what goes to it: at an end, the INPUT and OUTPUT of its own
+    // sockets; at a router, what it forwards in and out of the device toward the peer
+    let (from, to) = match opts.via.as_deref() {
+        None => (format!("INPUT -s {peer}/32"), format!("OUTPUT -d {peer}/32")),
+        Some(dev) => (format!("FORWARD -i {dev} -s {peer}/32"), format!("FORWARD -o {dev} -d {peer}/32")),
+    };
     Ok(match action {
         "blackhole" | "clear" => {
             let op = op("blackhole");
             let direction = opts.direction.as_deref().unwrap_or("both");
-            let input = format!("{op} INPUT -s {peer}/32 -j DROP\n");
-            let output = format!("{op} OUTPUT -d {peer}/32 -j DROP\n");
+            let input = format!("{op} {from} -j DROP\n");
+            let output = format!("{op} {to} -j DROP\n");
             match direction {
                 "both" => format!("{input}{output}"),
                 "in" => input,
@@ -103,7 +121,7 @@ fn rules(action: &str, peer: Ipv4Addr, opts: &Options) -> Result<String, Box<dyn
         }
         "rst" | "clear-rst" => {
             let op = op("rst");
-            format!("{op} INPUT -s {peer}/32 -p tcp -j REJECT --reject-with tcp-reset\n")
+            format!("{op} {from} -p tcp -j REJECT --reject-with tcp-reset\n")
         }
         "reject" | "clear-reject" => {
             let op = op("reject");
@@ -114,14 +132,14 @@ fn rules(action: &str, peer: Ipv4Addr, opts: &Options) -> Result<String, Box<dyn
                 "admin" => "icmp-admin-prohibited",
                 other => return Err(format!("unknown code {other}").into()),
             };
-            format!("{op} INPUT -s {peer}/32 -p tcp -j REJECT --reject-with {with}\n")
+            format!("{op} {from} -p tcp -j REJECT --reject-with {with}\n")
         }
         // the long segments drop where they arrive: in the sender's own namespace, an OUTPUT drop
         // would fail its sends with EPERM rather than lose them on the way
         "mtu" | "clear-mtu" => {
             let op = op("mtu");
             let above = opts.above.ok_or("--above N")?;
-            format!("{op} INPUT -s {peer}/32 -p tcp -m length --length {}:65535 -j DROP\n", above + 1)
+            format!("{op} {from} -p tcp -m length --length {}:65535 -j DROP\n", above + 1)
         }
         other => return Err(format!("unknown action {other}").into()),
     })
@@ -165,9 +183,63 @@ fn command(program: &str, args: &[&str]) -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
+/// an HTTP path: a slash, then letters, digits and `/ - _ .` only
+fn http_path(path: &str) -> Result<&str, Box<dyn Error>> {
+    if path.starts_with('/')
+        && path.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '/' | '-' | '_' | '.'))
+    {
+        Ok(path)
+    } else {
+        Err(format!("not an HTTP path: {path}").into())
+    }
+}
+
+/// the head of one HTTP/1.1 request, with its body
+fn http_request(method: &str, host: &str, path: &str, body: &str) -> Result<String, Box<dyn Error>> {
+    if !matches!(method, "GET" | "POST" | "DELETE") {
+        return Err(format!("not a method: {method}").into());
+    }
+    Ok(format!(
+        "{method} {path} HTTP/1.1\r\nHost: {host}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+        body.len()
+    ))
+}
+
+/// one request to an API: the status, and the stamps around the request and its answer
+fn http(args: &[String]) -> Result<(), Box<dyn Error>> {
+    use std::io::Read;
+    use std::net::{SocketAddrV4, TcpStream};
+    let method = args.get(1).ok_or("http METHOD ADDR:PORT PATH [BODY]")?.as_str();
+    let addr: SocketAddrV4 = args.get(2).ok_or("and the address and port")?.parse()?;
+    let path = http_path(args.get(3).ok_or("and the path")?)?;
+    let body = args.get(4).map_or("", String::as_str);
+    let request = http_request(method, &addr.to_string(), path, body)?;
+    let mut stream = TcpStream::connect(addr)?;
+    let t0 = monotonic_ns();
+    stream.write_all(request.as_bytes())?;
+    let mut answer = Vec::new();
+    stream.read_to_end(&mut answer)?;
+    let t1 = monotonic_ns();
+    let text = String::from_utf8_lossy(&answer);
+    let status: u16 = text
+        .split_whitespace()
+        .nth(1)
+        .ok_or("no status in the answer")?
+        .parse()?;
+    if !(200..300).contains(&status) {
+        let detail = text.split("\r\n\r\n").nth(1).unwrap_or("").trim();
+        return Err(format!("{method} {path} answered {status}: {detail}").into());
+    }
+    println!("{}", json!({ "action": "http", "method": method, "path": path, "status": status, "t0": t0, "t1": t1 }));
+    Ok(())
+}
+
 fn run() -> Result<(), Box<dyn Error>> {
     let args: Vec<String> = std::env::args().skip(1).collect();
-    let action = args.first().ok_or("say an action: blackhole, clear, rst, reject, mtu, netem, offload …")?.as_str();
+    let action = args.first().ok_or("say an action: blackhole, clear, rst, reject, mtu, netem, offload, http …")?.as_str();
+    if action == "http" {
+        return http(&args);
+    }
     match action {
         "netem" | "clear-netem" | "offload" => {
             let dev = device(args.get(1).ok_or("and a device")?)?;
@@ -265,4 +337,44 @@ mod tests {
         );
         assert!(rules("reject", peer, &Options::default()).is_err());
     }
+
+    #[test]
+    fn a_router_gets_forward_rules_that_face_one_device() {
+        let peer: Ipv4Addr = "10.0.0.2".parse().unwrap();
+        let via = |direction: Option<&str>| Options {
+            direction: direction.map(Into::into),
+            via: Some("eth0".into()),
+            ..Options::default()
+        };
+        assert_eq!(
+            rules("blackhole", peer, &via(None)).unwrap(),
+            "-I FORWARD -i eth0 -s 10.0.0.2/32 -j DROP\n-I FORWARD -o eth0 -d 10.0.0.2/32 -j DROP\n"
+        );
+        assert_eq!(rules("clear", peer, &via(Some("out"))).unwrap(), "-D FORWARD -o eth0 -d 10.0.0.2/32 -j DROP\n");
+        assert_eq!(
+            rules("rst", peer, &via(None)).unwrap(),
+            "-I FORWARD -i eth0 -s 10.0.0.2/32 -p tcp -j REJECT --reject-with tcp-reset\n"
+        );
+        let host = Options { code: Some("host".into()), ..via(None) };
+        assert_eq!(
+            rules("clear-reject", peer, &host).unwrap(),
+            "-D FORWARD -i eth0 -s 10.0.0.2/32 -p tcp -j REJECT --reject-with icmp-host-unreachable\n"
+        );
+        // a device name is letters, digits, dots and dashes: nothing else reaches the rules
+        assert!(options(&["--via".into(), "eth0;x".into()]).is_err());
+        assert_eq!(options(&["--via".into(), "eth1".into()]).unwrap().via.as_deref(), Some("eth1"));
+    }
+
+    #[test]
+    fn http_requests_take_only_a_method_a_path_and_a_length() {
+        assert!(http_path("/proxies/redis/toxics").is_ok());
+        for path in ["proxies", "/a b", "/a;b", "/a?b=1", "/$(x)"] {
+            assert!(http_path(path).is_err(), "{path}");
+        }
+        let request = http_request("POST", "10.0.0.3:8474", "/proxies/redis/toxics", "{}").unwrap();
+        assert!(request.starts_with("POST /proxies/redis/toxics HTTP/1.1\r\n"));
+        assert!(request.contains("Content-Length: 2\r\n"));
+        assert!(request.ends_with("\r\n\r\n{}"));
+        assert!(http_request("PUT", "h", "/", "").is_err());
+    }
 }

package/image/router/Dockerfile

@@ -0,0 +1,8 @@
+# The router of the harness: a NAT box with conntrack, the middle of a stateful middlebox chain. It
+# forgets an idle flow at its idle timeout, and its form tells what a late packet then meets (mbx.sh).
+# lab.ts stages the build context: this folder, under harness/image/router. The base is pinned by
+# digest, and the manifest of each run records the packages that apk installs.
+FROM alpine:3.23@sha256:865b95f46d98cf867a156fe4a135ad3fe50d2056aa3f25ed31662dff6da4eb62
+RUN apk add --no-cache conntrack-tools nftables iproute2 python3
+COPY harness/image/router/mbx.sh harness/image/router/ctwatch.py harness/image/router/resetter.py /opt/mbx/
+ENTRYPOINT ["/bin/sh", "/opt/mbx/mbx.sh"]

package/image/router/ctwatch.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+"""The conntrack events of the router, as JSON lines on the monotonic clock.
+
+Each line is {"ev": "ct", "t": <CLOCK_MONOTONIC ns>, "type": "NEW|UPDATE|DESTROY", "state": ...,
+"orig": "src:sport>dst:dport", "reply": "..."}. The stamp is the time that this process read the
+event, which is within a millisecond of the kernel. A listing every PUMP seconds evicts the expired
+entries that it walks past, thus a DESTROY follows an expiry within PUMP plus two listings. A
+listing that takes longer than SLOW seconds writes a "pump-slow" line with its start and its
+length: near it, that bound of the lag does not hold.
+
+  ctwatch.py [--pump SECONDS] [--slow SECONDS]
+"""
+import argparse, json, re, subprocess, sys, threading, time
+
+p = argparse.ArgumentParser()
+p.add_argument("--pump", type=float, default=0.1)
+p.add_argument("--slow", type=float, default=0.02)
+a = p.parse_args()
+
+PAT = re.compile(
+    r"\[(?P<type>NEW|UPDATE|DESTROY)\]\s+tcp\s+\d+\s+(?:\d+\s+)?(?P<state>[A-Z_]+)?\s*"
+    r"src=(?P<s>\S+) dst=(?P<d>\S+) sport=(?P<sp>\d+) dport=(?P<dp>\d+).*?"
+    r"src=(?P<rs>\S+) dst=(?P<rd>\S+) sport=(?P<rsp>\d+) dport=(?P<rdp>\d+)"
+)
+lock = threading.Lock()
+
+
+def now():
+    return time.clock_gettime_ns(time.CLOCK_MONOTONIC)
+
+
+def emit(**fields):
+    with lock:
+        print(json.dumps({"t": now(), **fields}), flush=True)
+
+
+def pump():
+    while True:
+        start = now()
+        subprocess.run(["conntrack", "-L", "-p", "tcp"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        took = (now() - start) / 1e9
+        if took > a.slow:
+            with lock:
+                print(json.dumps({"ev": "pump-slow", "t": start, "took": took}), flush=True)
+        time.sleep(a.pump)
+
+
+if a.pump > 0:
+    threading.Thread(target=pump, daemon=True).start()
+proc = subprocess.Popen(
+    ["conntrack", "-E", "-p", "tcp"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1
+)
+emit(ev="ct-watch", pump=a.pump, slow=a.slow)
+for line in proc.stdout:
+    m = PAT.search(line)
+    if m is None:
+        emit(ev="ct-unparsed", line=line.strip())
+        continue
+    emit(
+        ev="ct",
+        type=m["type"],
+        state=m["state"],
+        orig=f'{m["s"]}:{m["sp"]}>{m["d"]}:{m["dp"]}',
+        reply=f'{m["rs"]}:{m["rsp"]}>{m["rd"]}:{m["rdp"]}',
+    )
+sys.exit(proc.wait())

package/image/router/mbx.sh

@@ -0,0 +1,61 @@
+#!/bin/sh
+# The router of a middlebox chain: one image for the forms of a middlebox at its idle expiry.
+#
+#   FORM=reset-both|reset-next|silent-drop|recreate|none IDLE=<seconds> [LAN=eth0] [WAN=eth1] [PUMP=0.1] [SLOW=0.02]
+#
+# The container is privileged, so it sets the sysctls of its own namespace. Docker Desktop keeps
+# /proc/sys read-only in a container that is not. The network of the side of the client is LAN,
+# and the harness connects the network of the side of the target (WAN) after the start.
+#
+# FORM         at the expiry                  a late packet from inside      a late packet from outside
+# reset-both   a reset to the two ends        dropped                        dropped
+# reset-next   nothing                        a reset to its sender          dropped
+# silent-drop  nothing                        dropped                        dropped
+# recreate     nothing                        passes, and starts the flow    dropped
+# none         nothing                        passes (a plain NAT router)    a reset from the router
+#
+# A lookup treats an expired entry as gone, so the router forgets a flow at its timeout to the
+# jiffy. The DESTROY event of conntrack waits for the garbage collector or a lookup: ctwatch.py
+# lists the table every PUMP seconds, so the event follows the expiry within PUMP and two listings.
+# A listing longer than SLOW seconds writes a pump-slow line: near it, that bound does not hold.
+set -eu
+FORM=${FORM:-none}; LAN=${LAN:-eth0}; WAN=${WAN:-eth1}; PUMP=${PUMP:-0.1}; SLOW=${SLOW:-0.02}; IDLE=${IDLE:-432000}
+sysctl -q -w net.ipv4.ip_forward=1
+# the conntrack sysctls of a namespace appear when it first uses conntrack: a listing does that
+conntrack -L >/dev/null 2>&1 || true
+sysctl -q -w net.netfilter.nf_conntrack_events=1
+sysctl -q -w "net.netfilter.nf_conntrack_tcp_timeout_established=$IDLE"
+i=0
+while ! ip link show "$WAN" >/dev/null 2>&1; do
+  i=$((i + 1)); [ "$i" -gt 600 ] && { echo "mbx: no $WAN" >&2; exit 65; }; sleep 0.1
+done
+case $FORM in
+  reset-both)  INSIDE=drop; OUTSIDE=drop ;;
+  reset-next)  INSIDE=rst;  OUTSIDE=drop ;;
+  silent-drop) INSIDE=drop; OUTSIDE=drop ;;
+  recreate)    INSIDE=pass; OUTSIDE=drop ;;
+  none)        INSIDE=pass; OUTSIDE=rst ;;
+  *) echo "mbx: unknown FORM=$FORM" >&2; exit 64 ;;
+esac
+verdict() { case $1 in rst) echo "reject with tcp reset" ;; drop) echo drop ;; *) echo accept ;; esac; }
+# OUTSIDE=rst: accept, and the stack of the router answers the unknown port with an exact reset
+outv=$(verdict "$OUTSIDE"); [ "$OUTSIDE" = rst ] && outv=accept
+nft -f - <<EOF
+table ip mbx {
+  chain mbx_post { type nat hook postrouting priority srcnat; policy accept; oifname "$WAN" masquerade; }
+  chain mbx_fwd {
+    type filter hook forward priority filter; policy accept;
+    iifname "$LAN" meta l4proto tcp ct state new,invalid tcp flags & (syn|ack) != syn counter $(verdict "$INSIDE")
+  }
+  chain mbx_in {
+    type filter hook input priority filter; policy accept;
+    iifname "$WAN" meta l4proto tcp ct state new,invalid tcp flags & (syn|ack) != syn counter $outv
+  }
+}
+EOF
+echo "{\"ev\":\"mbx-ready\",\"t\":0,\"form\":\"$FORM\",\"idle\":$(cat /proc/sys/net/netfilter/nf_conntrack_tcp_timeout_established),\"inside\":\"$INSIDE\",\"outside\":\"$OUTSIDE\"}"
+if [ "$FORM" = reset-both ]; then
+  python3 /opt/mbx/ctwatch.py --pump "$PUMP" --slow "$SLOW" &
+  exec python3 /opt/mbx/resetter.py "$LAN" "$WAN"
+fi
+exec python3 /opt/mbx/ctwatch.py --pump "$PUMP" --slow "$SLOW"

package/image/router/resetter.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""The reset at expiry: the form of a middlebox that resets the two ends when it forgets a flow.
+
+It listens to the DESTROY events of conntrack in this namespace. For each destroyed TCP flow that
+it saw carry traffic, and never saw closed (FIN or RST), it sends a reset to each end. Each reset
+has the sequence number that its end expects: the last acknowledgment number that the end sent,
+read by a raw sniffer on the interfaces, because conntrack events carry no sequence numbers.
+Each line is JSON on the monotonic clock: {"ev": "reset", "t": ..., "flow": ..., "to": [...]}.
+
+  resetter.py IFACE [IFACE...]
+"""
+import argparse, json, re, socket, struct, subprocess, threading, time
+
+p = argparse.ArgumentParser()
+p.add_argument("ifaces", nargs="+")
+a = p.parse_args()
+
+plock = threading.Lock()
+flock = threading.Lock()
+flows = {}  # (src, sport, dst, dport) -> {"ack", "closed"}
+
+
+def emit(**fields):
+    with plock:
+        print(json.dumps({"t": time.clock_gettime_ns(time.CLOCK_MONOTONIC), **fields}), flush=True)
+
+
+def sniff(ifname):
+    s = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(0x0800))
+    s.bind((ifname, 0))
+    while True:
+        pkt = s.recv(65535)
+        ip = pkt[14:]
+        if len(ip) < 40 or ip[0] >> 4 != 4 or ip[9] != 6:
+            continue
+        ihl = (ip[0] & 0xF) * 4
+        src, dst = socket.inet_ntoa(ip[12:16]), socket.inet_ntoa(ip[16:20])
+        tcp = ip[ihl:]
+        sport, dport, _seq, ack = struct.unpack("!HHII", tcp[:12])
+        fl = tcp[13]
+        with flock:
+            f = flows.setdefault((src, sport, dst, dport), {"ack": None, "closed": False})
+            if fl & 0x10:
+                f["ack"] = ack
+            if fl & 0x05:
+                f["closed"] = True
+
+
+raw = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_RAW)
+
+
+def csum(b):
+    if len(b) % 2:
+        b += b"\0"
+    s = sum(struct.unpack("!%dH" % (len(b) // 2), b))
+    while s >> 16:
+        s = (s & 0xFFFF) + (s >> 16)
+    return ~s & 0xFFFF
+
+
+def send_rst(src, sport, dst, dport, seq):
+    tcp = struct.pack("!HHIIBBHHH", sport, dport, seq, 0, 5 << 4, 0x04, 0, 0, 0)
+    pseudo = socket.inet_aton(src) + socket.inet_aton(dst) + struct.pack("!BBH", 0, 6, len(tcp))
+    tcp = tcp[:16] + struct.pack("!H", csum(pseudo + tcp)) + tcp[18:]
+    iph = struct.pack(
+        "!BBHHHBBH4s4s", 0x45, 0, 20 + len(tcp), 0, 0x4000, 64, 6, 0, socket.inet_aton(src), socket.inet_aton(dst)
+    )
+    raw.sendto(iph + tcp, (dst, 0))
+
+
+PAT = re.compile(r"src=(\S+) dst=(\S+) sport=(\d+) dport=(\d+) .*?src=(\S+) dst=(\S+) sport=(\d+) dport=(\d+)")
+
+
+def main():
+    for i in a.ifaces:
+        threading.Thread(target=sniff, args=(i,), daemon=True).start()
+    proc = subprocess.Popen(
+        ["conntrack", "-E", "-e", "DESTROY", "-p", "tcp"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1
+    )
+    emit(ev="resetter", ifaces=a.ifaces)
+    for line in proc.stdout:
+        m = PAT.search(line)
+        if not m:
+            continue
+        c, s, cp, sp, s2, r, sp2, rp = m.groups()
+        cp, sp, sp2, rp = int(cp), int(sp), int(sp2), int(rp)
+        k_cs, k_sr = (c, cp, s, sp), (s2, sp2, r, rp)
+        with flock:
+            fs = {k: dict(flows[k]) for k in (k_cs, (s, sp, c, cp), (r, rp, s2, sp2), k_sr) if k in flows}
+        if not fs or any(f["closed"] for f in fs.values()):
+            continue
+        seq_c = fs.get(k_cs, {}).get("ack")  # the RCV.NXT of the client
+        seq_s = fs.get(k_sr, {}).get("ack")  # the RCV.NXT of the server, on the translated side
+        sent = []
+        if seq_c is not None:
+            send_rst(s, sp, c, cp, seq_c)
+            sent.append("client")
+        if seq_s is not None:
+            send_rst(r, rp, s2, sp2, seq_s)
+            sent.append("server")
+        emit(ev="reset", flow=f"{c}:{cp}>{s}:{sp}", nat=f"{r}:{rp}", to=sent)
+
+
+main()

package/package.json

@@ -1,8 +1,8 @@
 {
   "$schema": "https://json.schemastore.org/package",
   "name": "@mark1russell7/harness",
-  "version": "0.2.0",
-  "description": "A fault-injection harness for connection timers: a cell runs a client against a stock target in Docker, injects a fault, measures the kernel's own records and judges a prediction.",
+  "version": "0.4.0",
+  "description": "A fault-injection harness for connection timers: a cell runs a client against a stock target in Docker, through a topology of routers and proxies, injects a fault, measures the kernel's own records and judges a prediction.",
   "keywords": [
     "fault-injection",
     "chaos-engineering",
@@ -45,6 +45,10 @@
     "image/node/client-ioredis.mjs",
     "image/node/package.json",
     "image/node/npm-lock.json",
+    "image/router/Dockerfile",
+    "image/router/mbx.sh",
+    "image/router/ctwatch.py",
+    "image/router/resetter.py",
     "README.md",
     "LICENSE"
   ],
@@ -52,7 +56,7 @@
     "node": ">=22.19"
   },
   "dependencies": {
-    "@mark1russell7/prober": "^0.1.0",
+    "@mark1russell7/prober": "^0.2.0",
     "@mark1russell7/quantity": "^0.1.0",
     "testcontainers": "12.2.0"
   },