@mark1russell7/harness 0.1.0 → 0.3.0

package/image/native/src/bin/client.rs

@@ -1,5 +1,5 @@
-//! client HOST PORT --mode idle|inflight [--keepalive] [--user-timeout-ms N] [--go PATH]
-//!        [--request TEXT] [--sample-ms N] [--max-s N]
+//! client HOST PORT --mode idle|inflight|stream [--keepalive] [--user-timeout-ms N] [--go PATH]
+//!        [--request TEXT] [--count N] [--interval-ms N] [--sample-ms N] [--max-s N]
 //!
 //! A cell's TCP client: one connection, every event a JSON line stamped on CLOCK_MONOTONIC. It
 //! sets no per-socket TCP timer unless the cell asks, so the namespace's sysctls are what gets
@@ -14,6 +14,9 @@
 //! names that second request, a Redis inline command: PING unless it says otherwise. BLPOP with a
 //! timeout makes the server answer only when the timeout runs out, so its reply can be the one in
 //! flight when a fault lands.
+//! stream: then wait for --go, and send the request --count times, each when the reply to the one
+//! before it has come and --interval-ms has passed: a distribution of reply times, for a fault
+//! whose outcome is one (loss, reordering). It ends with a done line.
 
 use prober::{errno_name, monotonic_ns, quantity};
 use serde_json::{json, Value};
@@ -101,6 +104,9 @@ struct Args {
     host: String,
     port: u16,
     inflight: bool,
+    /// in stream mode, how many requests, and the pause after each reply
+    stream: Option<u32>,
+    interval: Duration,
     keepalive: bool,
     user_timeout_ms: Option<i32>,
     go: String,
@@ -113,14 +119,17 @@ struct Args {
 fn args() -> Result<Args, Box<dyn Error>> {
     let raw: Vec<String> = std::env::args().skip(1).collect();
     let value = |flag: &str| raw.iter().position(|arg| arg == flag).and_then(|at| raw.get(at + 1)).cloned();
-    let mode = value("--mode").ok_or("--mode idle|inflight is required")?;
-    if mode != "idle" && mode != "inflight" {
-        return Err(format!("--mode is idle or inflight, not {mode}").into());
+    let mode = value("--mode").ok_or("--mode idle|inflight|stream is required")?;
+    if mode != "idle" && mode != "inflight" && mode != "stream" {
+        return Err(format!("--mode is idle, inflight or stream, not {mode}").into());
     }
+    let count: u32 = value("--count").map(|v| v.parse()).transpose()?.unwrap_or(100);
     Ok(Args {
         host: raw.first().ok_or("the target's host comes first")?.clone(),
         port: raw.get(1).ok_or("then its port")?.parse()?,
-        inflight: mode == "inflight",
+        inflight: mode != "idle",
+        stream: (mode == "stream").then_some(count),
+        interval: Duration::from_millis(value("--interval-ms").map(|v| v.parse()).transpose()?.unwrap_or(20)),
         keepalive: raw.iter().any(|arg| arg == "--keepalive"),
         user_timeout_ms: value("--user-timeout-ms").map(|v| v.parse()).transpose()?,
         go: value("--go").unwrap_or_else(|| "/tmp/go".into()),
@@ -133,7 +142,12 @@ fn args() -> Result<Args, Box<dyn Error>> {
 fn run() -> Result<(), Box<dyn Error>> {
     let args = args()?;
     let boot_id = std::fs::read_to_string("/proc/sys/kernel/random/boot_id")?.trim().to_string();
-    emit("start", json!({ "boot_id": boot_id, "mode": if args.inflight { "inflight" } else { "idle" } }));
+    let mode = match (args.inflight, args.stream) {
+        (_, Some(_)) => "stream",
+        (true, None) => "inflight",
+        (false, None) => "idle",
+    };
+    emit("start", json!({ "boot_id": boot_id, "mode": mode }));
     let deadline = monotonic_ns() + args.max.as_nanos() as u64;
 
     let mut stream = TcpStream::connect((args.host.as_str(), args.port))?;
@@ -169,6 +183,10 @@ fn run() -> Result<(), Box<dyn Error>> {
     let mut waiting = args.inflight;
     let mut seen: Option<[u32; 7]> = None;
     let mut age = info.tcpi_last_ack_recv;
+    // stream mode: requests sent, whether one waits for its reply, and when the next may go
+    let mut sent: u32 = 0;
+    let mut awaiting = false;
+    let mut next_at: u64 = 0;
     loop {
         if readable(fd, tick)? {
             match stream.read(&mut buf) {
@@ -176,7 +194,13 @@ fn run() -> Result<(), Box<dyn Error>> {
                     emit("eof", json!({}));
                     return Ok(());
                 }
-                Ok(n) => emit("data", json!({ "n": n, "text": String::from_utf8_lossy(&buf[..n.min(40)]) })),
+                Ok(n) => {
+                    emit("data", json!({ "n": n, "text": String::from_utf8_lossy(&buf[..n.min(40)]) }));
+                    if awaiting {
+                        awaiting = false;
+                        next_at = monotonic_ns() + args.interval.as_nanos() as u64;
+                    }
+                }
                 // nothing after all; ETIMEDOUT is the connection's own end
                 Err(error) if error.kind() == ErrorKind::WouldBlock || error.kind() == ErrorKind::Interrupted => {}
                 Err(error) => {
@@ -208,6 +232,21 @@ fn run() -> Result<(), Box<dyn Error>> {
         if waiting && Path::new(&args.go).exists() {
             send(&mut stream, &args.request)?;
             waiting = false;
+            sent = 1;
+            awaiting = args.stream.is_some();
+        }
+        if let Some(count) = args.stream {
+            if !waiting && !awaiting {
+                if sent >= count {
+                    emit("done", json!({ "requests": sent }));
+                    return Ok(());
+                }
+                if monotonic_ns() >= next_at {
+                    send(&mut stream, &args.request)?;
+                    sent += 1;
+                    awaiting = true;
+                }
+            }
         }
         if monotonic_ns() > deadline {
             emit("deadline", json!({}));

package/image/native/src/bin/inject.rs

@@ -1,14 +1,31 @@
-//! inject blackhole PEER | inject clear PEER [--at NS] | inject rst PEER | inject clear-rst PEER
+//! inject ACTION ARGS... [--at NS]
 //!
-//! A cell's fault, applied inside the network namespace this sidecar shares with the target, so
-//! the target's image stays stock. Each lands in one iptables-restore commit,
-//! bracketed by CLOCK_MONOTONIC stamps taken in the VM: a docker exec from Windows lands 190 ms late
-//! at the median, and the bracket does not care.
+//! A cell's fault, applied inside the network namespace this sidecar shares with the target (or,
+//! for the server's side, with the client), so the target's image stays stock. Each lands in one
+//! commit, bracketed by CLOCK_MONOTONIC stamps taken in the VM: a docker exec from Windows lands
+//! 190 ms late at the median, and the bracket does not care.
 //!
-//! - blackhole: both directions drop, so nothing comes back;
-//! - rst: the peer's next segment is answered with a reset, so its next send fails at once. A
-//!   REJECT resets only the sender of the matched segment: the target's process
-//!   never hears of it.
+//! The actions, each with its lift:
+//!
+//! - `blackhole PEER [--direction both|in|out]` / `clear PEER [--direction …]`: the peer's
+//!   segments drop, in both directions or one. `in` drops what comes from the peer, `out` what
+//!   goes to it: a simplex partition.
+//! - `rst PEER` / `clear-rst PEER`: the peer's next segment is answered with a reset, so its next
+//!   send fails at once. A REJECT resets only the sender of the matched segment: the target's
+//!   process never hears of it.
+//! - `reject PEER --code host|net|port|admin` / `clear-reject PEER --code …`: each segment from the
+//!   peer is answered with that ICMP unreachable. On an established socket Linux keeps it as a
+//!   soft error (tcp_v4_err) and goes on retransmitting.
+//! - `mtu PEER --above N` / `clear-mtu PEER --above N`: each segment from the peer longer than N
+//!   bytes drops, as a path MTU black hole does, in the namespace that receives it. It acts on
+//!   segments only when the sender's TSO and GSO and the receiver's GRO are off (`offload`).
+//! - `netem DEV SPEC…` / `clear-netem DEV`: tc netem on the device's egress, for example
+//!   `netem eth0 delay 100ms 10ms` or `netem eth0 loss 5%`.
+//! - `offload DEV on|off`: the device's TSO, GSO and GRO, so that netem and length matches act on
+//!   segments rather than on GSO super-packets of up to 64 KB.
+//! - `http METHOD ADDR:PORT PATH [BODY]`: one HTTP/1.1 request to an API in the VM, for example a
+//!   toxic of Toxiproxy. The stamps bracket the request and its answer, after the connect, so the
+//!   commit of the toxic is known on the same clock as the kernel's records.
 //!
 //! `--at NS` holds the commit until that instant on CLOCK_MONOTONIC, the clock every stamp is on.
 //! A stall's end is timed that way, from inside the VM: the exec only has to arrive before it, and
@@ -46,46 +63,276 @@ fn until(at: u64) -> Result<(), Box<dyn Error>> {
     }
 }
 
-fn run() -> Result<(), Box<dyn Error>> {
-    let args: Vec<String> = std::env::args().skip(1).collect();
-    let action = args.first().ok_or("say blackhole, clear, rst or clear-rst")?.as_str();
-    // parsed, never pasted: only an address reaches the rules
-    let peer: Ipv4Addr = args.get(1).ok_or("and the peer's address")?.parse()?;
-    let at: Option<u64> = match args.get(2).map(String::as_str) {
-        None => None,
-        Some("--at") => Some(args.get(3).ok_or("--at takes an instant in ns")?.parse()?),
-        Some(other) => return Err(format!("unknown option {other}").into()),
-    };
-    let rules = match action {
+/// The options after the positional arguments: `--at NS`, `--direction D`, `--code C`, `--above N`.
+#[derive(Default)]
+struct Options {
+    at: Option<u64>,
+    direction: Option<String>,
+    code: Option<String>,
+    above: Option<u32>,
+}
+
+fn options(args: &[String]) -> Result<Options, Box<dyn Error>> {
+    let mut out = Options::default();
+    let mut rest = args.iter();
+    while let Some(flag) = rest.next() {
+        let value = rest.next().ok_or_else(|| format!("{flag} takes a value"))?;
+        match flag.as_str() {
+            "--at" => out.at = Some(value.parse()?),
+            "--direction" => out.direction = Some(value.clone()),
+            "--code" => out.code = Some(value.clone()),
+            "--above" => out.above = Some(value.parse()?),
+            other => return Err(format!("unknown option {other}").into()),
+        }
+    }
+    Ok(out)
+}
+
+/// The iptables rules of an action that is a filter rule, or None for an action that is not.
+fn rules(action: &str, peer: Ipv4Addr, opts: &Options) -> Result<String, Box<dyn Error>> {
+    let op = |set: &str| if action == set { "-I" } else { "-D" };
+    Ok(match action {
         "blackhole" | "clear" => {
-            let op = if action == "blackhole" { "-I" } else { "-D" };
-            format!("{op} INPUT -s {peer}/32 -j DROP\n{op} OUTPUT -d {peer}/32 -j DROP\n")
+            let op = op("blackhole");
+            let direction = opts.direction.as_deref().unwrap_or("both");
+            let input = format!("{op} INPUT -s {peer}/32 -j DROP\n");
+            let output = format!("{op} OUTPUT -d {peer}/32 -j DROP\n");
+            match direction {
+                "both" => format!("{input}{output}"),
+                "in" => input,
+                "out" => output,
+                other => return Err(format!("unknown direction {other}").into()),
+            }
         }
         "rst" | "clear-rst" => {
-            let op = if action == "rst" { "-I" } else { "-D" };
+            let op = op("rst");
             format!("{op} INPUT -s {peer}/32 -p tcp -j REJECT --reject-with tcp-reset\n")
         }
+        "reject" | "clear-reject" => {
+            let op = op("reject");
+            let with = match opts.code.as_deref().ok_or("--code host|net|port|admin")? {
+                "host" => "icmp-host-unreachable",
+                "net" => "icmp-net-unreachable",
+                "port" => "icmp-port-unreachable",
+                "admin" => "icmp-admin-prohibited",
+                other => return Err(format!("unknown code {other}").into()),
+            };
+            format!("{op} INPUT -s {peer}/32 -p tcp -j REJECT --reject-with {with}\n")
+        }
+        // the long segments drop where they arrive: in the sender's own namespace, an OUTPUT drop
+        // would fail its sends with EPERM rather than lose them on the way
+        "mtu" | "clear-mtu" => {
+            let op = op("mtu");
+            let above = opts.above.ok_or("--above N")?;
+            format!("{op} INPUT -s {peer}/32 -p tcp -m length --length {}:65535 -j DROP\n", above + 1)
+        }
         other => return Err(format!("unknown action {other}").into()),
-    };
-    let script = format!("*filter\n{rules}COMMIT\n");
-    if let Some(at) = at {
-        until(at)?;
+    })
+}
+
+/// a token of a netem specification: a word, or a number with an optional unit
+fn netem_token(token: &str) -> bool {
+    let word = !token.is_empty() && token.chars().all(|c| c.is_ascii_lowercase());
+    let number = token
+        .trim_end_matches(|c: char| c.is_ascii_alphabetic() || c == '%')
+        .parse::<f64>()
+        .is_ok()
+        && token
+            .trim_start_matches(|c: char| c.is_ascii_digit() || c == '.')
+            .chars()
+            .all(|c| c.is_ascii_alphabetic() || c == '%');
+    word || number
+}
+
+/// a device name: letters, digits, dots, dashes
+fn device(name: &str) -> Result<&str, Box<dyn Error>> {
+    if !name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-') {
+        Ok(name)
+    } else {
+        Err(format!("not a device: {name}").into())
+    }
+}
+
+/// runs a command and fails with its stderr when it fails
+fn command(program: &str, args: &[&str]) -> Result<(), Box<dyn Error>> {
+    let out = Command::new(program).args(args).output()?;
+    if !out.status.success() {
+        return Err(format!(
+            "{program} {} exited {}: {}",
+            args.join(" "),
+            out.status,
+            String::from_utf8_lossy(&out.stderr).trim()
+        )
+        .into());
+    }
+    Ok(())
+}
+
+/// an HTTP path: a slash, then letters, digits and `/ - _ .` only
+fn http_path(path: &str) -> Result<&str, Box<dyn Error>> {
+    if path.starts_with('/')
+        && path.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '/' | '-' | '_' | '.'))
+    {
+        Ok(path)
+    } else {
+        Err(format!("not an HTTP path: {path}").into())
     }
+}
+
+/// the head of one HTTP/1.1 request, with its body
+fn http_request(method: &str, host: &str, path: &str, body: &str) -> Result<String, Box<dyn Error>> {
+    if !matches!(method, "GET" | "POST" | "DELETE") {
+        return Err(format!("not a method: {method}").into());
+    }
+    Ok(format!(
+        "{method} {path} HTTP/1.1\r\nHost: {host}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+        body.len()
+    ))
+}
+
+/// one request to an API: the status, and the stamps around the request and its answer
+fn http(args: &[String]) -> Result<(), Box<dyn Error>> {
+    use std::io::Read;
+    use std::net::{SocketAddrV4, TcpStream};
+    let method = args.get(1).ok_or("http METHOD ADDR:PORT PATH [BODY]")?.as_str();
+    let addr: SocketAddrV4 = args.get(2).ok_or("and the address and port")?.parse()?;
+    let path = http_path(args.get(3).ok_or("and the path")?)?;
+    let body = args.get(4).map_or("", String::as_str);
+    let request = http_request(method, &addr.to_string(), path, body)?;
+    let mut stream = TcpStream::connect(addr)?;
     let t0 = monotonic_ns();
-    let mut child = Command::new("iptables-restore").arg("--noflush").stdin(Stdio::piped()).spawn()?;
-    child.stdin.take().ok_or("iptables-restore took no stdin")?.write_all(script.as_bytes())?;
-    let status = child.wait()?;
+    stream.write_all(request.as_bytes())?;
+    let mut answer = Vec::new();
+    stream.read_to_end(&mut answer)?;
     let t1 = monotonic_ns();
-    if !status.success() {
-        return Err(format!("iptables-restore exited {status}").into());
+    let text = String::from_utf8_lossy(&answer);
+    let status: u16 = text
+        .split_whitespace()
+        .nth(1)
+        .ok_or("no status in the answer")?
+        .parse()?;
+    if !(200..300).contains(&status) {
+        let detail = text.split("\r\n\r\n").nth(1).unwrap_or("").trim();
+        return Err(format!("{method} {path} answered {status}: {detail}").into());
     }
-    println!("{}", json!({ "action": action, "peer": peer.to_string(), "t0": t0, "t1": t1 }));
+    println!("{}", json!({ "action": "http", "method": method, "path": path, "status": status, "t0": t0, "t1": t1 }));
     Ok(())
 }
 
+fn run() -> Result<(), Box<dyn Error>> {
+    let args: Vec<String> = std::env::args().skip(1).collect();
+    let action = args.first().ok_or("say an action: blackhole, clear, rst, reject, mtu, netem, offload, http …")?.as_str();
+    if action == "http" {
+        return http(&args);
+    }
+    match action {
+        "netem" | "clear-netem" | "offload" => {
+            let dev = device(args.get(1).ok_or("and a device")?)?;
+            let rest: Vec<&str> = args[2..].iter().map(String::as_str).collect();
+            let (spec, at) = match rest.iter().position(|token| *token == "--at") {
+                Some(index) => (
+                    rest[..index].to_vec(),
+                    Some(rest.get(index + 1).ok_or("--at takes an instant in ns")?.parse::<u64>()?),
+                ),
+                None => (rest, None),
+            };
+            if let Some(at) = at {
+                until(at)?;
+            }
+            let t0 = monotonic_ns();
+            match action {
+                "netem" => {
+                    if spec.is_empty() || !spec.iter().all(|token| netem_token(token)) {
+                        return Err(format!("not a netem specification: {}", spec.join(" ")).into());
+                    }
+                    let mut tc = vec!["qdisc", "replace", "dev", dev, "root", "netem"];
+                    tc.extend(spec.iter().copied());
+                    command("tc", &tc)?;
+                }
+                "clear-netem" => command("tc", &["qdisc", "del", "dev", dev, "root"])?,
+                _ => {
+                    let state = match spec.as_slice() {
+                        ["on"] => "on",
+                        ["off"] => "off",
+                        _ => return Err("offload takes on or off".into()),
+                    };
+                    command("ethtool", &["-K", dev, "tso", state, "gso", state, "gro", state])?;
+                }
+            }
+            let t1 = monotonic_ns();
+            println!("{}", json!({ "action": action, "device": dev, "spec": spec, "t0": t0, "t1": t1 }));
+            Ok(())
+        }
+        _ => {
+            // parsed, never pasted: only an address reaches the rules
+            let peer: Ipv4Addr = args.get(1).ok_or("and the peer's address")?.parse()?;
+            let opts = options(&args[2..])?;
+            let script = format!("*filter\n{}COMMIT\n", rules(action, peer, &opts)?);
+            if let Some(at) = opts.at {
+                until(at)?;
+            }
+            let t0 = monotonic_ns();
+            let mut child =
+                Command::new("iptables-restore").arg("--noflush").stdin(Stdio::piped()).spawn()?;
+            child.stdin.take().ok_or("iptables-restore took no stdin")?.write_all(script.as_bytes())?;
+            let status = child.wait()?;
+            let t1 = monotonic_ns();
+            if !status.success() {
+                return Err(format!("iptables-restore exited {status}").into());
+            }
+            println!("{}", json!({ "action": action, "peer": peer.to_string(), "t0": t0, "t1": t1 }));
+            Ok(())
+        }
+    }
+}
+
 fn main() {
     if let Err(error) = run() {
         eprintln!("inject: {error}");
         std::process::exit(1);
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn netem_specifications_are_words_and_numbers_with_units() {
+        for token in ["delay", "100ms", "10ms", "loss", "5%", "25.5%", "distribution", "normal", "1"] {
+            assert!(netem_token(token), "{token}");
+        }
+        for token in ["", ";", "100ms;rm", "$(x)", "-1", "a b"] {
+            assert!(!netem_token(token), "{token}");
+        }
+    }
+
+    #[test]
+    fn rules_cover_each_direction_code_and_length() {
+        let peer: Ipv4Addr = "10.0.0.2".parse().unwrap();
+        let with = |direction: &str| Options { direction: Some(direction.into()), ..Options::default() };
+        assert_eq!(rules("blackhole", peer, &with("in")).unwrap(), "-I INPUT -s 10.0.0.2/32 -j DROP\n");
+        assert_eq!(rules("clear", peer, &with("out")).unwrap(), "-D OUTPUT -d 10.0.0.2/32 -j DROP\n");
+        let code = Options { code: Some("net".into()), ..Options::default() };
+        assert!(rules("reject", peer, &code).unwrap().contains("icmp-net-unreachable"));
+        let above = Options { above: Some(1200), ..Options::default() };
+        assert_eq!(
+            rules("mtu", peer, &above).unwrap(),
+            "-I INPUT -s 10.0.0.2/32 -p tcp -m length --length 1201:65535 -j DROP\n"
+        );
+        assert!(rules("reject", peer, &Options::default()).is_err());
+    }
+
+    #[test]
+    fn http_requests_take_only_a_method_a_path_and_a_length() {
+        assert!(http_path("/proxies/redis/toxics").is_ok());
+        for path in ["proxies", "/a b", "/a;b", "/a?b=1", "/$(x)"] {
+            assert!(http_path(path).is_err(), "{path}");
+        }
+        let request = http_request("POST", "10.0.0.3:8474", "/proxies/redis/toxics", "{}").unwrap();
+        assert!(request.starts_with("POST /proxies/redis/toxics HTTP/1.1\r\n"));
+        assert!(request.contains("Content-Length: 2\r\n"));
+        assert!(request.ends_with("\r\n\r\n{}"));
+        assert!(http_request("PUT", "h", "/", "").is_err());
+    }
+}

package/image/router/Dockerfile

@@ -0,0 +1,8 @@
+# The router of the harness: a NAT box with conntrack, the middle of a stateful middlebox chain. It
+# forgets an idle flow at its idle timeout, and its form tells what a late packet then meets (mbx.sh).
+# lab.ts stages the build context: this folder, under harness/image/router. The base is pinned by
+# digest, and the manifest of each run records the packages that apk installs.
+FROM alpine:3.23@sha256:865b95f46d98cf867a156fe4a135ad3fe50d2056aa3f25ed31662dff6da4eb62
+RUN apk add --no-cache conntrack-tools nftables iproute2 python3
+COPY harness/image/router/mbx.sh harness/image/router/ctwatch.py harness/image/router/resetter.py /opt/mbx/
+ENTRYPOINT ["/bin/sh", "/opt/mbx/mbx.sh"]

package/image/router/ctwatch.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+"""The conntrack events of the router, as JSON lines on the monotonic clock.
+
+Each line is {"ev": "ct", "t": <CLOCK_MONOTONIC ns>, "type": "NEW|UPDATE|DESTROY", "state": ...,
+"orig": "src:sport>dst:dport", "reply": "..."}. The stamp is the time that this process read the
+event, which is within a millisecond of the kernel. A listing every PUMP seconds evicts the expired
+entries that it walks past, thus a DESTROY follows an expiry within PUMP plus two listings. A
+listing that takes longer than SLOW seconds writes a "pump-slow" line with its start and its
+length: near it, that bound of the lag does not hold.
+
+  ctwatch.py [--pump SECONDS] [--slow SECONDS]
+"""
+import argparse, json, re, subprocess, sys, threading, time
+
+p = argparse.ArgumentParser()
+p.add_argument("--pump", type=float, default=0.1)
+p.add_argument("--slow", type=float, default=0.02)
+a = p.parse_args()
+
+PAT = re.compile(
+    r"\[(?P<type>NEW|UPDATE|DESTROY)\]\s+tcp\s+\d+\s+(?:\d+\s+)?(?P<state>[A-Z_]+)?\s*"
+    r"src=(?P<s>\S+) dst=(?P<d>\S+) sport=(?P<sp>\d+) dport=(?P<dp>\d+).*?"
+    r"src=(?P<rs>\S+) dst=(?P<rd>\S+) sport=(?P<rsp>\d+) dport=(?P<rdp>\d+)"
+)
+lock = threading.Lock()
+
+
+def now():
+    return time.clock_gettime_ns(time.CLOCK_MONOTONIC)
+
+
+def emit(**fields):
+    with lock:
+        print(json.dumps({"t": now(), **fields}), flush=True)
+
+
+def pump():
+    while True:
+        start = now()
+        subprocess.run(["conntrack", "-L", "-p", "tcp"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        took = (now() - start) / 1e9
+        if took > a.slow:
+            with lock:
+                print(json.dumps({"ev": "pump-slow", "t": start, "took": took}), flush=True)
+        time.sleep(a.pump)
+
+
+if a.pump > 0:
+    threading.Thread(target=pump, daemon=True).start()
+proc = subprocess.Popen(
+    ["conntrack", "-E", "-p", "tcp"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1
+)
+emit(ev="ct-watch", pump=a.pump, slow=a.slow)
+for line in proc.stdout:
+    m = PAT.search(line)
+    if m is None:
+        emit(ev="ct-unparsed", line=line.strip())
+        continue
+    emit(
+        ev="ct",
+        type=m["type"],
+        state=m["state"],
+        orig=f'{m["s"]}:{m["sp"]}>{m["d"]}:{m["dp"]}',
+        reply=f'{m["rs"]}:{m["rsp"]}>{m["rd"]}:{m["rdp"]}',
+    )
+sys.exit(proc.wait())

package/image/router/mbx.sh

@@ -0,0 +1,61 @@
+#!/bin/sh
+# The router of a middlebox chain: one image for the forms of a middlebox at its idle expiry.
+#
+#   FORM=reset-both|reset-next|silent-drop|recreate|none IDLE=<seconds> [LAN=eth0] [WAN=eth1] [PUMP=0.1] [SLOW=0.02]
+#
+# The container is privileged, so it sets the sysctls of its own namespace. Docker Desktop keeps
+# /proc/sys read-only in a container that is not. The network of the side of the client is LAN,
+# and the harness connects the network of the side of the target (WAN) after the start.
+#
+# FORM         at the expiry                  a late packet from inside      a late packet from outside
+# reset-both   a reset to the two ends        dropped                        dropped
+# reset-next   nothing                        a reset to its sender          dropped
+# silent-drop  nothing                        dropped                        dropped
+# recreate     nothing                        passes, and starts the flow    dropped
+# none         nothing                        passes (a plain NAT router)    a reset from the router
+#
+# A lookup treats an expired entry as gone, so the router forgets a flow at its timeout to the
+# jiffy. The DESTROY event of conntrack waits for the garbage collector or a lookup: ctwatch.py
+# lists the table every PUMP seconds, so the event follows the expiry within PUMP and two listings.
+# A listing longer than SLOW seconds writes a pump-slow line: near it, that bound does not hold.
+set -eu
+FORM=${FORM:-none}; LAN=${LAN:-eth0}; WAN=${WAN:-eth1}; PUMP=${PUMP:-0.1}; SLOW=${SLOW:-0.02}; IDLE=${IDLE:-432000}
+sysctl -q -w net.ipv4.ip_forward=1
+# the conntrack sysctls of a namespace appear when it first uses conntrack: a listing does that
+conntrack -L >/dev/null 2>&1 || true
+sysctl -q -w net.netfilter.nf_conntrack_events=1
+sysctl -q -w "net.netfilter.nf_conntrack_tcp_timeout_established=$IDLE"
+i=0
+while ! ip link show "$WAN" >/dev/null 2>&1; do
+  i=$((i + 1)); [ "$i" -gt 600 ] && { echo "mbx: no $WAN" >&2; exit 65; }; sleep 0.1
+done
+case $FORM in
+  reset-both)  INSIDE=drop; OUTSIDE=drop ;;
+  reset-next)  INSIDE=rst;  OUTSIDE=drop ;;
+  silent-drop) INSIDE=drop; OUTSIDE=drop ;;
+  recreate)    INSIDE=pass; OUTSIDE=drop ;;
+  none)        INSIDE=pass; OUTSIDE=rst ;;
+  *) echo "mbx: unknown FORM=$FORM" >&2; exit 64 ;;
+esac
+verdict() { case $1 in rst) echo "reject with tcp reset" ;; drop) echo drop ;; *) echo accept ;; esac; }
+# OUTSIDE=rst: accept, and the stack of the router answers the unknown port with an exact reset
+outv=$(verdict "$OUTSIDE"); [ "$OUTSIDE" = rst ] && outv=accept
+nft -f - <<EOF
+table ip mbx {
+  chain mbx_post { type nat hook postrouting priority srcnat; policy accept; oifname "$WAN" masquerade; }
+  chain mbx_fwd {
+    type filter hook forward priority filter; policy accept;
+    iifname "$LAN" meta l4proto tcp ct state new,invalid tcp flags & (syn|ack) != syn counter $(verdict "$INSIDE")
+  }
+  chain mbx_in {
+    type filter hook input priority filter; policy accept;
+    iifname "$WAN" meta l4proto tcp ct state new,invalid tcp flags & (syn|ack) != syn counter $outv
+  }
+}
+EOF
+echo "{\"ev\":\"mbx-ready\",\"t\":0,\"form\":\"$FORM\",\"idle\":$(cat /proc/sys/net/netfilter/nf_conntrack_tcp_timeout_established),\"inside\":\"$INSIDE\",\"outside\":\"$OUTSIDE\"}"
+if [ "$FORM" = reset-both ]; then
+  python3 /opt/mbx/ctwatch.py --pump "$PUMP" --slow "$SLOW" &
+  exec python3 /opt/mbx/resetter.py "$LAN" "$WAN"
+fi
+exec python3 /opt/mbx/ctwatch.py --pump "$PUMP" --slow "$SLOW"