@mark1russell7/harness 0.2.0 → 0.3.0

package/image/native/src/bin/inject.rs

@@ -23,6 +23,9 @@
 //!   `netem eth0 delay 100ms 10ms` or `netem eth0 loss 5%`.
 //! - `offload DEV on|off`: the device's TSO, GSO and GRO, so that netem and length matches act on
 //!   segments rather than on GSO super-packets of up to 64 KB.
+//! - `http METHOD ADDR:PORT PATH [BODY]`: one HTTP/1.1 request to an API in the VM, for example a
+//!   toxic of Toxiproxy. The stamps bracket the request and its answer, after the connect, so the
+//!   commit of the toxic is known on the same clock as the kernel's records.
 //!
 //! `--at NS` holds the commit until that instant on CLOCK_MONOTONIC, the clock every stamp is on.
 //! A stall's end is timed that way, from inside the VM: the exec only has to arrive before it, and
@@ -165,9 +168,63 @@ fn command(program: &str, args: &[&str]) -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
+/// an HTTP path: a slash, then letters, digits and `/ - _ .` only
+fn http_path(path: &str) -> Result<&str, Box<dyn Error>> {
+    if path.starts_with('/')
+        && path.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '/' | '-' | '_' | '.'))
+    {
+        Ok(path)
+    } else {
+        Err(format!("not an HTTP path: {path}").into())
+    }
+}
+
+/// the head of one HTTP/1.1 request, with its body
+fn http_request(method: &str, host: &str, path: &str, body: &str) -> Result<String, Box<dyn Error>> {
+    if !matches!(method, "GET" | "POST" | "DELETE") {
+        return Err(format!("not a method: {method}").into());
+    }
+    Ok(format!(
+        "{method} {path} HTTP/1.1\r\nHost: {host}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+        body.len()
+    ))
+}
+
+/// one request to an API: the status, and the stamps around the request and its answer
+fn http(args: &[String]) -> Result<(), Box<dyn Error>> {
+    use std::io::Read;
+    use std::net::{SocketAddrV4, TcpStream};
+    let method = args.get(1).ok_or("http METHOD ADDR:PORT PATH [BODY]")?.as_str();
+    let addr: SocketAddrV4 = args.get(2).ok_or("and the address and port")?.parse()?;
+    let path = http_path(args.get(3).ok_or("and the path")?)?;
+    let body = args.get(4).map_or("", String::as_str);
+    let request = http_request(method, &addr.to_string(), path, body)?;
+    let mut stream = TcpStream::connect(addr)?;
+    let t0 = monotonic_ns();
+    stream.write_all(request.as_bytes())?;
+    let mut answer = Vec::new();
+    stream.read_to_end(&mut answer)?;
+    let t1 = monotonic_ns();
+    let text = String::from_utf8_lossy(&answer);
+    let status: u16 = text
+        .split_whitespace()
+        .nth(1)
+        .ok_or("no status in the answer")?
+        .parse()?;
+    if !(200..300).contains(&status) {
+        let detail = text.split("\r\n\r\n").nth(1).unwrap_or("").trim();
+        return Err(format!("{method} {path} answered {status}: {detail}").into());
+    }
+    println!("{}", json!({ "action": "http", "method": method, "path": path, "status": status, "t0": t0, "t1": t1 }));
+    Ok(())
+}
+
 fn run() -> Result<(), Box<dyn Error>> {
     let args: Vec<String> = std::env::args().skip(1).collect();
-    let action = args.first().ok_or("say an action: blackhole, clear, rst, reject, mtu, netem, offload …")?.as_str();
+    let action = args.first().ok_or("say an action: blackhole, clear, rst, reject, mtu, netem, offload, http …")?.as_str();
+    if action == "http" {
+        return http(&args);
+    }
     match action {
         "netem" | "clear-netem" | "offload" => {
             let dev = device(args.get(1).ok_or("and a device")?)?;
@@ -265,4 +322,17 @@ mod tests {
         );
         assert!(rules("reject", peer, &Options::default()).is_err());
     }
+
+    #[test]
+    fn http_requests_take_only_a_method_a_path_and_a_length() {
+        assert!(http_path("/proxies/redis/toxics").is_ok());
+        for path in ["proxies", "/a b", "/a;b", "/a?b=1", "/$(x)"] {
+            assert!(http_path(path).is_err(), "{path}");
+        }
+        let request = http_request("POST", "10.0.0.3:8474", "/proxies/redis/toxics", "{}").unwrap();
+        assert!(request.starts_with("POST /proxies/redis/toxics HTTP/1.1\r\n"));
+        assert!(request.contains("Content-Length: 2\r\n"));
+        assert!(request.ends_with("\r\n\r\n{}"));
+        assert!(http_request("PUT", "h", "/", "").is_err());
+    }
 }

package/image/router/Dockerfile

@@ -0,0 +1,8 @@
+# The router of the harness: a NAT box with conntrack, the middle of a stateful middlebox chain. It
+# forgets an idle flow at its idle timeout, and its form tells what a late packet then meets (mbx.sh).
+# lab.ts stages the build context: this folder, under harness/image/router. The base is pinned by
+# digest, and the manifest of each run records the packages that apk installs.
+FROM alpine:3.23@sha256:865b95f46d98cf867a156fe4a135ad3fe50d2056aa3f25ed31662dff6da4eb62
+RUN apk add --no-cache conntrack-tools nftables iproute2 python3
+COPY harness/image/router/mbx.sh harness/image/router/ctwatch.py harness/image/router/resetter.py /opt/mbx/
+ENTRYPOINT ["/bin/sh", "/opt/mbx/mbx.sh"]

package/image/router/ctwatch.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+"""The conntrack events of the router, as JSON lines on the monotonic clock.
+
+Each line is {"ev": "ct", "t": <CLOCK_MONOTONIC ns>, "type": "NEW|UPDATE|DESTROY", "state": ...,
+"orig": "src:sport>dst:dport", "reply": "..."}. The stamp is the time that this process read the
+event, which is within a millisecond of the kernel. A listing every PUMP seconds evicts the expired
+entries that it walks past, thus a DESTROY follows an expiry within PUMP plus two listings. A
+listing that takes longer than SLOW seconds writes a "pump-slow" line with its start and its
+length: near it, that bound of the lag does not hold.
+
+  ctwatch.py [--pump SECONDS] [--slow SECONDS]
+"""
+import argparse, json, re, subprocess, sys, threading, time
+
+p = argparse.ArgumentParser()
+p.add_argument("--pump", type=float, default=0.1)
+p.add_argument("--slow", type=float, default=0.02)
+a = p.parse_args()
+
+PAT = re.compile(
+    r"\[(?P<type>NEW|UPDATE|DESTROY)\]\s+tcp\s+\d+\s+(?:\d+\s+)?(?P<state>[A-Z_]+)?\s*"
+    r"src=(?P<s>\S+) dst=(?P<d>\S+) sport=(?P<sp>\d+) dport=(?P<dp>\d+).*?"
+    r"src=(?P<rs>\S+) dst=(?P<rd>\S+) sport=(?P<rsp>\d+) dport=(?P<rdp>\d+)"
+)
+lock = threading.Lock()
+
+
+def now():
+    return time.clock_gettime_ns(time.CLOCK_MONOTONIC)
+
+
+def emit(**fields):
+    with lock:
+        print(json.dumps({"t": now(), **fields}), flush=True)
+
+
+def pump():
+    while True:
+        start = now()
+        subprocess.run(["conntrack", "-L", "-p", "tcp"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        took = (now() - start) / 1e9
+        if took > a.slow:
+            with lock:
+                print(json.dumps({"ev": "pump-slow", "t": start, "took": took}), flush=True)
+        time.sleep(a.pump)
+
+
+if a.pump > 0:
+    threading.Thread(target=pump, daemon=True).start()
+proc = subprocess.Popen(
+    ["conntrack", "-E", "-p", "tcp"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1
+)
+emit(ev="ct-watch", pump=a.pump, slow=a.slow)
+for line in proc.stdout:
+    m = PAT.search(line)
+    if m is None:
+        emit(ev="ct-unparsed", line=line.strip())
+        continue
+    emit(
+        ev="ct",
+        type=m["type"],
+        state=m["state"],
+        orig=f'{m["s"]}:{m["sp"]}>{m["d"]}:{m["dp"]}',
+        reply=f'{m["rs"]}:{m["rsp"]}>{m["rd"]}:{m["rdp"]}',
+    )
+sys.exit(proc.wait())

package/image/router/mbx.sh

@@ -0,0 +1,61 @@
+#!/bin/sh
+# The router of a middlebox chain: one image for the forms of a middlebox at its idle expiry.
+#
+#   FORM=reset-both|reset-next|silent-drop|recreate|none IDLE=<seconds> [LAN=eth0] [WAN=eth1] [PUMP=0.1] [SLOW=0.02]
+#
+# The container is privileged, so it sets the sysctls of its own namespace. Docker Desktop keeps
+# /proc/sys read-only in a container that is not. The network of the side of the client is LAN,
+# and the harness connects the network of the side of the target (WAN) after the start.
+#
+# FORM         at the expiry                  a late packet from inside      a late packet from outside
+# reset-both   a reset to the two ends        dropped                        dropped
+# reset-next   nothing                        a reset to its sender          dropped
+# silent-drop  nothing                        dropped                        dropped
+# recreate     nothing                        passes, and starts the flow    dropped
+# none         nothing                        passes (a plain NAT router)    a reset from the router
+#
+# A lookup treats an expired entry as gone, so the router forgets a flow at its timeout to the
+# jiffy. The DESTROY event of conntrack waits for the garbage collector or a lookup: ctwatch.py
+# lists the table every PUMP seconds, so the event follows the expiry within PUMP and two listings.
+# A listing longer than SLOW seconds writes a pump-slow line: near it, that bound does not hold.
+set -eu
+FORM=${FORM:-none}; LAN=${LAN:-eth0}; WAN=${WAN:-eth1}; PUMP=${PUMP:-0.1}; SLOW=${SLOW:-0.02}; IDLE=${IDLE:-432000}
+sysctl -q -w net.ipv4.ip_forward=1
+# the conntrack sysctls of a namespace appear when it first uses conntrack: a listing does that
+conntrack -L >/dev/null 2>&1 || true
+sysctl -q -w net.netfilter.nf_conntrack_events=1
+sysctl -q -w "net.netfilter.nf_conntrack_tcp_timeout_established=$IDLE"
+i=0
+while ! ip link show "$WAN" >/dev/null 2>&1; do
+  i=$((i + 1)); [ "$i" -gt 600 ] && { echo "mbx: no $WAN" >&2; exit 65; }; sleep 0.1
+done
+case $FORM in
+  reset-both)  INSIDE=drop; OUTSIDE=drop ;;
+  reset-next)  INSIDE=rst;  OUTSIDE=drop ;;
+  silent-drop) INSIDE=drop; OUTSIDE=drop ;;
+  recreate)    INSIDE=pass; OUTSIDE=drop ;;
+  none)        INSIDE=pass; OUTSIDE=rst ;;
+  *) echo "mbx: unknown FORM=$FORM" >&2; exit 64 ;;
+esac
+verdict() { case $1 in rst) echo "reject with tcp reset" ;; drop) echo drop ;; *) echo accept ;; esac; }
+# OUTSIDE=rst: accept, and the stack of the router answers the unknown port with an exact reset
+outv=$(verdict "$OUTSIDE"); [ "$OUTSIDE" = rst ] && outv=accept
+nft -f - <<EOF
+table ip mbx {
+  chain mbx_post { type nat hook postrouting priority srcnat; policy accept; oifname "$WAN" masquerade; }
+  chain mbx_fwd {
+    type filter hook forward priority filter; policy accept;
+    iifname "$LAN" meta l4proto tcp ct state new,invalid tcp flags & (syn|ack) != syn counter $(verdict "$INSIDE")
+  }
+  chain mbx_in {
+    type filter hook input priority filter; policy accept;
+    iifname "$WAN" meta l4proto tcp ct state new,invalid tcp flags & (syn|ack) != syn counter $outv
+  }
+}
+EOF
+echo "{\"ev\":\"mbx-ready\",\"t\":0,\"form\":\"$FORM\",\"idle\":$(cat /proc/sys/net/netfilter/nf_conntrack_tcp_timeout_established),\"inside\":\"$INSIDE\",\"outside\":\"$OUTSIDE\"}"
+if [ "$FORM" = reset-both ]; then
+  python3 /opt/mbx/ctwatch.py --pump "$PUMP" --slow "$SLOW" &
+  exec python3 /opt/mbx/resetter.py "$LAN" "$WAN"
+fi
+exec python3 /opt/mbx/ctwatch.py --pump "$PUMP" --slow "$SLOW"

package/image/router/resetter.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""The reset at expiry: the form of a middlebox that resets the two ends when it forgets a flow.
+
+It listens to the DESTROY events of conntrack in this namespace. For each destroyed TCP flow that
+it saw carry traffic, and never saw closed (FIN or RST), it sends a reset to each end. Each reset
+has the sequence number that its end expects: the last acknowledgment number that the end sent,
+read by a raw sniffer on the interfaces, because conntrack events carry no sequence numbers.
+Each line is JSON on the monotonic clock: {"ev": "reset", "t": ..., "flow": ..., "to": [...]}.
+
+  resetter.py IFACE [IFACE...]
+"""
+import argparse, json, re, socket, struct, subprocess, threading, time
+
+p = argparse.ArgumentParser()
+p.add_argument("ifaces", nargs="+")
+a = p.parse_args()
+
+plock = threading.Lock()
+flock = threading.Lock()
+flows = {}  # (src, sport, dst, dport) -> {"ack", "closed"}
+
+
+def emit(**fields):
+    with plock:
+        print(json.dumps({"t": time.clock_gettime_ns(time.CLOCK_MONOTONIC), **fields}), flush=True)
+
+
+def sniff(ifname):
+    s = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(0x0800))
+    s.bind((ifname, 0))
+    while True:
+        pkt = s.recv(65535)
+        ip = pkt[14:]
+        if len(ip) < 40 or ip[0] >> 4 != 4 or ip[9] != 6:
+            continue
+        ihl = (ip[0] & 0xF) * 4
+        src, dst = socket.inet_ntoa(ip[12:16]), socket.inet_ntoa(ip[16:20])
+        tcp = ip[ihl:]
+        sport, dport, _seq, ack = struct.unpack("!HHII", tcp[:12])
+        fl = tcp[13]
+        with flock:
+            f = flows.setdefault((src, sport, dst, dport), {"ack": None, "closed": False})
+            if fl & 0x10:
+                f["ack"] = ack
+            if fl & 0x05:
+                f["closed"] = True
+
+
+raw = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_RAW)
+
+
+def csum(b):
+    if len(b) % 2:
+        b += b"\0"
+    s = sum(struct.unpack("!%dH" % (len(b) // 2), b))
+    while s >> 16:
+        s = (s & 0xFFFF) + (s >> 16)
+    return ~s & 0xFFFF
+
+
+def send_rst(src, sport, dst, dport, seq):
+    tcp = struct.pack("!HHIIBBHHH", sport, dport, seq, 0, 5 << 4, 0x04, 0, 0, 0)
+    pseudo = socket.inet_aton(src) + socket.inet_aton(dst) + struct.pack("!BBH", 0, 6, len(tcp))
+    tcp = tcp[:16] + struct.pack("!H", csum(pseudo + tcp)) + tcp[18:]
+    iph = struct.pack(
+        "!BBHHHBBH4s4s", 0x45, 0, 20 + len(tcp), 0, 0x4000, 64, 6, 0, socket.inet_aton(src), socket.inet_aton(dst)
+    )
+    raw.sendto(iph + tcp, (dst, 0))
+
+
+PAT = re.compile(r"src=(\S+) dst=(\S+) sport=(\d+) dport=(\d+) .*?src=(\S+) dst=(\S+) sport=(\d+) dport=(\d+)")
+
+
+def main():
+    for i in a.ifaces:
+        threading.Thread(target=sniff, args=(i,), daemon=True).start()
+    proc = subprocess.Popen(
+        ["conntrack", "-E", "-e", "DESTROY", "-p", "tcp"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1
+    )
+    emit(ev="resetter", ifaces=a.ifaces)
+    for line in proc.stdout:
+        m = PAT.search(line)
+        if not m:
+            continue
+        c, s, cp, sp, s2, r, sp2, rp = m.groups()
+        cp, sp, sp2, rp = int(cp), int(sp), int(sp2), int(rp)
+        k_cs, k_sr = (c, cp, s, sp), (s2, sp2, r, rp)
+        with flock:
+            fs = {k: dict(flows[k]) for k in (k_cs, (s, sp, c, cp), (r, rp, s2, sp2), k_sr) if k in flows}
+        if not fs or any(f["closed"] for f in fs.values()):
+            continue
+        seq_c = fs.get(k_cs, {}).get("ack")  # the RCV.NXT of the client
+        seq_s = fs.get(k_sr, {}).get("ack")  # the RCV.NXT of the server, on the translated side
+        sent = []
+        if seq_c is not None:
+            send_rst(s, sp, c, cp, seq_c)
+            sent.append("client")
+        if seq_s is not None:
+            send_rst(r, rp, s2, sp2, seq_s)
+            sent.append("server")
+        emit(ev="reset", flow=f"{c}:{cp}>{s}:{sp}", nat=f"{r}:{rp}", to=sent)
+
+
+main()

package/package.json

@@ -1,8 +1,8 @@
 {
   "$schema": "https://json.schemastore.org/package",
   "name": "@mark1russell7/harness",
-  "version": "0.2.0",
-  "description": "A fault-injection harness for connection timers: a cell runs a client against a stock target in Docker, injects a fault, measures the kernel's own records and judges a prediction.",
+  "version": "0.3.0",
+  "description": "A fault-injection harness for connection timers: a cell runs a client against a stock target in Docker, through a topology of routers and proxies, injects a fault, measures the kernel's own records and judges a prediction.",
   "keywords": [
     "fault-injection",
     "chaos-engineering",
@@ -45,6 +45,10 @@
     "image/node/client-ioredis.mjs",
     "image/node/package.json",
     "image/node/npm-lock.json",
+    "image/router/Dockerfile",
+    "image/router/mbx.sh",
+    "image/router/ctwatch.py",
+    "image/router/resetter.py",
     "README.md",
     "LICENSE"
   ],