@mark1russell7/kind 0.1.2 → 0.1.4

package/image/agent.mjs

@@ -2,15 +2,24 @@
 // container, thus it sees the processes of each pod of the node, and PID 1 is the init of the node.
 //
 //   node agent.mjs serve       polls the rules of kube-proxy in the namespaces of the node, as
-//                              /tmp/rules.json asks, and prints each change of the backends
+//                              /tmp/rules.json asks, and prints each change of the backends. It
+//                              also reads the conntrack table of the node (procfs, or netlink
+//                              with the conntrack tool of the node). It prints each entry
+//                              that keeps its NAT to a pod that the rules dropped (`stale`), each
+//                              handshake that stopped for a second (`stalled`). Each line
+//                              `rules-alive` also counts the reads of the table (`ct`). Each
+//                              second it reads the counters of dropped packets of the node and
+//                              of each client pod, and prints each one that grew (`drops`)
 //   node agent.mjs act JSON    does one act and prints its stamps on CLOCK_MONOTONIC
 //
 // The acts: signal (a signal to a PID), pause (SIGSTOP, then SIGCONT after ms), write (a file, for
 // example cpu.max or a flag file in the root of a container), remove (a file), stall (the stall
 // seam of the app: the ms in /tmp/lab-stall, then SIGUSR2), and now.
 import { spawnSync } from "node:child_process";
-import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { backendsOf } from "./backends.ts";
+import { entriesOf, flowsOf, staleOf, stalledOf } from "./conntrack.ts";
+import { devOf, grownOf, snmpOf, softnetOf, unresolvedOf } from "./drops.ts";
 
 const now = () => Number(process.hrtime.bigint()) / 1e6;
 const node = process.env.NODE_NAME ?? "";
@@ -24,6 +33,115 @@ const DUMP = {
   ipvs: ["nsenter", "-t", "1", "-n", "--", "cat", "/proc/net/ip_vs"],
 };
 
+/**
+ * the TCP lines of the conntrack table of the node's network namespace: those of the ClusterIP and
+ * those of a handshake. A kernel without CONFIG_NF_CONNTRACK_PROCFS has no /proc/net/nf_conntrack
+ * (the kernel of GitHub's runner, 6.17), thus the agent then asks netlink with the conntrack tool of
+ * the node. Both give the same form of line. The first read decides the source. `text` is null when
+ * no source answers, and `why` tells why.
+ *
+ * A dump through netlink walks each bucket of the table of the whole kernel with the softirqs off,
+ * whatever its filter. A segment that comes in that walk waits for its end. Thus each read is one
+ * dump of TCP, and the agent filters it. The workflow makes the table small (kind.yml)
+ */
+const SOURCES = {
+  procfs: (ip) => [["nsenter", "-t", "1", "-n", "--", "grep", "-F", "-e", ` dst=${ip} `, "-e", "[UNREPLIED]", "-e", " SYN_RECV ", "/proc/net/nf_conntrack"]],
+  netlink: () => [["nsenter", "-t", "1", "-m", "-n", "--", "conntrack", "-L", "-p", "tcp"]],
+};
+let source = null;
+const conntrackOf = (ip) => {
+  const why = [];
+  for (const name of source === null ? Object.keys(SOURCES) : [source]) {
+    const lines = new Set();
+    let ok = true;
+    for (const [command, ...args] of SOURCES[name](ip)) {
+      const run = spawnSync(command, args, { encoding: "utf8", maxBuffer: 64 << 20 });
+      // grep exits 1 when no line matches; conntrack exits 0 with no entry
+      if (run.status === 0 || (name === "procfs" && run.status === 1)) {
+        for (const line of (run.stdout ?? "").split("\n")) if (line.trim() !== "") lines.add(line);
+      } else {
+        ok = false;
+        why.push(`${name}: ${(run.stderr ?? "").trim().slice(0, 200) || `exit ${run.status ?? run.signal ?? run.error?.message}`}`);
+        break;
+      }
+    }
+    if (ok) {
+      source = name;
+      return { text: [...lines].join("\n"), why: null };
+    }
+  }
+  return { text: null, why: why.join("; ") };
+};
+/** the PIDs of the clients of the node: the agent is in the PID namespace of the node, thus /proc holds each pod */
+const clientsOf = () => {
+  const out = [];
+  for (const name of readdirSync("/proc")) {
+    if (!/^\d+$/.test(name)) continue;
+    try {
+      if (readFileSync(`/proc/${name}/cmdline`, "utf8").includes("client.mjs")) out.push(name);
+    } catch {
+      // the process ended in the scan
+    }
+  }
+  return out;
+};
+
+/**
+ * the counters of dropped packets, by name. For the node: the drops of the backlogs and the squeezes
+ * of the softirqs (the kernel, all CPUs), and the drops of each veth. For each client pod, through
+ * /proc/<pid>/net: the IP discards and the sends with no route, the failed connects, the drops of
+ * each interface, and the neighbors with no address yet
+ */
+const countersOf = () => {
+  const values = {};
+  const read = (path) => {
+    try {
+      return readFileSync(path, "utf8");
+    } catch {
+      return null;
+    }
+  };
+  const softnet = read("/proc/1/net/softnet_stat");
+  if (softnet !== null) {
+    const { dropped, squeezed } = softnetOf(softnet);
+    values["kernel.backlog-drops"] = dropped;
+    values["kernel.squeezes"] = squeezed;
+  }
+  const dev = read("/proc/1/net/dev");
+  if (dev !== null) {
+    for (const [name, drops] of Object.entries(devOf(dev))) {
+      if (!name.startsWith("veth")) continue;
+      values[`node.${name}.rx-drops`] = drops.rx;
+      values[`node.${name}.tx-drops`] = drops.tx;
+    }
+  }
+  for (const pid of clientsOf()) {
+    const snmp = read(`/proc/${pid}/net/snmp`);
+    const own = read(`/proc/${pid}/net/dev`);
+    const arp = read(`/proc/${pid}/net/arp`);
+    if (snmp === null || own === null || arp === null) continue;
+    const ip = snmpOf(snmp, "Ip");
+    const tcp = snmpOf(snmp, "Tcp");
+    values[`client.${pid}.ip-out-discards`] = ip["OutDiscards"] ?? 0;
+    values[`client.${pid}.ip-out-no-routes`] = ip["OutNoRoutes"] ?? 0;
+    values[`client.${pid}.tcp-attempt-fails`] = tcp["AttemptFails"] ?? 0;
+    for (const [name, drops] of Object.entries(devOf(own))) {
+      if (name === "lo") continue;
+      values[`client.${pid}.${name}.rx-drops`] = drops.rx;
+      values[`client.${pid}.${name}.tx-drops`] = drops.tx;
+    }
+    values[`client.${pid}.unresolved-neighbors`] = unresolvedOf(arp);
+  }
+  return values;
+};
+
+/**
+ * how often the agent reads the conntrack table, in ms. Three dumps each 250 ms on each node took
+ * three quarters of the CPUs of a runner (run 39066066532), and the lab waited for an idle VM until
+ * the end. An entry of a SYN with no reply lives 120 s, thus a read each second sees it
+ */
+const CONNTRACK_EVERY = 1000;
+
 const serve = async () => {
   let last = null;
   let previous = null;
@@ -31,6 +149,15 @@ const serve = async () => {
   let spent = 0;
   let beat = now();
   let dumped = null;
+  // the conntrack read: its last time, the stale entries and the stalled handshakes that it saw,
+  // whether the table exists, and the count of the reads since the last beat
+  let read = 0;
+  let seen = new Map();
+  let stuck = new Map();
+  let conntrack = true;
+  let count = { reads: 0, ms: 0, lines: 0, vip: 0, states: {}, syn: [], error: null };
+  // the counters of dropped packets at the last read, and the count of that last read's values
+  let counters = null;
   for (;;) {
     if (!existsSync("/tmp/rules.json")) {
       last = null;
@@ -58,10 +185,64 @@ const serve = async () => {
         say({ ev: "rules", t, prev: previous, took: Math.round(took * 1000) / 1000, b: backends });
         last = key;
       }
+      // a new SYN on the port of a stale entry goes to the pod of the entry, not to a backend of the rules
+      if (conntrack && t - read >= CONNTRACK_EVERY) {
+        read = t;
+        try {
+          const { text, why } = conntrackOf(rule.ip);
+          count.reads += 1;
+          count.ms += now() - t;
+          if (text === null) {
+            // the lab drops the lines before a run, thus each beat tells it again in ct.error
+            say({ ev: "conntrack-fail", t, why });
+            count.error = why;
+            conntrack = false;
+          } else {
+            const flows = flowsOf(text);
+            const entries = entriesOf(text, rule.ip, rule.port);
+            count.lines = flows.length;
+            count.vip = entries.length;
+            count.states = {};
+            for (const entry of entries) count.states[entry.state] = (count.states[entry.state] ?? 0) + 1;
+            // a sample of the entries in SYN_SENT: a fresh SYN has the life 120, and an old one less
+            count.syn = entries.filter((entry) => entry.state === "SYN_SENT").slice(0, 3).map((entry) => ({ client: entry.client, backend: entry.backend, ttl: entry.ttl, unreplied: entry.unreplied }));
+            const next = new Map();
+            for (const entry of staleOf(entries, backends)) {
+              const id = `${entry.client}>${entry.backend}`;
+              next.set(id, entry.state);
+              // each entry once, and again when its state changes
+              if (seen.get(id) !== entry.state) say({ ev: "stale", t, state: entry.state, ttl: entry.ttl, client: entry.client, backend: entry.backend, unreplied: entry.unreplied });
+            }
+            seen = next;
+            // a connection that times out: where its SYN went, and whether an answer came on this node
+            const held = new Map();
+            for (const flow of stalledOf(flows, 1)) {
+              const id = `${flow.from}>${flow.to}`;
+              held.set(id, flow.state);
+              if (stuck.get(id) !== flow.state) say({ ev: "stalled", t, state: flow.state, ttl: flow.ttl, from: flow.from, to: flow.to, reply: flow.reply, unreplied: flow.unreplied });
+            }
+            stuck = held;
+          }
+        } catch (error) {
+          // a read that fails stops the reads, and the next beat tells why
+          count.error = String(error?.stack ?? error).slice(0, 600);
+          conntrack = false;
+        }
+      }
     }
     previous = t;
     if (t - beat >= 1000) {
-      say({ ev: "rules-alive", t, polls, ms: Math.round((spent / polls) * 100) / 100 });
+      // a packet lost between a client and the conntrack of the node leaves a trace in one of these
+      const next = countersOf();
+      if (counters !== null) {
+        const grown = grownOf(counters, next);
+        if (Object.keys(grown).length > 0) say({ ev: "drops", t, grown });
+      }
+      counters = next;
+      // the reads of the conntrack table since the last beat: a beat without them did not read it
+      const ct = { source, reads: count.reads, ms: count.reads === 0 ? 0 : Math.round((count.ms / count.reads) * 100) / 100, lines: count.lines, vip: count.vip, states: count.states, syn: count.syn, on: conntrack, ...(count.error === null ? {} : { error: count.error }) };
+      say({ ev: "rules-alive", t, polls, ms: Math.round((spent / polls) * 100) / 100, ct });
+      count = { ...count, reads: 0, ms: 0 };
       beat = t;
       polls = 0;
       spent = 0;

package/image/client.mjs

@@ -1,7 +1,7 @@
 // A client of a kind cell: an open loop of requests to the ClusterIP of the service, each on a new
 // connection, thus kube-proxy picks a backend for each one. Each request is one JSON line with its
-// start on CLOCK_MONOTONIC, its duration and its outcome: ok, 503, refused, reset, timeout,
-// unreachable, or another error.
+// start on CLOCK_MONOTONIC, its duration, its outcome (ok, 503, refused, reset, timeout,
+// unreachable, or another error) and its local port (`lp`).
 import http from "node:http";
 
 const [host, port] = (process.env.TARGET ?? "").split(":");
@@ -29,16 +29,27 @@ const outcomeOf = (error) => {
   return "error";
 };
 
+// the local port of a socket, or null once its handle is closed. The kernel gives the port at
+// connect(), and a refusal closes the handle in the poll phase of the loop: before setImmediate
+const portOf = (socket) => {
+  const out = {};
+  socket?._handle?.getsockname?.(out);
+  return typeof out.port === "number" ? out.port : null;
+};
+
 const fire = () => {
   const t = now();
   let done = false;
-  // the local port of the connection: a failed request names it, to find a reuse of the port across connections
+  // the local port of the connection, on each request: a conntrack entry is per port, thus a failure
+  // on a port shows the connection before it on the same port, and the backend that it went to
   let lp = null;
+  let socket = null;
   const finish = (o, extra = {}) => {
     if (done) return;
     done = true;
     clearTimeout(timer);
-    say({ ev: "req", t, d: Math.round((now() - t) * 1000) / 1000, o, ...extra, ...(o === "ok" ? {} : { lp }) });
+    lp ??= portOf(socket);
+    say({ ev: "req", t, d: Math.round((now() - t) * 1000) / 1000, o, ...extra, lp });
   };
   const req = http.request({ host, port: Number(port), path: "/work", agent: false, headers: { connection: "close" } }, (res) => {
     let body = "";
@@ -50,15 +61,20 @@ const fire = () => {
     res.on("error", (error) => finish(outcomeOf(error), { e: error.code ?? String(error) }));
   });
   const timer = setTimeout(() => {
+    // before the destroy closes the handle
+    lp ??= portOf(socket);
     const error = Object.assign(new Error("no answer in time"), { code: "LAB_TIMEOUT" });
     req.destroy(error);
     finish("timeout", { e: "LAB_TIMEOUT" });
   }, timeout);
   req.on("error", (error) => finish(outcomeOf(error), { e: error.code ?? String(error) }));
-  req.on("socket", (socket) => {
-    // the kernel gives the port at connect(), which comes before the next turn of the loop
-    setImmediate(() => {
-      lp = socket.localPort ?? null;
+  req.on("socket", (opened) => {
+    socket = opened;
+    // connect() comes in the tick before this event, thus the handle has the port here, a refused
+    // connection's too: in 200 refusals, each port was here and none at setImmediate
+    lp = portOf(opened);
+    opened.once("connect", () => {
+      lp ??= portOf(opened);
     });
   });
   req.end();

package/package.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package",
   "name": "@mark1russell7/kind",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "Kubernetes in Docker as a lab for connection timers: a kind cluster from a scenario, faults in the namespaces of its pods, observations on the one monotonic clock of the VM, and predictions from the ordering law. Its judge entry reads and judges a raw record without Docker.",
   "keywords": [
     "kubernetes",
@@ -51,6 +51,8 @@
     "image/watch.mjs",
     "src/compose.ts",
     "src/backends.ts",
+    "src/conntrack.ts",
+    "src/drops.ts",
     "README.md",
     "LICENSE"
   ],
@@ -58,7 +60,7 @@
     "node": ">=22.19"
   },
   "dependencies": {
-    "@mark1russell7/harness": "^0.11.0",
+    "@mark1russell7/harness": "^0.11.1",
     "@mark1russell7/quantity": "^0.2.0",
     "@mark1russell7/service": "^0.4.1",
     "testcontainers": "12.2.0"

package/src/conntrack.ts

@@ -0,0 +1,99 @@
+// The parser of the conntrack table of a node. The agent of each node reads the table with it, and
+// the image copies this file, thus it imports nothing.
+
+/** A flow is one conntrack entry of TCP: its state, its life in seconds, its original tuple and the source of its replies. */
+export interface ConntrackFlow {
+  readonly state: string;
+  readonly ttl: number;
+  /** This address is the original source of the flow, as `ip:port`. */
+  readonly from: string;
+  /** This address is the original destination of the flow, as `ip:port`: a ClusterIP before its DNAT. */
+  readonly to: string;
+  /** This address is the source of the replies, as `ip:port`: the backend that the DNAT of the entry chose. */
+  readonly reply: string;
+  /** This flag tells that no packet came back on the flow. */
+  readonly unreplied: boolean;
+}
+
+/** An entry is one conntrack entry of a TCP flow to the ClusterIP. It has a state, a life in seconds, a client and the backend of its NAT. */
+export interface ConntrackEntry {
+  readonly state: string;
+  readonly ttl: number;
+  /** This address is the original source of the flow: the client, as `ip:port`. */
+  readonly client: string;
+  /** This address is the source of the replies: the backend that the DNAT of the entry chose, as `ip:port`. */
+  readonly backend: string;
+  /** This flag tells that no packet came back on the flow. */
+  readonly unreplied: boolean;
+}
+
+/**
+ * These states keep the NAT of an entry for a new SYN on the same port (nf_conntrack_proto_tcp.c at
+ * v6.8). A SYN on an entry in TIME_WAIT removes the entry and looks up again (:1011-1027), thus that
+ * SYN gets a backend from the rules. In each other state, the SYN goes to the backend of the entry.
+ * A CLOSE that a reset from the server made does not reopen either.
+ */
+export const KEEPS_NAT: ReadonlySet<string> = new Set(["SYN_SENT", "SYN_SENT2", "SYN_RECV", "ESTABLISHED", "FIN_WAIT", "CLOSE_WAIT", "LAST_ACK", "CLOSE"]);
+
+/**
+ * These are the lives of an entry in the states of a handshake, in seconds
+ * (nf_conntrack_tcp_timeout_syn_sent and _syn_recv). A retransmission of the SYN does not renew the
+ * life of SYN_SENT (nf_conntrack_proto_tcp.c at v6.8). Thus the life of such an entry tells the age
+ * of its first SYN.
+ */
+export const HANDSHAKE_LIFE: Readonly<Record<string, number>> = { SYN_SENT: 120, SYN_RECV: 60 };
+
+/** This form is one TCP line of `/proc/net/nf_conntrack` or `conntrack -L`. It has the timeout and state, the original tuple, the flags and counters, then the reply tuple. */
+const LINE = /\btcp\s+\d+\s+(\d+)\s+(\S+)\s+src=(\S+) dst=(\S+) sport=(\d+) dport=(\d+)(.*?)\bsrc=(\S+) dst=\S+ sport=(\d+) dport=\d+/;
+
+/** This function gives the TCP flows of a dump of the conntrack table. */
+export const flowsOf = (text: string): ConntrackFlow[] => {
+  const out: ConntrackFlow[] = [];
+  for (const line of text.split("\n")) {
+    const match = LINE.exec(line);
+    if (match === null) continue;
+    out.push({
+      ttl: Number(match[1]),
+      state: match[2]!,
+      from: `${match[3]}:${match[5]}`,
+      to: `${match[4]}:${match[6]}`,
+      reply: `${match[8]}:${match[9]}`,
+      unreplied: match[7]!.includes("[UNREPLIED]"),
+    });
+  }
+  return out;
+};
+
+/** This function gives the TCP entries of a dump of the conntrack table whose original destination is the ClusterIP and port. */
+export const entriesOf = (text: string, ip: string, port: number): ConntrackEntry[] =>
+  flowsOf(text)
+    .filter((flow) => flow.to === `${ip}:${port}`)
+    .map((flow) => ({ state: flow.state, ttl: flow.ttl, client: flow.from, backend: flow.reply, unreplied: flow.unreplied }));
+
+/**
+ * This function gives the entries that keep their NAT to a pod that is not a backend of the rules.
+ * A new SYN from the client port of such an entry goes to that pod, whatever the rules say.
+ */
+export const staleOf = (entries: readonly ConntrackEntry[], backends: readonly string[]): ConntrackEntry[] =>
+  entries.filter((entry) => KEEPS_NAT.has(entry.state) && !backends.includes(entry.backend));
+
+/** This function gives the count of entries that `conntrack -D` deleted, from its last line: "N flow entries have been deleted". */
+export const deletedOf = (text: string): number => {
+  const match = /(\d+) flow entr(?:y|ies) ha(?:s|ve) been deleted/.exec(text);
+  if (match === null) throw new Error(`conntrack -D gave no count: ${text.trim().slice(0, 200)}`);
+  return Number(match[1]);
+};
+
+/**
+ * This function gives the flows whose handshake stopped `age` seconds ago or more. Such a flow is
+ * a SYN with no reply (SYN_SENT), or a SYN and its SYN-ACK with no ACK (SYN_RECV). It is a
+ * connection that times out. On the node of the client, its reply address is the backend of its
+ * DNAT. On the node of that backend, a SYN_SENT tells that the backend did not answer. There, a
+ * SYN_RECV tells that its answer did not come back.
+ */
+export const stalledOf = (flows: readonly ConntrackFlow[], age: number): ConntrackFlow[] =>
+  flows.filter((flow) => {
+    const life = HANDSHAKE_LIFE[flow.state];
+    if (life === undefined || (flow.state === "SYN_SENT" && !flow.unreplied)) return false;
+    return flow.ttl <= life - age;
+  });

package/src/drops.ts

@@ -0,0 +1,63 @@
+// The counters of the dropped packets of a node and of its client pods. The agent of each node
+// reads them each second, and the image copies this file, thus it imports nothing.
+
+/** This function gives the values of one protocol of `/proc/net/snmp`: a line of names, then a line of values. */
+export const snmpOf = (text: string, protocol: string): Record<string, number> => {
+  const lines = text.split("\n").filter((line) => line.startsWith(`${protocol}:`));
+  if (lines.length < 2) return {};
+  const names = lines[0]!.trim().split(/\s+/).slice(1);
+  const values = lines[1]!.trim().split(/\s+/).slice(1).map(Number);
+  return Object.fromEntries(names.map((name, at) => [name, values[at] ?? 0]));
+};
+
+/** These are the dropped packets of one interface: received and sent. */
+export interface InterfaceDrops {
+  readonly rx: number;
+  readonly tx: number;
+}
+
+/** This function gives the dropped packets of each interface of `/proc/net/dev`. The fourth field of each direction is its drops. */
+export const devOf = (text: string): Record<string, InterfaceDrops> => {
+  const out: Record<string, InterfaceDrops> = {};
+  for (const line of text.split("\n").slice(2)) {
+    const match = /^\s*([^:\s]+):\s*(.*)$/.exec(line);
+    if (match === null) continue;
+    const fields = match[2]!.trim().split(/\s+/).map(Number);
+    out[match[1]!] = { rx: fields[3] ?? 0, tx: fields[11] ?? 0 };
+  }
+  return out;
+};
+
+/**
+ * This function gives two sums over the CPUs of `/proc/net/softnet_stat`. The first is the packets
+ * that a full backlog dropped. The second is the times that a softirq used all of its budget. Each
+ * line is one CPU in hexadecimal: the processed packets, the drops, then the squeezes.
+ */
+export const softnetOf = (text: string): { readonly dropped: number; readonly squeezed: number } => {
+  let dropped = 0;
+  let squeezed = 0;
+  for (const line of text.trim().split("\n")) {
+    const fields = line.trim().split(/\s+/);
+    if (fields.length < 3) continue;
+    dropped += parseInt(fields[1]!, 16);
+    squeezed += parseInt(fields[2]!, 16);
+  }
+  return { dropped, squeezed };
+};
+
+/** This function counts the entries of `/proc/net/arp` with no address yet (flags 0x0). A packet to such a neighbor waits in a queue, then drops. */
+export const unresolvedOf = (text: string): number =>
+  text
+    .split("\n")
+    .slice(1)
+    .filter((line) => line.trim() !== "" && line.trim().split(/\s+/)[2] === "0x0").length;
+
+/** This function gives each counter that grew from one read to the next, with its growth. A counter of the second read only is not in it. */
+export const grownOf = (before: Readonly<Record<string, number>>, after: Readonly<Record<string, number>>): Record<string, number> => {
+  const out: Record<string, number> = {};
+  for (const [name, value] of Object.entries(after)) {
+    const last = before[name];
+    if (last !== undefined && value > last) out[name] = value - last;
+  }
+  return out;
+};