@mark1russell7/kind 0.0.0-stage → 0.1.1

package/image/npm-lock.json

@@ -0,0 +1,106 @@
+{
+  "name": "systems-kind-lab",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "systems-kind-lab",
+      "dependencies": {
+        "ioredis": "6.0.0",
+        "zod": "4.6.6"
+      }
+    },
+    "node_modules/@ioredis/commands": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-2.0.0.tgz",
+      "integrity": "sha512-vrx0AE/T0h7cRZwfo1M39Cr+ZhZrkf0V8mQN75wucKCxCLD9l/VX6no3gFvrLqD1IlG/1LtzWovqEw3t0Vr9zg==",
+      "license": "MIT"
+    },
+    "node_modules/cluster-key-slot": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz",
+      "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/debug": {
+      "version": "4.4.3",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
+      "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
+      "license": "MIT",
+      "dependencies": {
+        "ms": "^2.1.3"
+      },
+      "engines": {
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/denque": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+      "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
+    "node_modules/ioredis": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-6.0.0.tgz",
+      "integrity": "sha512-f+Dtubxfpf6KYFq7WVXJoOLn0bk4TJrMrN9SzeE+jrWrCWj7XX3fA6vkryafhADX+GMymRxgDJDOI33COkJc0w==",
+      "license": "MIT",
+      "dependencies": {
+        "@ioredis/commands": "2.0.0",
+        "cluster-key-slot": "1.1.1",
+        "debug": "4.4.3",
+        "denque": "2.1.0",
+        "redis-errors": "1.2.0",
+        "standard-as-callback": "2.1.0"
+      },
+      "engines": {
+        "node": ">=20.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/ioredis"
+      }
+    },
+    "node_modules/ms": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
+      "license": "MIT"
+    },
+    "node_modules/redis-errors": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
+      "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/standard-as-callback": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
+      "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",
+      "license": "MIT"
+    },
+    "node_modules/zod": {
+      "version": "4.6.6",
+      "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.6.tgz",
+      "integrity": "sha512-4UmzQUF5fxvfcYHlbjBmejU+w+hbDnWFx4aS8kBx7nPBHcO2SqPbpAG8snbdg+Xse4pRefR98hgjFgIMCI/UhA==",
+      "license": "MIT",
+      "funding": {
+        "url": "https://github.com/sponsors/colinhacks"
+      }
+    }
+  }
+}

package/image/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "systems-kind-lab",
+  "private": true,
+  "type": "module",
+  "dependencies": {
+    "ioredis": "6.0.0",
+    "zod": "4.6.6"
+  }
+}

package/image/watch.mjs

@@ -0,0 +1,175 @@
+// The watcher of a kind cell. It runs in a container on the Docker network of the cluster, thus in
+// the VM, and it stamps each event on CLOCK_MONOTONIC when it arrives. It watches the pods, the
+// EndpointSlices and the events of one namespace through the API, and it follows the log of each
+// pod with the label lab.systems/log=1. A log line is JSON with its own stamp, and the watcher
+// passes it on as it is.
+import https from "node:https";
+
+const kube = JSON.parse(process.env.KUBE ?? "{}");
+const namespaces = (process.env.NAMESPACES ?? "default").split(",");
+const now = () => Number(process.hrtime.bigint()) / 1e6;
+const server = new URL(kube.server);
+const agent = new https.Agent({
+  ca: Buffer.from(kube.ca, "base64"),
+  cert: Buffer.from(kube.cert, "base64"),
+  key: Buffer.from(kube.key, "base64"),
+  keepAlive: true,
+});
+
+let pending = [];
+const flush = () => {
+  if (pending.length > 0) process.stdout.write(pending.join(""));
+  pending = [];
+};
+setInterval(flush, 50);
+const say = (line) => pending.push(`${JSON.stringify(line)}\n`);
+
+/** a stream of lines from the API, again after each end, from the last resource version it saw */
+const stream = (path, query, onLine, again = true) => {
+  let version = null;
+  const open = () => {
+    const search = new URLSearchParams(query);
+    if (version !== null) search.set("resourceVersion", version);
+    const req = https.request(
+      { host: server.hostname, port: server.port, path: `${path}?${search}`, method: "GET", agent },
+      (res) => {
+        if (res.statusCode !== 200) {
+          say({ ev: "watch-fail", t: now(), path, status: res.statusCode });
+          res.resume();
+          if (res.statusCode === 410) version = null;
+          if (again) setTimeout(open, 500);
+          return;
+        }
+        let partial = "";
+        res.setEncoding("utf8");
+        res.on("data", (chunk) => {
+          const t = now();
+          partial += chunk;
+          const split = partial.split("\n");
+          partial = split.pop() ?? "";
+          for (const text of split) {
+            if (text.trim() === "") continue;
+            const next = onLine(text, t);
+            if (typeof next === "string") version = next;
+          }
+        });
+        res.on("end", () => {
+          if (again) setTimeout(open, 100);
+        });
+        res.on("error", () => undefined);
+      },
+    );
+    req.on("error", (error) => {
+      say({ ev: "watch-fail", t: now(), path, why: String(error.message) });
+      if (again) setTimeout(open, 500);
+    });
+    req.end();
+  };
+  open();
+};
+
+const stateOf = (state) => {
+  if (state?.running) return "running";
+  if (state?.waiting) return `waiting:${state.waiting.reason ?? ""}`;
+  if (state?.terminated) return `terminated:${state.terminated.reason ?? ""}:${state.terminated.exitCode}`;
+  return "none";
+};
+
+const following = new Set();
+const follow = (ns, pod, container, id) => {
+  const key = `${ns}/${pod}/${container}/${id}`;
+  if (following.has(key)) return;
+  following.add(key);
+  stream(
+    `/api/v1/namespaces/${ns}/pods/${pod}/log`,
+    { container, follow: "true" },
+    (text) => {
+      if (text.startsWith("{")) pending.push(`${text}\n`);
+      else say({ ev: "text", t: now(), p: pod, text: text.slice(0, 500) });
+    },
+    false,
+  );
+};
+
+for (const ns of namespaces) {
+  stream(`/api/v1/namespaces/${ns}/pods`, { watch: "1", allowWatchBookmarks: "true" }, (text, t) => {
+    const event = JSON.parse(text);
+    const pod = event.object;
+    if (event.type === "BOOKMARK") return pod?.metadata?.resourceVersion;
+    if (event.type === "ERROR") {
+      say({ ev: "watch-fail", t, path: "pods", status: pod?.code });
+      return null;
+    }
+    const ready = (pod.status?.conditions ?? []).find((condition) => condition.type === "Ready")?.status === "True";
+    const containers = (pod.status?.containerStatuses ?? []).map((status) => ({
+      n: status.name,
+      r: status.ready,
+      rc: status.restartCount,
+      st: stateOf(status.state),
+      last: stateOf(status.lastState),
+      id: status.containerID ?? null,
+    }));
+    say({
+      ev: "pod",
+      t,
+      type: event.type,
+      ns,
+      name: pod.metadata.name,
+      node: pod.spec?.nodeName ?? null,
+      ip: pod.status?.podIP ?? null,
+      del: pod.metadata.deletionTimestamp !== undefined,
+      ready,
+      phase: pod.status?.phase ?? null,
+      cs: containers,
+      labels: pod.metadata.labels ?? {},
+    });
+    if (event.type !== "DELETED" && pod.metadata.labels?.["lab.systems/log"] === "1")
+      for (const status of pod.status?.containerStatuses ?? [])
+        if (status.state?.running && status.containerID) follow(ns, pod.metadata.name, status.name, status.containerID);
+    return pod.metadata.resourceVersion;
+  });
+  stream(`/apis/discovery.k8s.io/v1/namespaces/${ns}/endpointslices`, { watch: "1", allowWatchBookmarks: "true" }, (text, t) => {
+    const event = JSON.parse(text);
+    const slice = event.object;
+    if (event.type === "BOOKMARK") return slice?.metadata?.resourceVersion;
+    if (event.type === "ERROR") return null;
+    say({
+      ev: "slice",
+      t,
+      type: event.type,
+      ns,
+      name: slice.metadata.name,
+      svc: slice.metadata.labels?.["kubernetes.io/service-name"] ?? null,
+      eps: (slice.endpoints ?? []).map((endpoint) => ({
+        ip: endpoint.addresses?.[0] ?? null,
+        r: endpoint.conditions?.ready ?? null,
+        s: endpoint.conditions?.serving ?? null,
+        x: endpoint.conditions?.terminating ?? null,
+        pod: endpoint.targetRef?.name ?? null,
+        node: endpoint.nodeName ?? null,
+      })),
+    });
+    return slice.metadata.resourceVersion;
+  });
+  stream(`/api/v1/namespaces/${ns}/events`, { watch: "1", allowWatchBookmarks: "true" }, (text, t) => {
+    const event = JSON.parse(text);
+    const object = event.object;
+    if (event.type === "BOOKMARK") return object?.metadata?.resourceVersion;
+    if (event.type === "ERROR") return null;
+    say({
+      ev: "event",
+      t,
+      type: event.type,
+      ns,
+      reason: object.reason ?? null,
+      msg: String(object.message ?? "").slice(0, 300),
+      obj: object.involvedObject?.name ?? null,
+      kind: object.involvedObject?.kind ?? null,
+      n: object.count ?? object.series?.count ?? 1,
+      last: object.lastTimestamp ?? object.eventTime ?? null,
+      src: object.source?.component ?? object.reportingComponent ?? null,
+    });
+    return object.metadata.resourceVersion;
+  });
+}
+say({ ev: "watcher", what: "start", t: now(), namespaces });

package/package.json

@@ -1,6 +1,86 @@
 {
+  "$schema": "https://json.schemastore.org/package",
   "name": "@mark1russell7/kind",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.1.1",
+  "description": "Kubernetes in Docker as a lab for connection timers: a kind cluster from a scenario, faults in the namespaces of its pods, observations on the one monotonic clock of the VM, and predictions from the ordering law. Its judge entry reads and judges a raw record without Docker.",
+  "keywords": [
+    "kubernetes",
+    "kind",
+    "kube-proxy",
+    "endpoints",
+    "readiness",
+    "graceful-shutdown",
+    "fault-injection",
+    "chaos-engineering",
+    "testcontainers"
+  ],
+  "homepage": "https://mark1russell7.github.io/systems/",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mark1russell7/systems.git",
+    "directory": "packages/kind"
+  },
+  "bugs": {
+    "url": "https://github.com/mark1russell7/systems/issues"
+  },
+  "license": "MIT",
+  "author": "Mark Russell (https://github.com/mark1russell7)",
+  "type": "module",
+  "sideEffects": false,
+  "main": "./build/index.js",
+  "types": "./build/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./build/index.d.ts",
+      "default": "./build/index.js"
+    },
+    "./judge": {
+      "types": "./build/judge.d.ts",
+      "default": "./build/judge.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "build",
+    "image/Dockerfile",
+    "image/package.json",
+    "image/npm-lock.json",
+    "image/app.mjs",
+    "image/client.mjs",
+    "image/agent.mjs",
+    "image/watch.mjs",
+    "src/compose.ts",
+    "src/backends.ts",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=22.19"
+  },
+  "dependencies": {
+    "@mark1russell7/harness": "^0.9.0",
+    "@mark1russell7/quantity": "^0.2.0",
+    "@mark1russell7/service": "^0.4.0",
+    "testcontainers": "12.2.0"
+  },
+  "peerDependencies": {
+    "zod": "^4.1.0"
+  },
+  "devDependencies": {
+    "@mark1russell7/cue": "github:mark1russell7/cue",
+    "@types/node": "^26.6.4",
+    "tsdown": "^0.23.1",
+    "typescript": "^7.0.2",
+    "vitest": "^5.0.3",
+    "zod": "^4.6.6"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc -p tsconfig.json --noEmit --emitDeclarationOnly false && tsc -p tsconfig.test.json"
+  }
 }

package/src/backends.ts

@@ -0,0 +1,70 @@
+// The parser of the rules of kube-proxy. The agent of each node reads the rules with it, and the
+// image copies this file, thus it imports nothing.
+
+/** A service rule names the Service whose backends the agent reads: its namespace, its name, and its ClusterIP and port for IPVS. */
+export interface ServiceRule {
+  readonly mode: "iptables" | "ipvs" | "nftables";
+  readonly ns: string;
+  readonly svc: string;
+  readonly ip: string;
+  readonly port: number;
+}
+
+const hexIp = (hex: string): string =>
+  [0, 2, 4, 6].map((at) => parseInt(hex.slice(at, at + 2), 16)).join(".");
+
+/**
+ * This function gives the backends of a Service in a dump of the rules of kube-proxy, as
+ * `ip:port`, sorted. A backend of IPVS with the weight zero takes no new connection, thus it is
+ * not a backend.
+ * - iptables: the `KUBE-SVC` chain of the Service jumps to a `KUBE-SEP` chain for each backend, and that chain does the DNAT.
+ * - nftables: the `service-` chain of the Service does the DNAT to each backend, or it names an `endpoint-` chain for each one.
+ * - IPVS: `/proc/net/ip_vs` has the line of the virtual service, then a line for each real server.
+ */
+export const backendsOf = (mode: ServiceRule["mode"], text: string, rule: Pick<ServiceRule, "ns" | "svc" | "ip" | "port">): string[] => {
+  const out = new Set<string>();
+  const lines = text.split("\n");
+  if (mode === "iptables") {
+    const seps = new Set<string>();
+    for (const line of lines) {
+      if (!line.startsWith("-A KUBE-SVC-") || !line.includes(`${rule.ns}/${rule.svc}:`)) continue;
+      const jump = / -j (KUBE-SEP-\S+)/.exec(line);
+      if (jump !== null) seps.add(jump[1]!);
+    }
+    for (const line of lines) {
+      const chain = /^-A (KUBE-SEP-\S+) /.exec(line);
+      if (chain === null || !seps.has(chain[1]!)) continue;
+      const to = / --to-destination (\S+)/.exec(line);
+      if (to !== null) out.add(to[1]!);
+    }
+  } else if (mode === "nftables") {
+    // a chain name with a slash can print in quotes: `chain "service-…" {` and `goto "endpoint-…"`
+    let inside = false;
+    const ours = `-${rule.ns}/${rule.svc}/`;
+    for (const line of lines) {
+      const head = /^\s*chain\s+"?([^"\s{]+)"?\s*\{/.exec(line);
+      if (head !== null) inside = head[1]!.startsWith("service-") && head[1]!.includes(ours);
+      else if (/^\s*(set|map)\s/.test(line)) inside = false;
+      if (!inside) continue;
+      // one chain for each endpoint, as kube-proxy 1.31 writes it
+      for (const match of line.matchAll(/(?:goto|jump)\s+"?endpoint-[0-9A-Za-z]+(-[^\s,}"]+)__([0-9.]+)\/(\d+)"?/g))
+        if (match[1]!.startsWith(ours)) out.add(`${match[2]}:${match[3]}`);
+      // a DNAT in the service chain, as kube-proxy 1.37 writes it: `numgen random mod 2 map { 0 : IP . PORT, … }`, or `dnat to IP:PORT`
+      for (const match of line.matchAll(/(?:^|[\s{,:])(\d+\.\d+\.\d+\.\d+) \. (\d+)(?=\s*[,}]|\s*$)/g)) out.add(`${match[1]}:${match[2]}`);
+      for (const match of line.matchAll(/\bdnat (?:ip )?to (\d+\.\d+\.\d+\.\d+):(\d+)/g)) out.add(`${match[1]}:${match[2]}`);
+    }
+  } else {
+    let inside = false;
+    for (const line of lines) {
+      // each virtual service starts a block: a UDP or an SCTP one ends the block of the TCP service before it
+      const service = /^(TCP|UDP|SCTP)\s+([0-9A-F]{8}):([0-9A-F]{4})\s/.exec(line);
+      if (service !== null) {
+        inside = service[1] === "TCP" && hexIp(service[2]!) === rule.ip && parseInt(service[3]!, 16) === rule.port;
+        continue;
+      }
+      const real = /^\s+->\s+([0-9A-F]{8}):([0-9A-F]{4})\s+\S+\s+(\d+)/.exec(line);
+      if (inside && real !== null && Number(real[3]) > 0) out.add(`${hexIp(real[1]!)}:${parseInt(real[2]!, 16)}`);
+    }
+  }
+  return [...out].sort();
+};

package/src/compose.ts

@@ -0,0 +1,68 @@
+// The composition of the service of a kind cell. The app of the image and `lint()` compose the same
+// service from the same declaration: the image copies this file, and Node strips its types. Thus
+// this file imports only packages and types.
+import type { CheckFn, HealthServer, ServiceBuilder, TransportImpl } from "@mark1russell7/service";
+import { service } from "@mark1russell7/service";
+import type { RedisClient } from "@mark1russell7/service/redis";
+import type { Declaration } from "./cell.ts";
+
+/** These are the parts of a running service. A composition for `lint()` gives none of them. */
+export interface Parts {
+  readonly redis?: RedisClient | undefined;
+  readonly flag?: CheckFn | undefined;
+  readonly http?: TransportImpl | undefined;
+  readonly health?: HealthServer | undefined;
+}
+
+/** This name is the controller of the service, and its one route reads Redis when the service has Redis. */
+export const CONTROLLER = "work";
+export const ROUTE = "GET /work";
+/** This name is the check that a flag file fails: the seam of the liveness restart. */
+export const FLAG_CHECK = "self.flag";
+/** This name is the watchdog of the event loop. */
+export const WATCHDOG = "self.event-loop";
+
+/**
+ * This function composes the service of a declaration: its policies, its couplings, its checks,
+ * one server, one controller and each stage of the shutdown. A declaration with Redis needs the
+ * client, because the client declares the dependency and its checks.
+ */
+export const compose = (name: string, declaration: Declaration, parts: Parts = {}): ServiceBuilder => {
+  const svc = service(name)
+    .readiness(declaration.readiness)
+    .liveness(declaration.liveness)
+    .startup(declaration.startup)
+    .cooperativeWindow(declaration.cooperativeWindow);
+  for (const coupling of declaration.couplings) svc.couple(coupling);
+  if (declaration.redis !== null) {
+    if (parts.redis === undefined) throw new Error(`${name} declares Redis, thus its composition needs the client`);
+    parts.redis.attach(svc, { poll: declaration.poll, steps: { "round-trip": declaration.redis.gates } });
+  }
+  if (declaration.flag !== null)
+    svc.check(
+      {
+        name: FLAG_CHECK,
+        dependency: null,
+        step: "local-state",
+        ownConnection: true,
+        gates: declaration.flag,
+        poll: declaration.poll,
+      },
+      parts.flag,
+    );
+  if (declaration.watchdog !== null)
+    svc.watchdog({
+      name: WATCHDOG,
+      tick: declaration.watchdog.tick,
+      threshold: declaration.watchdog.threshold,
+      gates: "liveness",
+      poll: declaration.poll,
+    });
+  svc.server("http", {}, parts.http);
+  const controller = svc.controller(CONTROLLER);
+  if (declaration.redis !== null) controller.requires(parts.redis!.name);
+  controller.route(ROUTE, { idempotent: true });
+  if (parts.health !== undefined) svc.healthServer(parts.health);
+  for (const [stage, cap] of declaration.phases) svc.phase(stage, cap);
+  return svc;
+};