@mark1russell7/kind 0.11.12 → 0.12.0

package/build/judge.d.ts

@@ -1,2 +1,2 @@
-import { $ as unitOf, $t as secondsOf, A as TriggerV1, At as restartsAt, B as RunCheck, Bt as stoppingOf, C as KindCellResult, Cn as WindowPrediction, Ct as kernelCounts, D as PredictionV1, Dt as recordsOf, E as LoadSample, Et as podsOf, F as KIND_NOISE, Ft as sliceEmptyOf, G as factsOf, Gt as CLIENT, H as checkRun, Ht as tunnelsOf, I as KindDecision, It as sliceReadyAt, J as observationEnd, Jt as PORT, K as judgeKind, Kt as HEALTH_PORT, L as KindVerdict, Lt as sliceReadyOf, M as upgrade, Mt as signalOf, N as upgradeCell, Nt as signalsOf, O as RunCheckV1, On as KubeProxyMode, Ot as requestsOf, P as Deleted, Pt as sliceDropOf, Q as reusesReset, Qt as seconds, R as LAST_SLACK, Rt as slicesOf, S as KindCellRecordV1, Sn as TriggerKind, St as isTunnelState, T as KindVerdictV1, Tt as podsAt, U as cyclesOf, Ut as ztunnelPodsOf, V as RunFacts, Vt as transitionOf, W as deletionsOf, Wt as ztunnelReplaced, X as propagationSource, Xt as SERVER, Y as propagationOf, Yt as REDIS, Z as referenceOf, Zt as bandOf, _ as ClusterManifest, _n as Replica, _t as deletionOf, a as STAGES, an as Flap, at as PropagationSource, b as FORMAT_V1, bn as Template, bt as failuresOf, c as readRecord, cn as KubeletProbe, ct as RulesLine, d as calibrationOf, dn as MeshMode, dt as TCP_STATE, en as showBand, et as updatesOf, f as cellsTable, fn as NonePrediction, ft as TunnelLine, g as BandV1, gn as Reference, gt as byNode, h as timeSpread, hn as RedisDeclaration, ht as byClients, i as quantileOf, in as EdgeProtocol, it as PodLine, j as judge, jt as rulesOf, k as RunFactsV1, kt as restartOf, l as Judged, ln as Mesh, lt as SliceLine, m as predictionText, mn as Prediction, mt as backendsAt, n as medianInterval, nn as Cycle, nt as HBONE_PORT, o as kindCellRecordSchema, on as KindCell, ot as RecordLine, p as medianText, pn as PodSettings, pt as ZtunnelPodLine, q as observationAfter, qt as LOG_LABEL, r as middle, rn as Declaration, rt as Line, s as kindCellRecordV1Schema, sn as KindRun, st as Request, t as RELATIVE, tn as Band, tt as Endpoint, u as TimeSpread, un as MeshFlavor, ut as SocketLine, v as EnvironmentManifest, vn as ResetReuse, vt as dropOf, w as KindCellV1, wn as Workload, wt as outcomesOf, x as KindCellRecord, xn as Trigger, xt as intakeOf, y as FORMAT, yn as Stack, yt as emptyOf, z as Propagated, zt as socketsOf } from "./judge-BKe6dJ3O.js";
-export { type Band, type BandV1, CLIENT, type ClusterManifest, type Cycle, type Declaration, type Deleted, type EdgeProtocol, Endpoint, type EnvironmentManifest, FORMAT, FORMAT_V1, type Flap, HBONE_PORT, HEALTH_PORT, type Judged, KIND_NOISE, type KindCell, type KindCellRecord, type KindCellRecordV1, type KindCellResult, type KindCellV1, type KindDecision, type KindRun, type KindVerdict, type KindVerdictV1, type KubeProxyMode, type KubeletProbe, LAST_SLACK, LOG_LABEL, Line, type LoadSample, type Mesh, type MeshFlavor, type MeshMode, type NonePrediction, PORT, PodLine, type PodSettings, type Prediction, type PredictionV1, type Propagated, PropagationSource, REDIS, RELATIVE, RecordLine, type RedisDeclaration, type Reference, type Replica, Request, type ResetReuse, RulesLine, type RunCheck, type RunCheckV1, type RunFacts, type RunFactsV1, SERVER, STAGES, SliceLine, SocketLine, type Stack, TCP_STATE, type Template, type TimeSpread, type Trigger, type TriggerKind, type TriggerV1, TunnelLine, type WindowPrediction, type Workload, ZtunnelPodLine, backendsAt, bandOf, byClients, byNode, calibrationOf, cellsTable, checkRun, cyclesOf, deletionOf, deletionsOf, dropOf, emptyOf, factsOf, failuresOf, intakeOf, isTunnelState, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, medianInterval, medianText, middle, observationAfter, observationEnd, outcomesOf, podsAt, podsOf, predictionText, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, referenceOf, requestsOf, restartOf, restartsAt, reusesReset, rulesOf, seconds, secondsOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stoppingOf, timeSpread, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, ztunnelPodsOf, ztunnelReplaced };
+import { $ as unitOf, $t as secondsOf, A as TriggerV1, An as KubeProxyMode, At as restartsAt, B as RunCheck, Bt as stoppingOf, C as KindCellResult, Cn as WindowPrediction, Ct as kernelCounts, D as PredictionV1, Dt as recordsOf, E as LoadSample, En as Dataplane, Et as podsOf, F as KIND_NOISE, Ft as sliceEmptyOf, G as factsOf, Gt as CLIENT, H as checkRun, Ht as tunnelsOf, I as KindDecision, It as sliceReadyAt, J as observationEnd, Jt as PORT, K as judgeKind, Kt as HEALTH_PORT, L as KindVerdict, Lt as sliceReadyOf, M as upgrade, Mt as signalOf, N as upgradeCell, Nt as signalsOf, O as RunCheckV1, Ot as requestsOf, P as Deleted, Pt as sliceDropOf, Q as reusesReset, Qt as seconds, R as LAST_SLACK, Rt as slicesOf, S as KindCellRecordV1, Sn as TriggerKind, St as isTunnelState, T as KindVerdictV1, Tt as podsAt, U as cyclesOf, Ut as ztunnelPodsOf, V as RunFacts, Vt as transitionOf, W as deletionsOf, Wt as ztunnelReplaced, X as propagationSource, Xt as SERVER, Y as propagationOf, Yt as REDIS, Z as referenceOf, Zt as bandOf, _ as ClusterManifest, _n as Replica, _t as deletionOf, a as STAGES, an as Flap, at as PropagationSource, b as FORMAT_V1, bn as Template, bt as failuresOf, c as readRecord, cn as KubeletProbe, ct as RulesLine, d as calibrationOf, dn as MeshMode, dt as TCP_STATE, en as showBand, et as updatesOf, f as cellsTable, fn as NonePrediction, ft as TunnelLine, g as BandV1, gn as Reference, gt as byNode, h as timeSpread, hn as RedisDeclaration, ht as byClients, i as quantileOf, in as EdgeProtocol, it as PodLine, j as judge, jt as rulesOf, k as RunFactsV1, kt as restartOf, l as Judged, ln as Mesh, lt as SliceLine, m as predictionText, mn as Prediction, mt as backendsAt, n as medianInterval, nn as Cycle, nt as HBONE_PORT, o as kindCellRecordSchema, on as KindCell, ot as RecordLine, p as medianText, pn as PodSettings, pt as ZtunnelPodLine, q as observationAfter, qt as LOG_LABEL, r as middle, rn as Declaration, rt as Line, s as kindCellRecordV1Schema, sn as KindRun, st as Request, t as RELATIVE, tn as Band, tt as Endpoint, u as TimeSpread, un as MeshFlavor, ut as SocketLine, v as EnvironmentManifest, vn as ResetReuse, vt as dropOf, w as KindCellV1, wn as Workload, wt as outcomesOf, x as KindCellRecord, xn as Trigger, xt as intakeOf, y as FORMAT, yn as Stack, yt as emptyOf, z as Propagated, zt as socketsOf } from "./judge-BH4WtL3f.js";
+export { type Band, type BandV1, CLIENT, type ClusterManifest, type Cycle, type Dataplane, type Declaration, type Deleted, type EdgeProtocol, Endpoint, type EnvironmentManifest, FORMAT, FORMAT_V1, type Flap, HBONE_PORT, HEALTH_PORT, type Judged, KIND_NOISE, type KindCell, type KindCellRecord, type KindCellRecordV1, type KindCellResult, type KindCellV1, type KindDecision, type KindRun, type KindVerdict, type KindVerdictV1, type KubeProxyMode, type KubeletProbe, LAST_SLACK, LOG_LABEL, Line, type LoadSample, type Mesh, type MeshFlavor, type MeshMode, type NonePrediction, PORT, PodLine, type PodSettings, type Prediction, type PredictionV1, type Propagated, PropagationSource, REDIS, RELATIVE, RecordLine, type RedisDeclaration, type Reference, type Replica, Request, type ResetReuse, RulesLine, type RunCheck, type RunCheckV1, type RunFacts, type RunFactsV1, SERVER, STAGES, SliceLine, SocketLine, type Stack, TCP_STATE, type Template, type TimeSpread, type Trigger, type TriggerKind, type TriggerV1, TunnelLine, type WindowPrediction, type Workload, ZtunnelPodLine, backendsAt, bandOf, byClients, byNode, calibrationOf, cellsTable, checkRun, cyclesOf, deletionOf, deletionsOf, dropOf, emptyOf, factsOf, failuresOf, intakeOf, isTunnelState, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, medianInterval, medianText, middle, observationAfter, observationEnd, outcomesOf, podsAt, podsOf, predictionText, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, referenceOf, requestsOf, restartOf, restartsAt, reusesReset, rulesOf, seconds, secondsOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, stoppingOf, timeSpread, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, ztunnelPodsOf, ztunnelReplaced };

package/build/judge.js

@@ -1 +1 @@
-export { pt as CLIENT, u as FORMAT, d as FORMAT_V1, P as HBONE_PORT, mt as HEALTH_PORT, h as KIND_NOISE, g as LAST_SLACK, ht as LOG_LABEL, gt as PORT, _t as REDIS, A as RELATIVE, vt as SERVER, t as STAGES, F as TCP_STATE, I as backendsAt, yt as bandOf, L as byClients, R as byNode, a as calibrationOf, o as cellsTable, _ as checkRun, v as cyclesOf, z as deletionOf, y as deletionsOf, B as dropOf, V as emptyOf, b as factsOf, H as failuresOf, U as intakeOf, W as isTunnelState, f as judge, x as judgeKind, G as kernelCounts, n as kindCellRecordSchema, r as kindCellRecordV1Schema, j as medianInterval, s as medianText, M as middle, S as observationAfter, C as observationEnd, K as outcomesOf, q as podsAt, J as podsOf, c as predictionText, w as propagationOf, T as propagationSource, N as quantileOf, i as readRecord, Y as recordsOf, E as referenceOf, X as requestsOf, Z as restartOf, Q as restartsAt, D as reusesReset, $ as rulesOf, bt as seconds, xt as secondsOf, St as showBand, et as signalOf, tt as signalsOf, nt as sliceDropOf, rt as sliceEmptyOf, it as sliceReadyAt, at as sliceReadyOf, ot as slicesOf, st as socketsOf, ct as stoppingOf, l as timeSpread, lt as transitionOf, ut as tunnelsOf, O as unitOf, k as updatesOf, p as upgrade, m as upgradeCell, dt as ztunnelPodsOf, ft as ztunnelReplaced } from "./judge-n0rL_RR0.js";
+export { pt as CLIENT, u as FORMAT, d as FORMAT_V1, P as HBONE_PORT, mt as HEALTH_PORT, h as KIND_NOISE, g as LAST_SLACK, ht as LOG_LABEL, gt as PORT, _t as REDIS, A as RELATIVE, vt as SERVER, t as STAGES, F as TCP_STATE, I as backendsAt, yt as bandOf, L as byClients, R as byNode, a as calibrationOf, o as cellsTable, _ as checkRun, v as cyclesOf, z as deletionOf, y as deletionsOf, B as dropOf, V as emptyOf, b as factsOf, H as failuresOf, U as intakeOf, W as isTunnelState, f as judge, x as judgeKind, G as kernelCounts, n as kindCellRecordSchema, r as kindCellRecordV1Schema, j as medianInterval, s as medianText, M as middle, S as observationAfter, C as observationEnd, K as outcomesOf, q as podsAt, J as podsOf, c as predictionText, w as propagationOf, T as propagationSource, N as quantileOf, i as readRecord, Y as recordsOf, E as referenceOf, X as requestsOf, Z as restartOf, Q as restartsAt, D as reusesReset, $ as rulesOf, bt as seconds, xt as secondsOf, St as showBand, et as signalOf, tt as signalsOf, nt as sliceDropOf, rt as sliceEmptyOf, it as sliceReadyAt, at as sliceReadyOf, ot as slicesOf, st as socketsOf, ct as stoppingOf, l as timeSpread, lt as transitionOf, ut as tunnelsOf, O as unitOf, k as updatesOf, p as upgrade, m as upgradeCell, dt as ztunnelPodsOf, ft as ztunnelReplaced } from "./judge-CMQeiw8k.js";

package/image/agent.mjs

@@ -27,7 +27,7 @@
 // node and the freeze of its cgroup, or the thaw and the end of the drops), and now.
 import { spawn, spawnSync } from "node:child_process";
 import { existsSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
-import { backendsOf } from "./backends.ts";
+import { backendsOf, namesService } from "./backends.ts";
 import { entriesOf, flowsOf, staleOf, stalledOf } from "./conntrack.ts";
 import { devOf, grownOf, snmpOf, softnetOf, unresolvedOf } from "./drops.ts";
 import { clusterFilter, hostsOf } from "./eds.ts";
@@ -38,11 +38,16 @@ const node = process.env.NODE_NAME ?? "";
 const say = (line) => process.stdout.write(`${JSON.stringify({ ...line, node })}\n`);
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
-/** These commands give the backends of a Service in each mode of kube-proxy, as "ip:port" (with the weight for IPVS). */
+/**
+ * These commands give the backends of a Service in each dataplane, as "ip:port" (with the weight for
+ * IPVS). The agent of Cilium answers on its socket in the node (a hostPath of its chart), with the
+ * services of its load balancer as JSON (`GET /v1/service`).
+ */
 const DUMP = {
   iptables: ["nsenter", "-t", "1", "-m", "-n", "--", "iptables-save", "-t", "nat"],
   nftables: ["nsenter", "-t", "1", "-m", "-n", "--", "nft", "list", "table", "ip", "kube-proxy"],
   ipvs: ["nsenter", "-t", "1", "-n", "--", "cat", "/proc/net/ip_vs"],
+  cilium: ["nsenter", "-t", "1", "-m", "-n", "--", "curl", "-sS", "--fail", "--max-time", "2", "--unix-socket", "/var/run/cilium/cilium.sock", "http://localhost/v1/service"],
 };
 
 /**
@@ -185,14 +190,23 @@ const serve = async () => {
     const took = now() - t;
     polls += 1;
     spent += took;
-    if (run.status !== 0) {
-      say({ ev: "rules-fail", t, why: (run.stderr ?? "").trim().slice(0, 300) });
+    // a reply that the parser refuses (JSON that is not a list) is a failure, not a Service with no backend
+    let backends = null;
+    let refused = "";
+    if (run.status === 0) {
+      try {
+        backends = backendsOf(rule.mode, run.stdout, rule);
+      } catch (error) {
+        refused = `the dump does not parse: ${String(error)}: ${run.stdout.slice(0, 200)}`;
+      }
+    }
+    if (backends === null) {
+      say({ ev: "rules-fail", t, why: (refused || (run.stderr ?? "")).trim().slice(0, 300) });
     } else {
-      const backends = backendsOf(rule.mode, run.stdout, rule);
       const key = backends.join(",");
       // a dump that names the Service but gives no backend: once for each rule, its text, to read the format
       const config = JSON.stringify(rule);
-      if (backends.length === 0 && run.stdout.includes(`${rule.ns}/${rule.svc}`) && dumped !== config) {
+      if (backends.length === 0 && namesService(rule.mode, run.stdout, rule) && dumped !== config) {
         say({ ev: "rules-dump", t, text: run.stdout.slice(0, 30000) });
         dumped = config;
       }

package/package.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package",
   "name": "@mark1russell7/kind",
-  "version": "0.11.12",
+  "version": "0.12.0",
   "description": "Kubernetes in Docker as a lab for connection timers: a kind cluster from a scenario, faults in the namespaces of its pods, observations on the one monotonic clock of the VM, and predictions from the ordering law. Its judge entry reads and judges a raw record without Docker.",
   "keywords": [
     "kubernetes",

package/src/backends.ts

@@ -1,9 +1,9 @@
-// The parser of the rules of kube-proxy. The agent of each node reads the rules with it, and the
-// image copies this file, thus it imports nothing.
+// The parser of the rules of kube-proxy and of the services of Cilium. The agent of each node reads
+// the rules with it, and the image copies this file, thus it imports nothing.
 
-/** A service rule names the Service whose backends the agent reads: its namespace, its name, and its ClusterIP and port for IPVS. */
+/** A service rule names the Service whose backends the agent reads: its namespace, its name, and its ClusterIP and port for IPVS and Cilium. */
 export interface ServiceRule {
-  readonly mode: "iptables" | "ipvs" | "nftables";
+  readonly mode: "iptables" | "ipvs" | "nftables" | "cilium";
   readonly ns: string;
   readonly svc: string;
   readonly ip: string;
@@ -20,8 +20,10 @@ const hexIp = (hex: string): string =>
  * - iptables: the `KUBE-SVC` chain of the Service jumps to a `KUBE-SEP` chain for each backend, and that chain does the DNAT.
  * - nftables: the `service-` chain of the Service does the DNAT to each backend, or it names an `endpoint-` chain for each one.
  * - IPVS: `/proc/net/ip_vs` has the line of the virtual service, then a line for each real server.
+ * - Cilium: the JSON of the services of the agent (`ciliumBackendsOf`).
  */
 export const backendsOf = (mode: ServiceRule["mode"], text: string, rule: Pick<ServiceRule, "ns" | "svc" | "ip" | "port">): string[] => {
+  if (mode === "cilium") return ciliumBackendsOf(text, rule);
   const out = new Set<string>();
   const lines = text.split("\n");
   if (mode === "iptables") {
@@ -68,3 +70,56 @@ export const backendsOf = (mode: ServiceRule["mode"], text: string, rule: Pick<S
   }
   return [...out].sort();
 };
+
+/** This is one service of the agent of Cilium, as `GET /v1/service` gives it: the fields that the parser reads. */
+interface CiliumService {
+  readonly spec?: {
+    readonly "frontend-address"?: { readonly ip?: string; readonly port?: number; readonly protocol?: string; readonly scope?: string };
+    readonly "backend-addresses"?: ReadonlyArray<{ readonly ip?: string; readonly port?: number; readonly state?: string }> | null;
+    readonly flags?: { readonly type?: string; readonly name?: string; readonly namespace?: string };
+  };
+}
+
+/** This function gives the services in a reply of the agent of Cilium. It refuses a reply that is not a JSON list. */
+const ciliumServices = (text: string): CiliumService[] => {
+  const data: unknown = JSON.parse(text);
+  if (!Array.isArray(data)) throw new Error("the reply of the agent of Cilium is not a list of services");
+  return data as CiliumService[];
+};
+
+/**
+ * This function gives the backends of a Service in the reply of `GET /v1/service` of the agent of
+ * Cilium 1.20, as `ip:port`, sorted. The model is `ServiceSpec` of `api/v1/openapi.yaml`. The
+ * frontend is the ClusterIP and the port of the rule, with the scope `external` or no scope.
+ *
+ * The backends are those in the state `active`, or with no state. If no backend is active, the
+ * backends are those in the state `terminating`. The BPF reconciler gives the slots of a service
+ * in this sequence (`pkg/loadbalancer/reconciler/bpf_reconciler.go`). Thus a new connection goes
+ * only to them. kube-proxy does the same with the endpoints that serve and stop. The other
+ * states take no new connection: `terminating-not-serving`, `quarantined` and `maintenance`.
+ */
+export const ciliumBackendsOf = (text: string, rule: Pick<ServiceRule, "ip" | "port">): string[] => {
+  const ours = ciliumServices(text).filter((service) => {
+    const front = service.spec?.["frontend-address"];
+    const protocol = (front?.protocol ?? "tcp").toLowerCase();
+    return front?.ip === rule.ip && front.port === rule.port && (protocol === "tcp" || protocol === "any") && (front.scope ?? "external") === "external";
+  });
+  const backends = ours.flatMap((service) => service.spec?.["backend-addresses"] ?? []);
+  const of = (states: readonly (string | undefined)[]): string[] =>
+    [...new Set(backends.filter((b) => states.includes(b.state) && b.ip !== undefined).map((b) => `${b.ip}:${b.port ?? 0}`))].sort();
+  const active = of(["active", undefined]);
+  return active.length > 0 ? active : of(["terminating"]);
+};
+
+/**
+ * This function tells if a dump of the rules names the Service of a rule. The rules of kube-proxy
+ * name it as `namespace/name`. The JSON of Cilium names it in the flags of its frontend.
+ */
+export const namesService = (mode: ServiceRule["mode"], text: string, rule: Pick<ServiceRule, "ns" | "svc">): boolean => {
+  if (mode !== "cilium") return text.includes(`${rule.ns}/${rule.svc}`);
+  try {
+    return ciliumServices(text).some((service) => service.spec?.flags?.namespace === rule.ns && service.spec.flags.name === rule.svc);
+  } catch {
+    return false;
+  }
+};