@mark1russell7/kind 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -1
- package/build/index.d.ts +126 -5
- package/build/index.js +376 -36
- package/build/index.js.map +1 -1
- package/build/{judge-reWPMVr4.js → judge-CRMYoEOw.js} +90 -20
- package/build/judge-CRMYoEOw.js.map +1 -0
- package/build/{judge-Dwa6MqP4.d.ts → judge-DWF4QdZ8.d.ts} +74 -9
- package/build/judge.d.ts +2 -2
- package/build/judge.js +1 -1
- package/image/Dockerfile +1 -1
- package/image/agent.mjs +102 -2
- package/image/watch.mjs +50 -4
- package/package.json +3 -1
- package/src/eds.ts +40 -0
- package/build/judge-reWPMVr4.js.map +0 -1
|
@@ -77,8 +77,26 @@ declare const cidrRange: (cidr: string) => [number, number];
|
|
|
77
77
|
declare const cidrOverlap: (a: string, b: string) => boolean;
|
|
78
78
|
//#endregion
|
|
79
79
|
//#region src/cell.d.ts
|
|
80
|
-
/**
|
|
81
|
-
|
|
80
|
+
/**
|
|
81
|
+
* A template is the topology of a cell:
|
|
82
|
+
* - `T07` is the endpoint lifecycle of a Service.
|
|
83
|
+
* - `T15` is the same lifecycle through an Istio mesh: a mesh hop at the client and a mesh hop at the server.
|
|
84
|
+
* - `T18` is a health-gated fleet.
|
|
85
|
+
*/
|
|
86
|
+
type Template = "T07" | "T15" | "T18";
|
|
87
|
+
/**
|
|
88
|
+
* A mesh flavor is the install of Istio 1.31.3 in the cluster:
|
|
89
|
+
* - `sidecar`: the profile `minimal`, istiod only. The injector puts an Envoy sidecar in each pod of a labeled namespace.
|
|
90
|
+
* - `ambient`: the profile `ambient`, with istio-cni and ztunnel. The lab does not install it yet.
|
|
91
|
+
*/
|
|
92
|
+
type MeshFlavor = "sidecar" | "ambient";
|
|
93
|
+
/** A mesh mode is the data plane of the pods of a cell: an Envoy sidecar in each pod, or ztunnel on each node. */
|
|
94
|
+
type MeshMode = "sidecar" | "ambient";
|
|
95
|
+
/** A mesh is the install of Istio in the cluster of a cell, and the data plane of the pods of the cell. */
|
|
96
|
+
interface Mesh {
|
|
97
|
+
readonly flavor: MeshFlavor;
|
|
98
|
+
readonly mode: MeshMode;
|
|
99
|
+
}
|
|
82
100
|
/**
|
|
83
101
|
* A stack is the server and the clients of a cell:
|
|
84
102
|
* - `service`: the service of this repository with its declaration, and the HTTP client, which opens a
|
|
@@ -245,6 +263,16 @@ type Prediction<A = Quantity> = {
|
|
|
245
263
|
* rolling update deletes each old replica in turn. Without the flag, the window is that of the target.
|
|
246
264
|
*/
|
|
247
265
|
readonly each?: boolean | undefined;
|
|
266
|
+
/**
|
|
267
|
+
* This band is the propagation that the window assumes. It starts at the deletion of the replica
|
|
268
|
+
* of a window. It ends at the first poll of each node without that replica. That poll is of the
|
|
269
|
+
* rules, or of the client sidecar in a sidecar mesh (`propagationSource`).
|
|
270
|
+
*
|
|
271
|
+
* The judge holds each poll to `hi` ± δ, and it does not test a window without the band. A later
|
|
272
|
+
* poll agrees with the law, but the band is short, thus its cell is `extend` (D-EXTEND). An
|
|
273
|
+
* earlier drop only closes the window earlier.
|
|
274
|
+
*/
|
|
275
|
+
readonly propagation?: Band<A> | undefined;
|
|
248
276
|
readonly why: string;
|
|
249
277
|
} | {
|
|
250
278
|
readonly kind: "removal";
|
|
@@ -320,6 +348,8 @@ interface KindCell<A = Quantity> {
|
|
|
320
348
|
readonly template: Template;
|
|
321
349
|
/** This stack is the server and the clients of the cell. Without it, the cell has the service (`service`). */
|
|
322
350
|
readonly stack?: Stack | undefined;
|
|
351
|
+
/** This mesh is the Istio of the cell (T15). Without it, the cell has no mesh. */
|
|
352
|
+
readonly mesh?: Mesh | undefined;
|
|
323
353
|
/** This text names a system of the plan that the cell replaces with the service of this repository, or null. */
|
|
324
354
|
readonly substitutes: string | null;
|
|
325
355
|
readonly mode: KubeProxyMode;
|
|
@@ -428,13 +458,24 @@ interface SliceLine {
|
|
|
428
458
|
readonly svc: string | null;
|
|
429
459
|
readonly eps: readonly Endpoint[];
|
|
430
460
|
}
|
|
431
|
-
/**
|
|
461
|
+
/**
|
|
462
|
+
* A rules line is the backends of the Service in the rules of one node, after a change. `prev` is the
|
|
463
|
+
* start of the poll before it. An EDS line has the same fields (`ev` is `eds`). It gives the healthy
|
|
464
|
+
* hosts of the server cluster in the Envoy of the client sidecar on the node.
|
|
465
|
+
*/
|
|
432
466
|
interface RulesLine {
|
|
433
467
|
readonly t: number;
|
|
434
468
|
readonly prev: number | null;
|
|
435
469
|
readonly node: string;
|
|
436
470
|
readonly b: readonly string[];
|
|
437
471
|
}
|
|
472
|
+
/**
|
|
473
|
+
* A propagation source is the poll whose lines give the backends of each node:
|
|
474
|
+
* - `rules`: the rules of kube-proxy in the namespaces of the node.
|
|
475
|
+
* - `eds`: the hosts of the server cluster in the Envoy of the client sidecar on the node. In a sidecar
|
|
476
|
+
* mesh, the client sidecar picks the backend of each call, thus kube-proxy is not on the path.
|
|
477
|
+
*/
|
|
478
|
+
type PropagationSource = "rules" | "eds";
|
|
438
479
|
/** A record line is one transition of a machine of the service, with its own stamp `at`. */
|
|
439
480
|
interface RecordLine {
|
|
440
481
|
readonly t: number;
|
|
@@ -450,7 +491,7 @@ interface RecordLine {
|
|
|
450
491
|
declare const requestsOf: (lines: readonly Line[]) => Request[];
|
|
451
492
|
declare const podsOf: (lines: readonly Line[]) => PodLine[];
|
|
452
493
|
declare const slicesOf: (lines: readonly Line[], svc: string) => SliceLine[];
|
|
453
|
-
declare const rulesOf: (lines: readonly Line[]) => RulesLine[];
|
|
494
|
+
declare const rulesOf: (lines: readonly Line[], source?: PropagationSource) => RulesLine[];
|
|
454
495
|
declare const recordsOf: (lines: readonly Line[]) => RecordLine[];
|
|
455
496
|
/** This function gives the requests that did not succeed, in sequence of their start. */
|
|
456
497
|
declare const failuresOf: (lines: readonly Line[]) => Request[];
|
|
@@ -486,8 +527,9 @@ declare const signalOf: (lines: readonly Line[], pod: string, after: number) =>
|
|
|
486
527
|
/**
|
|
487
528
|
* This function gives when the rules of a node stopped the selection of a backend: the first poll
|
|
488
529
|
* after a time without it. The change came after `prev`, the poll before it, and at the latest at `t`.
|
|
530
|
+
* The source `eds` gives the same for the client sidecar of the node.
|
|
489
531
|
*/
|
|
490
|
-
declare const dropOf: (lines: readonly Line[], node: string, backend: string, after: number) => RulesLine | null;
|
|
532
|
+
declare const dropOf: (lines: readonly Line[], node: string, backend: string, after: number, source?: PropagationSource) => RulesLine | null;
|
|
491
533
|
/** This function gives when the rules of a node had no backend of the Service, after a time. */
|
|
492
534
|
declare const emptyOf: (lines: readonly Line[], node: string, after: number) => RulesLine | null;
|
|
493
535
|
/** This function gives when an EndpointSlice of the Service first had no ready endpoint, after a time. */
|
|
@@ -610,18 +652,41 @@ declare const reusesReset: (request: Request, requests: readonly Request[], reus
|
|
|
610
652
|
* This function gives a reference event of a run, as a stamp, or null when the run does not have it.
|
|
611
653
|
* The references `deletion` and `intake` are those of the target, or of the replica `pod`.
|
|
612
654
|
*/
|
|
613
|
-
declare const referenceOf: (run: KindRun, from: Reference, pod?: string | undefined) => number | null;
|
|
655
|
+
declare const referenceOf: (run: Pick<KindRun, "start" | "acts" | "target" | "lines">, from: Reference, pod?: string | undefined) => number | null;
|
|
656
|
+
/**
|
|
657
|
+
* This function gives the source of the propagation of a cell. In a sidecar mesh, the client sidecar
|
|
658
|
+
* picks the backend of each call from its endpoints (EDS), thus the source is `eds`. Each other cell
|
|
659
|
+
* goes through the rules of kube-proxy (`rules`).
|
|
660
|
+
*/
|
|
661
|
+
declare const propagationSource: (cell: Pick<KindCell, "mesh">) => PropagationSource;
|
|
614
662
|
/**
|
|
615
663
|
* This function gives the propagation of each node for the target of a run, or for the replica of
|
|
616
664
|
* the address `ip`. It is the time from its deletion (or its crash) to the first poll of the rules
|
|
617
665
|
* of the node without it. The change came between the poll before (`after`) and that poll (`by`).
|
|
666
|
+
* The source `eds` gives the poll of the client sidecar of each node.
|
|
618
667
|
*/
|
|
619
|
-
declare const propagationOf: (run: KindRun, from: number | null, ip?: string | undefined) => Record<string, Propagated | null>;
|
|
668
|
+
declare const propagationOf: (run: KindRun, from: number | null, ip?: string | undefined, source?: PropagationSource) => Record<string, Propagated | null>;
|
|
620
669
|
/**
|
|
621
670
|
* This function gives each replica of a run that got a deletion after the start of the run, in the
|
|
622
671
|
* sequence of the deletions. A rolling update deletes each old replica in turn.
|
|
623
672
|
*/
|
|
624
673
|
declare const deletionsOf: (run: KindRun) => Deleted[];
|
|
674
|
+
/**
|
|
675
|
+
* This function gives how long a run observes after the end reference of its last window, or null
|
|
676
|
+
* when its prediction sets no such time. The judge closes a window `end.band.hi` + δ after its end
|
|
677
|
+
* reference. A request that starts by then ends at most the timeout of the workload later: the
|
|
678
|
+
* client cancels it then. A prediction that is not a window, or a window with no end, sets none.
|
|
679
|
+
*/
|
|
680
|
+
declare const observationAfter: (cell: Pick<KindCell, "prediction" | "workload">) => Quantity | null;
|
|
681
|
+
/**
|
|
682
|
+
* This function gives the earliest end of the observation of a run, as a stamp, or null when its
|
|
683
|
+
* prediction sets none (`observationAfter`). It is that time after the latest end reference of a
|
|
684
|
+
* window: of the target, or with `each`, of each replica that got a deletion. A run that ends before
|
|
685
|
+
* it can miss its own failures. In a rolling update of gRPC, a call waits in the queue of its channel
|
|
686
|
+
* for a stalled connect. It fails only at its timeout (kind run 40128267098). A reference that is not
|
|
687
|
+
* in the lines yet does not count.
|
|
688
|
+
*/
|
|
689
|
+
declare const observationEnd: (cell: KindCell, run: Pick<KindRun, "start" | "acts" | "target" | "replicas" | "lines">) => number | null;
|
|
625
690
|
/** This function gives the facts of a run that each table reads: its failures by outcome and node, and the times of its target. */
|
|
626
691
|
declare const factsOf: (cell: KindCell, run: KindRun) => RunFacts;
|
|
627
692
|
/** This function judges one run against the prediction of its cell. */
|
|
@@ -784,5 +849,5 @@ declare const quantileOf: (sorted: readonly number[], p: number) => number;
|
|
|
784
849
|
*/
|
|
785
850
|
declare const medianInterval: (sorted: readonly number[]) => [number, number];
|
|
786
851
|
//#endregion
|
|
787
|
-
export {
|
|
788
|
-
//# sourceMappingURL=judge-
|
|
852
|
+
export { Endpoint as $, ResetReuse as $t, TriggerV1 as A, CLIENT as At, RunCheck as B, Band as Bt, KindCellResult as C, signalOf as Ct, PredictionV1 as D, slicesOf as Dt, LoadSample as E, sliceEmptyOf as Et, KIND_NOISE as F, SERVER as Ft, judgeKind as G, Mesh as Gt, checkRun as H, KindCell as Ht, KindDecision as I, bandOf as It, propagationOf as J, PodSettings as Jt, observationAfter as K, MeshFlavor as Kt, KindVerdict as L, seconds as Lt, upgrade as M, LOG_LABEL as Mt, upgradeCell as N, PORT as Nt, RunCheckV1 as O, stoppingOf as Ot, Deleted as P, REDIS as Pt, unitOf as Q, Replica as Qt, LAST_SLACK as R, secondsOf as Rt, KindCellRecordV1 as S, rulesOf as St, KindVerdictV1 as T, sliceDropOf as Tt, deletionsOf as U, KindRun as Ut, RunFacts as V, Declaration as Vt, factsOf as W, KubeletProbe as Wt, referenceOf as X, RedisDeclaration as Xt, propagationSource as Y, Prediction as Yt, reusesReset as Z, Reference as Zt, ClusterManifest as _, scenario as _n, podsAt as _t, STAGES as a, KUBE_PROXY_MODES as an, RulesLine as at, FORMAT_V1 as b, requestsOf as bt, readRecord as c, KubeProxyMode as cn, byClients as ct, calibrationOf as d, cidrOverlap as dn, dropOf as dt, Stack as en, Line as et, cellsTable as f, cidrRange as fn, emptyOf as ft, BandV1 as g, nodeNames as gn, outcomesOf as gt, timeSpread as h, kubeProxyPatch as hn, kernelCounts as ht, quantileOf as i, Workload as in, Request as it, judge as j, HEALTH_PORT as jt, RunFactsV1 as k, transitionOf as kt, Judged as l, NODE_IMAGES as ln, byNode as lt, predictionText as m, kindConfig as mn, intakeOf as mt, medianInterval as n, Trigger as nn, PropagationSource as nt, kindCellRecordSchema as o, KindConfig as on, SliceLine as ot, medianText as p, goDuration as pn, failuresOf as pt, observationEnd as q, MeshMode as qt, middle as r, TriggerKind as rn, RecordLine as rt, kindCellRecordV1Schema as s, KindScenario as sn, backendsAt as st, RELATIVE as t, Template as tn, PodLine as tt, TimeSpread as u, NodeImage as un, deletionOf as ut, EnvironmentManifest as v, scenarioProblems as vn, podsOf as vt, KindCellV1 as w, signalsOf as wt, KindCellRecord as x, restartsAt as xt, FORMAT as y, recordsOf as yt, Propagated as z, showBand as zt };
|
|
853
|
+
//# sourceMappingURL=judge-DWF4QdZ8.d.ts.map
|
package/build/judge.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { $ as
|
|
2
|
-
export { type Band, type BandV1, CLIENT, type ClusterManifest, type Declaration, type Deleted, Endpoint, type EnvironmentManifest, FORMAT, FORMAT_V1, HEALTH_PORT, type Judged, KIND_NOISE, type KindCell, type KindCellRecord, type KindCellRecordV1, type KindCellResult, type KindCellV1, type KindDecision, type KindRun, type KindVerdict, type KindVerdictV1, type KubeProxyMode, type KubeletProbe, LAST_SLACK, LOG_LABEL, Line, type LoadSample, PORT, PodLine, type PodSettings, type Prediction, type PredictionV1, type Propagated, REDIS, RELATIVE, RecordLine, type RedisDeclaration, type Reference, type Replica, Request, type ResetReuse, RulesLine, type RunCheck, type RunCheckV1, type RunFacts, type RunFactsV1, SERVER, STAGES, SliceLine, type Stack, type Template, type TimeSpread, type Trigger, type TriggerKind, type TriggerV1, type Workload, backendsAt, bandOf, byClients, byNode, calibrationOf, cellsTable, checkRun, deletionOf, deletionsOf, dropOf, emptyOf, factsOf, failuresOf, intakeOf, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, medianInterval, medianText, middle, outcomesOf, podsAt, podsOf, predictionText, propagationOf, quantileOf, readRecord, recordsOf, referenceOf, requestsOf, restartsAt, reusesReset, rulesOf, seconds, secondsOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, stoppingOf, timeSpread, transitionOf, unitOf, upgrade, upgradeCell };
|
|
1
|
+
import { $ as Endpoint, $t as ResetReuse, A as TriggerV1, At as CLIENT, B as RunCheck, Bt as Band, C as KindCellResult, Ct as signalOf, D as PredictionV1, Dt as slicesOf, E as LoadSample, Et as sliceEmptyOf, F as KIND_NOISE, Ft as SERVER, G as judgeKind, Gt as Mesh, H as checkRun, Ht as KindCell, I as KindDecision, It as bandOf, J as propagationOf, Jt as PodSettings, K as observationAfter, Kt as MeshFlavor, L as KindVerdict, Lt as seconds, M as upgrade, Mt as LOG_LABEL, N as upgradeCell, Nt as PORT, O as RunCheckV1, Ot as stoppingOf, P as Deleted, Pt as REDIS, Q as unitOf, Qt as Replica, R as LAST_SLACK, Rt as secondsOf, S as KindCellRecordV1, St as rulesOf, T as KindVerdictV1, Tt as sliceDropOf, U as deletionsOf, Ut as KindRun, V as RunFacts, Vt as Declaration, W as factsOf, Wt as KubeletProbe, X as referenceOf, Xt as RedisDeclaration, Y as propagationSource, Yt as Prediction, Z as reusesReset, Zt as Reference, _ as ClusterManifest, _t as podsAt, a as STAGES, at as RulesLine, b as FORMAT_V1, bt as requestsOf, c as readRecord, cn as KubeProxyMode, ct as byClients, d as calibrationOf, dt as dropOf, en as Stack, et as Line, f as cellsTable, ft as emptyOf, g as BandV1, gt as outcomesOf, h as timeSpread, ht as kernelCounts, i as quantileOf, in as Workload, it as Request, j as judge, jt as HEALTH_PORT, k as RunFactsV1, kt as transitionOf, l as Judged, lt as byNode, m as predictionText, mt as intakeOf, n as medianInterval, nn as Trigger, nt as PropagationSource, o as kindCellRecordSchema, ot as SliceLine, p as medianText, pt as failuresOf, q as observationEnd, qt as MeshMode, r as middle, rn as TriggerKind, rt as RecordLine, s as kindCellRecordV1Schema, st as backendsAt, t as RELATIVE, tn as Template, tt as PodLine, u as TimeSpread, ut as deletionOf, v as EnvironmentManifest, vt as podsOf, w as KindCellV1, wt as signalsOf, x as KindCellRecord, xt as restartsAt, y as FORMAT, yt as recordsOf, z as Propagated, zt as showBand } from "./judge-DWF4QdZ8.js";
|
|
2
|
+
export { type Band, type BandV1, CLIENT, type ClusterManifest, type Declaration, type Deleted, Endpoint, type EnvironmentManifest, FORMAT, FORMAT_V1, HEALTH_PORT, type Judged, KIND_NOISE, type KindCell, type KindCellRecord, type KindCellRecordV1, type KindCellResult, type KindCellV1, type KindDecision, type KindRun, type KindVerdict, type KindVerdictV1, type KubeProxyMode, type KubeletProbe, LAST_SLACK, LOG_LABEL, Line, type LoadSample, type Mesh, type MeshFlavor, type MeshMode, PORT, PodLine, type PodSettings, type Prediction, type PredictionV1, type Propagated, PropagationSource, REDIS, RELATIVE, RecordLine, type RedisDeclaration, type Reference, type Replica, Request, type ResetReuse, RulesLine, type RunCheck, type RunCheckV1, type RunFacts, type RunFactsV1, SERVER, STAGES, SliceLine, type Stack, type Template, type TimeSpread, type Trigger, type TriggerKind, type TriggerV1, type Workload, backendsAt, bandOf, byClients, byNode, calibrationOf, cellsTable, checkRun, deletionOf, deletionsOf, dropOf, emptyOf, factsOf, failuresOf, intakeOf, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, medianInterval, medianText, middle, observationAfter, observationEnd, outcomesOf, podsAt, podsOf, predictionText, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, referenceOf, requestsOf, restartsAt, reusesReset, rulesOf, seconds, secondsOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, stoppingOf, timeSpread, transitionOf, unitOf, upgrade, upgradeCell };
|
package/build/judge.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export { tt as CLIENT, u as FORMAT, d as FORMAT_V1, nt as HEALTH_PORT, h as KIND_NOISE, g as LAST_SLACK, rt as LOG_LABEL, it as PORT, at as REDIS, O as RELATIVE, ot as SERVER, t as STAGES, M as backendsAt, st as bandOf, N as byClients, P as byNode, a as calibrationOf, o as cellsTable, _ as checkRun, F as deletionOf, v as deletionsOf, I as dropOf, L as emptyOf, y as factsOf, R as failuresOf, z as intakeOf, f as judge, b as judgeKind, B as kernelCounts, n as kindCellRecordSchema, r as kindCellRecordV1Schema, k as medianInterval, s as medianText, A as middle, x as observationAfter, S as observationEnd, V as outcomesOf, H as podsAt, U as podsOf, c as predictionText, C as propagationOf, w as propagationSource, j as quantileOf, i as readRecord, W as recordsOf, T as referenceOf, G as requestsOf, K as restartsAt, E as reusesReset, q as rulesOf, ct as seconds, lt as secondsOf, ut as showBand, J as signalOf, Y as signalsOf, X as sliceDropOf, Z as sliceEmptyOf, Q as slicesOf, $ as stoppingOf, l as timeSpread, et as transitionOf, D as unitOf, p as upgrade, m as upgradeCell } from "./judge-CRMYoEOw.js";
|
package/image/Dockerfile
CHANGED
|
@@ -14,5 +14,5 @@ RUN for name in quantity algebra service; do \
|
|
|
14
14
|
&& tar -xzf /tmp/workspace/mark1russell7-$name-*.tgz -C "node_modules/@mark1russell7/$name" --strip-components=1; \
|
|
15
15
|
done \
|
|
16
16
|
&& rm -rf /tmp/workspace
|
|
17
|
-
COPY app.mjs client.mjs client-grpc.mjs agent.mjs watch.mjs compose.ts backends.ts conntrack.ts drops.ts ./
|
|
17
|
+
COPY app.mjs client.mjs client-grpc.mjs agent.mjs watch.mjs compose.ts backends.ts conntrack.ts drops.ts eds.ts ./
|
|
18
18
|
CMD ["node", "app.mjs"]
|
package/image/agent.mjs
CHANGED
|
@@ -9,17 +9,23 @@
|
|
|
9
9
|
// handshake that stopped for a second (`stalled`). Each line
|
|
10
10
|
// `rules-alive` also counts the reads of the table (`ct`). Each
|
|
11
11
|
// second it reads the counters of dropped packets of the node and
|
|
12
|
-
// of each client pod, and prints each one that grew (`drops`)
|
|
12
|
+
// of each client pod, and prints each one that grew (`drops`).
|
|
13
|
+
// As /tmp/eds.json asks, it starts the EDS poller of each client pod
|
|
14
|
+
// of the node, and stops it when the file goes
|
|
15
|
+
// node agent.mjs eds JSON the EDS poller: in the network namespace of a client pod (nsenter -n),
|
|
16
|
+
// it reads /clusters of the Envoy of the sidecar each `every` ms, and
|
|
17
|
+
// prints the hosts of the server cluster at each change (`eds`)
|
|
13
18
|
// node agent.mjs act JSON does one act and prints its stamps on CLOCK_MONOTONIC
|
|
14
19
|
//
|
|
15
20
|
// The acts: signal (a signal to a PID), pause (SIGSTOP, then SIGCONT after ms), write (a file, for
|
|
16
21
|
// example cpu.max or a flag file in the root of a container), remove (a file), stall (the stall
|
|
17
22
|
// seam of the app: the ms in /tmp/lab-stall, then SIGUSR2), and now.
|
|
18
|
-
import { spawnSync } from "node:child_process";
|
|
23
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
19
24
|
import { existsSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
20
25
|
import { backendsOf } from "./backends.ts";
|
|
21
26
|
import { entriesOf, flowsOf, staleOf, stalledOf } from "./conntrack.ts";
|
|
22
27
|
import { devOf, grownOf, snmpOf, softnetOf, unresolvedOf } from "./drops.ts";
|
|
28
|
+
import { clusterFilter, hostsOf } from "./eds.ts";
|
|
23
29
|
|
|
24
30
|
const now = () => Number(process.hrtime.bigint()) / 1e6;
|
|
25
31
|
const node = process.env.NODE_NAME ?? "";
|
|
@@ -254,6 +260,97 @@ const serve = async () => {
|
|
|
254
260
|
}
|
|
255
261
|
};
|
|
256
262
|
|
|
263
|
+
/**
|
|
264
|
+
* the EDS pollers of the node: /tmp/eds.json gives the server cluster, the interval and each client
|
|
265
|
+
* pod of the node with a PID in it. The agent starts one poller for each pod in the network
|
|
266
|
+
* namespace of that PID, and stops each one when the file changes or goes. nsenter -n keeps the
|
|
267
|
+
* mount namespace of the agent, thus the poller is this file, and its clock is the one of the VM
|
|
268
|
+
*/
|
|
269
|
+
const pollers = async () => {
|
|
270
|
+
let config = null;
|
|
271
|
+
let children = [];
|
|
272
|
+
const stop = () => {
|
|
273
|
+
for (const child of children) child.kill("SIGKILL");
|
|
274
|
+
children = [];
|
|
275
|
+
};
|
|
276
|
+
for (;;) {
|
|
277
|
+
let text = null;
|
|
278
|
+
let spec = null;
|
|
279
|
+
try {
|
|
280
|
+
text = existsSync("/tmp/eds.json") ? readFileSync("/tmp/eds.json", "utf8") : null;
|
|
281
|
+
spec = text === null ? null : JSON.parse(text);
|
|
282
|
+
} catch (error) {
|
|
283
|
+
say({ ev: "eds-fail", t: now(), why: String(error?.message ?? error).slice(0, 300) });
|
|
284
|
+
text = config;
|
|
285
|
+
}
|
|
286
|
+
if (text !== config) {
|
|
287
|
+
stop();
|
|
288
|
+
config = text;
|
|
289
|
+
if (spec !== null) {
|
|
290
|
+
for (const { pod, pid } of spec.pods) {
|
|
291
|
+
const argv = ["-t", String(pid), "-n", "--", process.execPath, process.argv[1], "eds", JSON.stringify({ cluster: spec.cluster, every: spec.every, pod })];
|
|
292
|
+
const child = spawn("nsenter", argv, { stdio: ["ignore", "inherit", "inherit"] });
|
|
293
|
+
child.on("exit", (code, signal) => {
|
|
294
|
+
if (signal !== "SIGKILL") say({ ev: "eds-exit", t: now(), pod, code, signal });
|
|
295
|
+
});
|
|
296
|
+
child.on("error", (error) => say({ ev: "eds-fail", t: now(), pod, why: String(error?.message ?? error).slice(0, 300) }));
|
|
297
|
+
children.push(child);
|
|
298
|
+
}
|
|
299
|
+
say({ ev: "eds-start", t: now(), cluster: spec.cluster, every: spec.every, pods: spec.pods.map((each) => each.pod) });
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
await sleep(250);
|
|
303
|
+
}
|
|
304
|
+
};
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* one EDS poller. The admin endpoint of the sidecar is 127.0.0.1:15000 in the namespace of the pod
|
|
308
|
+
* (envoy_bootstrap.json of Istio 1.31.3). The filter of /clusters keeps the server cluster only
|
|
309
|
+
* (clusters_handler.cc of Envoy v1.39.4). A line comes at each change of its hosts, with the start of
|
|
310
|
+
* the poll before it (`prev`), as a rules line. Each second, a line `eds-alive` counts the polls
|
|
311
|
+
*/
|
|
312
|
+
const eds = async ({ cluster, every, pod }) => {
|
|
313
|
+
const url = `http://127.0.0.1:15000/clusters?filter=${encodeURIComponent(clusterFilter(cluster))}`;
|
|
314
|
+
let last = null;
|
|
315
|
+
let previous = null;
|
|
316
|
+
let failure = null;
|
|
317
|
+
let polls = 0;
|
|
318
|
+
let spent = 0;
|
|
319
|
+
let beat = now();
|
|
320
|
+
for (;;) {
|
|
321
|
+
const t = now();
|
|
322
|
+
try {
|
|
323
|
+
const response = await fetch(url, { signal: AbortSignal.timeout(1000) });
|
|
324
|
+
const text = await response.text();
|
|
325
|
+
const took = now() - t;
|
|
326
|
+
polls += 1;
|
|
327
|
+
spent += took;
|
|
328
|
+
if (!response.ok) throw new Error(`the admin endpoint answered ${response.status}: ${text.slice(0, 200)}`);
|
|
329
|
+
const { b, u } = hostsOf(text, cluster);
|
|
330
|
+
const key = JSON.stringify([b, u]);
|
|
331
|
+
if (key !== last) {
|
|
332
|
+
say({ ev: "eds", t, prev: previous, pod, took: Math.round(took * 1000) / 1000, b, ...(Object.keys(u).length === 0 ? {} : { u }) });
|
|
333
|
+
last = key;
|
|
334
|
+
}
|
|
335
|
+
failure = null;
|
|
336
|
+
} catch (error) {
|
|
337
|
+
// each failure once, until a poll succeeds
|
|
338
|
+
const why = String(error?.message ?? error).slice(0, 300);
|
|
339
|
+
if (why !== failure) say({ ev: "eds-fail", t, pod, why });
|
|
340
|
+
failure = why;
|
|
341
|
+
}
|
|
342
|
+
previous = t;
|
|
343
|
+
if (t - beat >= 1000) {
|
|
344
|
+
say({ ev: "eds-alive", t, pod, polls, ms: polls === 0 ? 0 : Math.round((spent / polls) * 100) / 100 });
|
|
345
|
+
beat = t;
|
|
346
|
+
polls = 0;
|
|
347
|
+
spent = 0;
|
|
348
|
+
}
|
|
349
|
+
const wait = every - (now() - t);
|
|
350
|
+
if (wait > 0) await sleep(wait);
|
|
351
|
+
}
|
|
352
|
+
};
|
|
353
|
+
|
|
257
354
|
const act = async (spec) => {
|
|
258
355
|
const t0 = now();
|
|
259
356
|
if (spec.op === "now") return { t0, t1: t0 };
|
|
@@ -292,7 +389,10 @@ const act = async (spec) => {
|
|
|
292
389
|
const [mode, arg] = process.argv.slice(2);
|
|
293
390
|
if (mode === "serve") {
|
|
294
391
|
say({ ev: "agent", what: "start", t: now() });
|
|
392
|
+
void pollers();
|
|
295
393
|
await serve();
|
|
394
|
+
} else if (mode === "eds") {
|
|
395
|
+
await eds(JSON.parse(arg));
|
|
296
396
|
} else if (mode === "act") {
|
|
297
397
|
const spec = JSON.parse(arg);
|
|
298
398
|
try {
|
package/image/watch.mjs
CHANGED
|
@@ -3,6 +3,13 @@
|
|
|
3
3
|
// EndpointSlices and the events of one namespace through the API, and it follows the log of each
|
|
4
4
|
// pod with the label lab.systems/log=1. A log line is JSON with its own stamp, and the watcher
|
|
5
5
|
// passes it on as it is.
|
|
6
|
+
//
|
|
7
|
+
// The sidecar of Istio (the container istio-proxy) is a native sidecar: an init container that
|
|
8
|
+
// runs beside the app. The watcher follows it too. Its lines are the access log of Envoy (JSON, a
|
|
9
|
+
// stamp of the wall clock) and the text of pilot-agent. Each one becomes a line `proxy` with the
|
|
10
|
+
// stamp of its arrival: evidence, not a time that a band reads. An access line of a call that
|
|
11
|
+
// succeeded at its first attempt is only counted (`proxy-ok`, each second), thus the record keeps
|
|
12
|
+
// the failures, the retries and the local replies.
|
|
6
13
|
import https from "node:https";
|
|
7
14
|
|
|
8
15
|
const kube = JSON.parse(process.env.KUBE ?? "{}");
|
|
@@ -75,6 +82,38 @@ const stateOf = (state) => {
|
|
|
75
82
|
return "none";
|
|
76
83
|
};
|
|
77
84
|
|
|
85
|
+
/** the container of the sidecar of Istio */
|
|
86
|
+
const PROXY = "istio-proxy";
|
|
87
|
+
/** the access lines of each sidecar that the watcher only counted, by pod, since the last count */
|
|
88
|
+
const counted = new Map();
|
|
89
|
+
setInterval(() => {
|
|
90
|
+
const t = now();
|
|
91
|
+
for (const [pod, n] of counted) say({ ev: "proxy-ok", t, p: pod, n });
|
|
92
|
+
counted.clear();
|
|
93
|
+
}, 1000);
|
|
94
|
+
|
|
95
|
+
/** an access line of Envoy that tells nothing more than its call: status 200, no flag, gRPC status 0, one attempt */
|
|
96
|
+
const plain = (log) =>
|
|
97
|
+
String(log.code) === "200" && (log.flags === "-" || log.flags === "" || log.flags === undefined) && String(log.grpc) === "0" && Number(log.attempts ?? 1) <= 1;
|
|
98
|
+
|
|
99
|
+
/** a line of the sidecar: an access line of Envoy as JSON, or a text line of pilot-agent */
|
|
100
|
+
const proxyLine = (pod, text) => {
|
|
101
|
+
if (text.startsWith("{")) {
|
|
102
|
+
try {
|
|
103
|
+
const log = JSON.parse(text);
|
|
104
|
+
if (plain(log)) {
|
|
105
|
+
counted.set(pod, (counted.get(pod) ?? 0) + 1);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
say({ ev: "proxy", t: now(), p: pod, log });
|
|
109
|
+
return;
|
|
110
|
+
} catch {
|
|
111
|
+
// a line that is not whole JSON is text
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
say({ ev: "proxy", t: now(), p: pod, text: text.slice(0, 500) });
|
|
115
|
+
};
|
|
116
|
+
|
|
78
117
|
const following = new Set();
|
|
79
118
|
const follow = (ns, pod, container, id) => {
|
|
80
119
|
const key = `${ns}/${pod}/${container}/${id}`;
|
|
@@ -84,7 +123,8 @@ const follow = (ns, pod, container, id) => {
|
|
|
84
123
|
`/api/v1/namespaces/${ns}/pods/${pod}/log`,
|
|
85
124
|
{ container, follow: "true" },
|
|
86
125
|
(text) => {
|
|
87
|
-
if (
|
|
126
|
+
if (container === PROXY) proxyLine(pod, text);
|
|
127
|
+
else if (text.startsWith("{")) pending.push(`${text}\n`);
|
|
88
128
|
else say({ ev: "text", t: now(), p: pod, text: text.slice(0, 500) });
|
|
89
129
|
},
|
|
90
130
|
false,
|
|
@@ -109,6 +149,10 @@ for (const ns of namespaces) {
|
|
|
109
149
|
last: stateOf(status.lastState),
|
|
110
150
|
id: status.containerID ?? null,
|
|
111
151
|
}));
|
|
152
|
+
// the native sidecar of Istio is a running init container: its state, apart from the containers
|
|
153
|
+
const sidecars = (pod.status?.initContainerStatuses ?? [])
|
|
154
|
+
.filter((status) => status.name === PROXY)
|
|
155
|
+
.map((status) => ({ n: status.name, r: status.ready, rc: status.restartCount, st: stateOf(status.state), id: status.containerID ?? null }));
|
|
112
156
|
say({
|
|
113
157
|
ev: "pod",
|
|
114
158
|
t,
|
|
@@ -121,11 +165,13 @@ for (const ns of namespaces) {
|
|
|
121
165
|
ready,
|
|
122
166
|
phase: pod.status?.phase ?? null,
|
|
123
167
|
cs: containers,
|
|
168
|
+
...(sidecars.length === 0 ? {} : { sc: sidecars }),
|
|
124
169
|
labels: pod.metadata.labels ?? {},
|
|
125
170
|
});
|
|
126
|
-
if (event.type !== "DELETED" && pod.metadata.labels?.["lab.systems/log"] === "1")
|
|
127
|
-
|
|
128
|
-
|
|
171
|
+
if (event.type !== "DELETED" && pod.metadata.labels?.["lab.systems/log"] === "1") {
|
|
172
|
+
const running = [...(pod.status?.containerStatuses ?? []), ...(pod.status?.initContainerStatuses ?? []).filter((status) => status.name === PROXY)];
|
|
173
|
+
for (const status of running) if (status.state?.running && status.containerID) follow(ns, pod.metadata.name, status.name, status.containerID);
|
|
174
|
+
}
|
|
129
175
|
return pod.metadata.resourceVersion;
|
|
130
176
|
});
|
|
131
177
|
stream(`/apis/discovery.k8s.io/v1/namespaces/${ns}/endpointslices`, { watch: "1", allowWatchBookmarks: "true" }, (text, t) => {
|
package/package.json
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json.schemastore.org/package",
|
|
3
3
|
"name": "@mark1russell7/kind",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.3.0",
|
|
5
5
|
"description": "Kubernetes in Docker as a lab for connection timers: a kind cluster from a scenario, faults in the namespaces of its pods, observations on the one monotonic clock of the VM, and predictions from the ordering law. Its judge entry reads and judges a raw record without Docker.",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"kubernetes",
|
|
8
8
|
"kind",
|
|
9
9
|
"kube-proxy",
|
|
10
|
+
"istio",
|
|
10
11
|
"endpoints",
|
|
11
12
|
"readiness",
|
|
12
13
|
"graceful-shutdown",
|
|
@@ -54,6 +55,7 @@
|
|
|
54
55
|
"src/backends.ts",
|
|
55
56
|
"src/conntrack.ts",
|
|
56
57
|
"src/drops.ts",
|
|
58
|
+
"src/eds.ts",
|
|
57
59
|
"README.md",
|
|
58
60
|
"LICENSE"
|
|
59
61
|
],
|
package/src/eds.ts
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// The hosts of one cluster in the Envoy of a sidecar, from the text of its admin endpoint
|
|
2
|
+
// `/clusters`. The agent of the lab image reads it with Node, thus this module imports nothing.
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* These are the hosts of a cluster: the healthy ones (`b`, in sequence), and each other host with its
|
|
6
|
+
* health flags (`u`). Each host is `ip:port`.
|
|
7
|
+
*/
|
|
8
|
+
export interface ClusterHosts {
|
|
9
|
+
readonly b: readonly string[];
|
|
10
|
+
readonly u: Readonly<Record<string, string>>;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* This function gives the name of the outbound cluster of a Service port in a sidecar of Istio:
|
|
15
|
+
* `outbound|<port>||<service>.<namespace>.svc.cluster.local`.
|
|
16
|
+
*/
|
|
17
|
+
export const outboundCluster = (service: string, namespace: string, port: number): string => `outbound|${port}||${service}.${namespace}.svc.cluster.local`;
|
|
18
|
+
|
|
19
|
+
/** This function gives the filter of `/clusters` that selects one cluster: a regular expression of RE2 with anchors. */
|
|
20
|
+
export const clusterFilter = (cluster: string): string => `^${cluster.replace(/[.|\\^$*+?()[\]{}]/g, (c) => `\\${c}`)}$`;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* This function gives the hosts of a cluster from the text of `/clusters`. Each host of the text has
|
|
24
|
+
* a line `<cluster>::<ip:port>::health_flags::<flags>`. The flags are `healthy`, or the names of the
|
|
25
|
+
* flags that the host has (`clusters_handler.cc` and `host_utility.cc` of Envoy v1.39.4).
|
|
26
|
+
*/
|
|
27
|
+
export const hostsOf = (text: string, cluster: string): ClusterHosts => {
|
|
28
|
+
const healthy: string[] = [];
|
|
29
|
+
const other: Record<string, string> = {};
|
|
30
|
+
const prefix = `${cluster}::`;
|
|
31
|
+
for (const line of text.split("\n")) {
|
|
32
|
+
if (!line.startsWith(prefix)) continue;
|
|
33
|
+
const parts = line.slice(prefix.length).split("::");
|
|
34
|
+
if (parts.length !== 3 || parts[1] !== "health_flags") continue;
|
|
35
|
+
const [host, , flags] = parts as [string, string, string];
|
|
36
|
+
if (flags.trim() === "healthy") healthy.push(host);
|
|
37
|
+
else other[host] = flags.trim();
|
|
38
|
+
}
|
|
39
|
+
return { b: healthy.sort(), u: other };
|
|
40
|
+
};
|