@mark1russell7/kind 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/build/index.d.ts +23 -5
- package/build/index.js +143 -7
- package/build/index.js.map +1 -1
- package/build/{judge-BS9pk96o.d.ts → judge-B1nSdbS0.d.ts} +98 -30
- package/build/{judge-Bkbs7VVG.js → judge-CmiF-Fcs.js} +192 -59
- package/build/judge-CmiF-Fcs.js.map +1 -0
- package/build/judge.d.ts +2 -2
- package/build/judge.js +1 -1
- package/image/Dockerfile +1 -1
- package/image/client-grpc.mjs +9 -0
- package/package.json +2 -1
- package/src/client-lines.ts +60 -0
- package/build/judge-Bkbs7VVG.js.map +0 -1
|
@@ -96,6 +96,12 @@ type MeshMode = "sidecar" | "ambient";
|
|
|
96
96
|
interface Mesh {
|
|
97
97
|
readonly flavor: MeshFlavor;
|
|
98
98
|
readonly mode: MeshMode;
|
|
99
|
+
/**
|
|
100
|
+
* These variables are the environment of the DaemonSet of ztunnel for the cell, in the flavor
|
|
101
|
+
* `ambient`. The lab sets them before the cell and removes them after it. Each change is a rollout
|
|
102
|
+
* of the DaemonSet. Without them, ztunnel has the environment of its chart.
|
|
103
|
+
*/
|
|
104
|
+
readonly ztunnel?: Readonly<Record<string, string>> | undefined;
|
|
99
105
|
}
|
|
100
106
|
/**
|
|
101
107
|
* A stack is the server and the clients of a cell:
|
|
@@ -118,16 +124,18 @@ type Stack = "service" | "grpc";
|
|
|
118
124
|
* - `rollout` is a rolling update while Redis is down.
|
|
119
125
|
* - `cpu-throttle` sets `cpu.max` in the cgroup of one replica.
|
|
120
126
|
* - `authorization-policy` applies an AuthorizationPolicy of Istio that denies the clients, for the hold (T15).
|
|
127
|
+
* - `idle` stops the calls of each client with SIGUSR1, and starts them again with SIGUSR2 after the hold: a gap (T15).
|
|
128
|
+
* - `ztunnel-restart` deletes the ztunnel pod of each node with its grace. The DaemonSet makes a new pod on each node (T15).
|
|
121
129
|
* - `none` injects nothing: a calibration measures the path itself.
|
|
122
130
|
*/
|
|
123
|
-
type TriggerKind = "pod-deletion" | "rolling-update" | "node-drain" | "scale-in" | "crash" | "liveness-restart" | "freeze-process" | "rst" | "blackhole" | "peer-gone-without-close" | "app-stall" | "rollout" | "cpu-throttle" | "authorization-policy";
|
|
131
|
+
type TriggerKind = "pod-deletion" | "rolling-update" | "node-drain" | "scale-in" | "crash" | "liveness-restart" | "freeze-process" | "rst" | "blackhole" | "peer-gone-without-close" | "app-stall" | "rollout" | "cpu-throttle" | "authorization-policy" | "idle" | "ztunnel-restart";
|
|
124
132
|
/**
|
|
125
133
|
* A trigger is its kind and the settings of its fault. The parameter `A` is the form of an amount:
|
|
126
134
|
* a quantity in the format `/2`, a number in the format `/1`.
|
|
127
135
|
*/
|
|
128
136
|
interface Trigger<A = Quantity> {
|
|
129
137
|
readonly kind: TriggerKind;
|
|
130
|
-
/** This time is how long a fault holds before it ends: a stall, a pause of kube-proxy, a black hole
|
|
138
|
+
/** This time is how long a fault holds before it ends: a stall, a pause of kube-proxy, a black hole, or a gap. */
|
|
131
139
|
readonly hold?: Quantity | undefined;
|
|
132
140
|
/** This time is how long before the deletion a pause of kube-proxy starts (T07 `freeze-process`). */
|
|
133
141
|
readonly lead?: Quantity | undefined;
|
|
@@ -219,8 +227,9 @@ interface Band<A = Quantity> {
|
|
|
219
227
|
* - `fault`: the commit of the fault, as the injector or the agent stamps it.
|
|
220
228
|
* - `resume`: the end of a pause of kube-proxy, as the agent stamps it.
|
|
221
229
|
* - `heal`: the start of the first act that ends the fault, for example the delete of a policy.
|
|
230
|
+
* - `restart`: the first deletion timestamp of a ztunnel pod in the run, as the watcher receives it.
|
|
222
231
|
*/
|
|
223
|
-
type Reference = "deletion" | "intake" | "fault" | "resume" | "heal";
|
|
232
|
+
type Reference = "deletion" | "intake" | "fault" | "resume" | "heal" | "restart";
|
|
224
233
|
/**
|
|
225
234
|
* A reset reuse is how a reset fails a later connection on the same port. A reset from the server
|
|
226
235
|
* leaves the conntrack entry of its flow in CLOSE, with the NAT to that server, for `life`. A SYN on
|
|
@@ -245,6 +254,9 @@ interface ResetReuse<A = Quantity> {
|
|
|
245
254
|
* - `restarts`: each replica receives SIGTERM in the band, or no replica does.
|
|
246
255
|
* - `stays`: no endpoint leaves the Service, and no request is refused.
|
|
247
256
|
* - `stalls`: the rollout does not continue while Redis is down, and it ends after Redis is back.
|
|
257
|
+
* - `ends`: the socket of each client ends in the band after the reference `from`. With no band, no
|
|
258
|
+
* socket of a client ends in the run. Then the first request of each client after the heal goes on
|
|
259
|
+
* the socket of its last request before the fault. With `window`, the failed requests also start in that window.
|
|
248
260
|
* - `measured`: no document bounds the outcome. The run keeps the measurement.
|
|
249
261
|
*/
|
|
250
262
|
type Prediction<A = Quantity> = {
|
|
@@ -252,7 +264,42 @@ type Prediction<A = Quantity> = {
|
|
|
252
264
|
/** These outcomes are the failures that the prediction excludes. Without them, it excludes each outcome but `ok`. */
|
|
253
265
|
readonly outcomes?: readonly string[] | undefined;
|
|
254
266
|
readonly why: string;
|
|
267
|
+
} | WindowPrediction<A> | {
|
|
268
|
+
readonly kind: "removal";
|
|
269
|
+
readonly band: Band<A>;
|
|
270
|
+
readonly why: string;
|
|
271
|
+
} | {
|
|
272
|
+
readonly kind: "leaves";
|
|
273
|
+
readonly band: Band<A>;
|
|
274
|
+
readonly why: string;
|
|
255
275
|
} | {
|
|
276
|
+
readonly kind: "restarts";
|
|
277
|
+
readonly band: Band<A> | null;
|
|
278
|
+
readonly why: string;
|
|
279
|
+
} | {
|
|
280
|
+
readonly kind: "stays";
|
|
281
|
+
readonly why: string;
|
|
282
|
+
} | {
|
|
283
|
+
readonly kind: "stalls";
|
|
284
|
+
readonly why: string;
|
|
285
|
+
} | {
|
|
286
|
+
readonly kind: "ends";
|
|
287
|
+
/** This reference is where the band of each close starts. */
|
|
288
|
+
readonly from: Reference;
|
|
289
|
+
/** This band is the close of the socket of each client after `from`, or null when no socket of a client ends. */
|
|
290
|
+
readonly band: Band<A> | null;
|
|
291
|
+
/** This window is where the failed requests start, or none when the prediction does not test them. */
|
|
292
|
+
readonly window?: WindowPrediction<A> | undefined;
|
|
293
|
+
readonly why: string;
|
|
294
|
+
} | {
|
|
295
|
+
readonly kind: "measured";
|
|
296
|
+
readonly why: string;
|
|
297
|
+
};
|
|
298
|
+
/**
|
|
299
|
+
* A window prediction is the requests that fail in a window. The window starts at `start` and ends in
|
|
300
|
+
* `end`. With `certain`, at least one request fails.
|
|
301
|
+
*/
|
|
302
|
+
interface WindowPrediction<A = Quantity> {
|
|
256
303
|
readonly kind: "window";
|
|
257
304
|
readonly nodes: "all" | "paused";
|
|
258
305
|
readonly start: {
|
|
@@ -284,28 +331,7 @@ type Prediction<A = Quantity> = {
|
|
|
284
331
|
/** These outcomes are the only failures that the window permits. A failure with another outcome fails the run. Without them, each outcome but `ok` can fail. */
|
|
285
332
|
readonly outcomes?: readonly string[] | undefined;
|
|
286
333
|
readonly why: string;
|
|
287
|
-
}
|
|
288
|
-
readonly kind: "removal";
|
|
289
|
-
readonly band: Band<A>;
|
|
290
|
-
readonly why: string;
|
|
291
|
-
} | {
|
|
292
|
-
readonly kind: "leaves";
|
|
293
|
-
readonly band: Band<A>;
|
|
294
|
-
readonly why: string;
|
|
295
|
-
} | {
|
|
296
|
-
readonly kind: "restarts";
|
|
297
|
-
readonly band: Band<A> | null;
|
|
298
|
-
readonly why: string;
|
|
299
|
-
} | {
|
|
300
|
-
readonly kind: "stays";
|
|
301
|
-
readonly why: string;
|
|
302
|
-
} | {
|
|
303
|
-
readonly kind: "stalls";
|
|
304
|
-
readonly why: string;
|
|
305
|
-
} | {
|
|
306
|
-
readonly kind: "measured";
|
|
307
|
-
readonly why: string;
|
|
308
|
-
};
|
|
334
|
+
}
|
|
309
335
|
/** A replica is one pod of the service at the start of a run. */
|
|
310
336
|
interface Replica {
|
|
311
337
|
readonly pod: string;
|
|
@@ -498,11 +524,25 @@ interface RecordLine {
|
|
|
498
524
|
readonly on: string;
|
|
499
525
|
readonly d: string | null;
|
|
500
526
|
}
|
|
527
|
+
/**
|
|
528
|
+
* A socket line is the connect or the close of a socket of the channel of a gRPC client
|
|
529
|
+
* (`client-lines.ts`). It gives the client (`p`) and its node (`n`), and the local port of the socket.
|
|
530
|
+
* A close also gives the code of an error of the socket (`err`), or null for a close with no error.
|
|
531
|
+
*/
|
|
532
|
+
interface SocketLine {
|
|
533
|
+
readonly t: number;
|
|
534
|
+
readonly what: "connect" | "close";
|
|
535
|
+
readonly p: string;
|
|
536
|
+
readonly n: string;
|
|
537
|
+
readonly lp: number | null;
|
|
538
|
+
readonly err?: string | null;
|
|
539
|
+
}
|
|
501
540
|
declare const requestsOf: (lines: readonly Line[]) => Request[];
|
|
502
541
|
declare const podsOf: (lines: readonly Line[]) => PodLine[];
|
|
503
542
|
declare const slicesOf: (lines: readonly Line[], svc: string) => SliceLine[];
|
|
504
543
|
declare const rulesOf: (lines: readonly Line[], source?: PropagationSource) => RulesLine[];
|
|
505
544
|
declare const recordsOf: (lines: readonly Line[]) => RecordLine[];
|
|
545
|
+
declare const socketsOf: (lines: readonly Line[]) => SocketLine[];
|
|
506
546
|
/** This function gives the requests that did not succeed, in sequence of their start. */
|
|
507
547
|
declare const failuresOf: (lines: readonly Line[]) => Request[];
|
|
508
548
|
/** This function counts the outcomes of the requests, by outcome. */
|
|
@@ -520,6 +560,29 @@ declare const byClients: <T extends {
|
|
|
520
560
|
declare const byNode: (requests: readonly Request[]) => Map<string, Request[]>;
|
|
521
561
|
/** This function gives the time that the watcher first received a deletion timestamp of one of the pods, after a time. */
|
|
522
562
|
declare const deletionOf: (lines: readonly Line[], pods: readonly string[], after: number) => number | null;
|
|
563
|
+
/**
|
|
564
|
+
* A ztunnel pod line is an event of a ztunnel pod in istio-system, from the watcher (`image/watch.mjs`).
|
|
565
|
+
* Its `type` is the type of the event of the API, and `cs` gives the readiness of each container (`r`).
|
|
566
|
+
*/
|
|
567
|
+
interface ZtunnelPodLine {
|
|
568
|
+
readonly t: number;
|
|
569
|
+
readonly type: string;
|
|
570
|
+
readonly name: string;
|
|
571
|
+
readonly node: string | null;
|
|
572
|
+
readonly del: boolean;
|
|
573
|
+
readonly cs: ReadonlyArray<{
|
|
574
|
+
readonly r: boolean;
|
|
575
|
+
}>;
|
|
576
|
+
}
|
|
577
|
+
declare const ztunnelPodsOf: (lines: readonly Line[]) => ZtunnelPodLine[];
|
|
578
|
+
/** This function gives the time that the watcher first received a deletion timestamp of a ztunnel pod, after a time: the reference `restart`. */
|
|
579
|
+
declare const restartOf: (lines: readonly Line[], after: number) => number | null;
|
|
580
|
+
/**
|
|
581
|
+
* This function tells if a restart of ztunnel is over. Each ztunnel pod that got a deletion in the
|
|
582
|
+
* lines is gone. Each node has a ready ztunnel pod with no deletion. It is false when no ztunnel pod
|
|
583
|
+
* got a deletion.
|
|
584
|
+
*/
|
|
585
|
+
declare const ztunnelReplaced: (lines: readonly Line[], nodes: readonly string[]) => boolean;
|
|
523
586
|
/** This function gives the stamp of a transition of the service in a pod: the first matching record of a machine, after a time. */
|
|
524
587
|
declare const transitionOf: (lines: readonly Line[], pod: string, machine: string, match: (record: RecordLine) => boolean, after: number) => number | null;
|
|
525
588
|
/**
|
|
@@ -610,6 +673,8 @@ interface RunFacts<A = Quantity> {
|
|
|
610
673
|
readonly updates?: Readonly<Record<string, Propagated<A> | null>> | undefined;
|
|
611
674
|
/** These are the first LDS updates of the sidecar of each server pod after the heal, by pod. */
|
|
612
675
|
readonly healUpdates?: Readonly<Record<string, Propagated<A> | null>> | undefined;
|
|
676
|
+
/** These are the first close of the socket of each client after the reference (`ends`), by client, or null for none. */
|
|
677
|
+
readonly closes?: Readonly<Record<string, A | null>> | undefined;
|
|
613
678
|
}
|
|
614
679
|
/** A deleted replica is a replica of a run that got a deletion. It gives its pod, and the stamps of its intake (or null) and of its deletion. */
|
|
615
680
|
interface Deleted {
|
|
@@ -648,9 +713,10 @@ interface KindVerdict<A = Quantity> {
|
|
|
648
713
|
}
|
|
649
714
|
/**
|
|
650
715
|
* This function gives the unit of the main value of a cell, from its prediction:
|
|
651
|
-
* - `s` for a time: the removal, the departure or the restart of a band,
|
|
716
|
+
* - `s` for a time: the removal, the departure or the restart of a band, the end of a stall, or a close.
|
|
652
717
|
* - `{request}` for the failed requests: no failure, a window, a fleet that stays, and a measurement.
|
|
653
718
|
* - `{replica}` for the replicas that received SIGTERM, when no replica must restart.
|
|
719
|
+
* - `{socket}` for the sockets of the clients that ended, when no socket must end.
|
|
654
720
|
*/
|
|
655
721
|
declare const unitOf: (prediction: {
|
|
656
722
|
readonly kind: Prediction["kind"];
|
|
@@ -697,7 +763,7 @@ declare const deletionsOf: (run: KindRun) => Deleted[];
|
|
|
697
763
|
* This function gives how long a run observes after the end reference of its last window, or null
|
|
698
764
|
* when its prediction sets no such time. The judge closes a window `end.band.hi` + δ after its end
|
|
699
765
|
* reference. A request that starts by then ends at most the timeout of the workload later: the
|
|
700
|
-
* client cancels it then. A prediction
|
|
766
|
+
* client cancels it then. A prediction with no window, or a window with no end, sets none.
|
|
701
767
|
*/
|
|
702
768
|
declare const observationAfter: (cell: Pick<KindCell, "prediction" | "workload">) => Quantity | null;
|
|
703
769
|
/**
|
|
@@ -707,6 +773,8 @@ declare const observationAfter: (cell: Pick<KindCell, "prediction" | "workload">
|
|
|
707
773
|
* it can miss its own failures. In a rolling update of gRPC, a call waits in the queue of its channel
|
|
708
774
|
* for a stalled connect. It fails only at its timeout (kind run 40128267098). A reference that is not
|
|
709
775
|
* in the lines yet does not count.
|
|
776
|
+
*
|
|
777
|
+
* The band of `ends` also sets an end: its reference, the end of the band and δ.
|
|
710
778
|
*/
|
|
711
779
|
declare const observationEnd: (cell: KindCell, run: Pick<KindRun, "start" | "acts" | "target" | "replicas" | "lines">) => number | null;
|
|
712
780
|
/** This function gives the facts of a run that each table reads: its failures by outcome and node, and the times of its target. */
|
|
@@ -818,7 +886,7 @@ interface TimeSpread {
|
|
|
818
886
|
}
|
|
819
887
|
/** This function gives the spread of a set of times, or null for none. The percentile is by nearest rank. */
|
|
820
888
|
declare const timeSpread: (times: readonly Quantity[]) => TimeSpread | null;
|
|
821
|
-
/** This function gives the prediction of a cell in a short form for a table. */
|
|
889
|
+
/** This function gives the prediction of a cell in a short form for a table. A prediction `ends` gives its window after its band. */
|
|
822
890
|
declare const predictionText: (prediction: Prediction) => string;
|
|
823
891
|
/**
|
|
824
892
|
* This function gives the main measurement of a cell in a short form: its median and what it
|
|
@@ -876,5 +944,5 @@ declare const quantileOf: (sorted: readonly number[], p: number) => number;
|
|
|
876
944
|
*/
|
|
877
945
|
declare const medianInterval: (sorted: readonly number[]) => [number, number];
|
|
878
946
|
//#endregion
|
|
879
|
-
export { updatesOf as $,
|
|
880
|
-
//# sourceMappingURL=judge-
|
|
947
|
+
export { updatesOf as $, MeshFlavor as $t, TriggerV1 as A, sliceEmptyOf as At, RunCheck as B, PORT as Bt, KindCellResult as C, kubeProxyPatch as Cn, requestsOf as Ct, PredictionV1 as D, signalOf as Dt, LoadSample as E, scenarioProblems as En, rulesOf as Et, KIND_NOISE as F, ztunnelPodsOf as Ft, judgeKind as G, secondsOf as Gt, checkRun as H, SERVER as Ht, KindDecision as I, ztunnelReplaced as It, propagationOf as J, Declaration as Jt, observationAfter as K, showBand as Kt, KindVerdict as L, CLIENT as Lt, upgrade as M, socketsOf as Mt, upgradeCell as N, stoppingOf as Nt, RunCheckV1 as O, signalsOf as Ot, Deleted as P, transitionOf as Pt, unitOf as Q, Mesh as Qt, LAST_SLACK as R, HEALTH_PORT as Rt, KindCellRecordV1 as S, kindConfig as Sn, recordsOf as St, KindVerdictV1 as T, scenario as Tn, restartsAt as Tt, deletionsOf as U, bandOf as Ut, RunFacts as V, REDIS as Vt, factsOf as W, seconds as Wt, referenceOf as X, KindRun as Xt, propagationSource as Y, KindCell as Yt, reusesReset as Z, KubeletProbe as Zt, ClusterManifest as _, NODE_IMAGES as _n, intakeOf as _t, STAGES as a, Replica as an, Request as at, FORMAT_V1 as b, cidrRange as bn, podsAt as bt, readRecord as c, Template as cn, SocketLine as ct, calibrationOf as d, WindowPrediction as dn, byClients as dt, MeshMode as en, Endpoint as et, cellsTable as f, Workload as fn, byNode as ft, BandV1 as g, KubeProxyMode as gn, failuresOf as gt, timeSpread as h, KindScenario as hn, emptyOf as ht, quantileOf as i, Reference as in, RecordLine as it, judge as j, slicesOf as jt, RunFactsV1 as k, sliceDropOf as kt, Judged as l, Trigger as ln, ZtunnelPodLine as lt, predictionText as m, KindConfig as mn, dropOf as mt, medianInterval as n, Prediction as nn, PodLine as nt, kindCellRecordSchema as o, ResetReuse as on, RulesLine as ot, medianText as p, KUBE_PROXY_MODES as pn, deletionOf as pt, observationEnd as q, Band as qt, middle as r, RedisDeclaration as rn, PropagationSource as rt, kindCellRecordV1Schema as s, Stack as sn, SliceLine as st, RELATIVE as t, PodSettings as tn, Line as tt, TimeSpread as u, TriggerKind as un, backendsAt as ut, EnvironmentManifest as v, NodeImage as vn, kernelCounts as vt, KindCellV1 as w, nodeNames as wn, restartOf as wt, KindCellRecord as x, goDuration as xn, podsOf as xt, FORMAT as y, cidrOverlap as yn, outcomesOf as yt, Propagated as z, LOG_LABEL as zt };
|
|
948
|
+
//# sourceMappingURL=judge-B1nSdbS0.d.ts.map
|
|
@@ -42,6 +42,7 @@ const podsOf = (lines) => of(lines, "pod").sort((a, b) => a.t - b.t);
|
|
|
42
42
|
const slicesOf = (lines, svc) => of(lines, "slice").filter((line) => line.svc === svc);
|
|
43
43
|
const rulesOf = (lines, source = "rules") => of(lines, source).sort((a, b) => a.t - b.t);
|
|
44
44
|
const recordsOf = (lines) => of(lines, "rec");
|
|
45
|
+
const socketsOf = (lines) => of(lines, "sock").sort((a, b) => a.t - b.t);
|
|
45
46
|
/** This function gives the requests that did not succeed, in sequence of their start. */
|
|
46
47
|
const failuresOf = (lines) => requestsOf(lines).filter((request) => request.o !== "ok").sort((a, b) => a.t - b.t);
|
|
47
48
|
/** This function counts the outcomes of the requests, by outcome. */
|
|
@@ -68,6 +69,22 @@ const byNode = (requests) => {
|
|
|
68
69
|
};
|
|
69
70
|
/** This function gives the time that the watcher first received a deletion timestamp of one of the pods, after a time. */
|
|
70
71
|
const deletionOf = (lines, pods, after) => podsOf(lines).find((pod) => pods.includes(pod.name) && pod.del && pod.t >= after)?.t ?? null;
|
|
72
|
+
const ztunnelPodsOf = (lines) => of(lines, "ztunnel-pod").sort((a, b) => a.t - b.t);
|
|
73
|
+
/** This function gives the time that the watcher first received a deletion timestamp of a ztunnel pod, after a time: the reference `restart`. */
|
|
74
|
+
const restartOf = (lines, after) => ztunnelPodsOf(lines).find((pod) => pod.del && pod.t >= after)?.t ?? null;
|
|
75
|
+
/**
|
|
76
|
+
* This function tells if a restart of ztunnel is over. Each ztunnel pod that got a deletion in the
|
|
77
|
+
* lines is gone. Each node has a ready ztunnel pod with no deletion. It is false when no ztunnel pod
|
|
78
|
+
* got a deletion.
|
|
79
|
+
*/
|
|
80
|
+
const ztunnelReplaced = (lines, nodes) => {
|
|
81
|
+
const last = /* @__PURE__ */ new Map();
|
|
82
|
+
for (const pod of ztunnelPodsOf(lines)) last.set(pod.name, pod);
|
|
83
|
+
const deleted = [...last.values()].filter((pod) => pod.del || pod.type === "DELETED");
|
|
84
|
+
if (deleted.length === 0 || deleted.some((pod) => pod.type !== "DELETED")) return false;
|
|
85
|
+
const ready = new Set([...last.values()].filter((pod) => !pod.del && pod.type !== "DELETED" && pod.cs.length > 0 && pod.cs.every((status) => status.r)).map((pod) => pod.node));
|
|
86
|
+
return nodes.every((node) => ready.has(node));
|
|
87
|
+
};
|
|
71
88
|
/** This function gives the stamp of a transition of the service in a pod: the first matching record of a machine, after a time. */
|
|
72
89
|
const transitionOf = (lines, pod, machine, match, after) => recordsOf(lines).find((record) => record.p === pod && record.m === machine && record.at >= after && match(record))?.at ?? null;
|
|
73
90
|
/**
|
|
@@ -192,9 +209,10 @@ const NOISE = valueIn(KIND_NOISE, "ms");
|
|
|
192
209
|
const SLACK = valueIn(LAST_SLACK, "ms");
|
|
193
210
|
/**
|
|
194
211
|
* This function gives the unit of the main value of a cell, from its prediction:
|
|
195
|
-
* - `s` for a time: the removal, the departure or the restart of a band,
|
|
212
|
+
* - `s` for a time: the removal, the departure or the restart of a band, the end of a stall, or a close.
|
|
196
213
|
* - `{request}` for the failed requests: no failure, a window, a fleet that stays, and a measurement.
|
|
197
214
|
* - `{replica}` for the replicas that received SIGTERM, when no replica must restart.
|
|
215
|
+
* - `{socket}` for the sockets of the clients that ended, when no socket must end.
|
|
198
216
|
*/
|
|
199
217
|
const unitOf = (prediction) => {
|
|
200
218
|
switch (prediction.kind) {
|
|
@@ -202,6 +220,7 @@ const unitOf = (prediction) => {
|
|
|
202
220
|
case "leaves":
|
|
203
221
|
case "stalls": return "s";
|
|
204
222
|
case "restarts": return prediction.band === null ? "{replica}" : "s";
|
|
223
|
+
case "ends": return prediction.band === null ? "{socket}" : "s";
|
|
205
224
|
default: return "{request}";
|
|
206
225
|
}
|
|
207
226
|
};
|
|
@@ -227,6 +246,7 @@ const referenceOf = (run, from, pod = run.target?.pod) => {
|
|
|
227
246
|
if (from === "fault") return run.acts.fault;
|
|
228
247
|
if (from === "resume") return run.acts.resume;
|
|
229
248
|
if (from === "heal") return run.acts.heal;
|
|
249
|
+
if (from === "restart") return restartOf(lines, run.start);
|
|
230
250
|
if (pod === void 0) return null;
|
|
231
251
|
if (from === "deletion") return deletionOf(lines, [pod], run.start);
|
|
232
252
|
return intakeOf(lines, pod, run.start);
|
|
@@ -296,16 +316,18 @@ const deletionsOf = (run) => run.replicas.flatMap((replica) => {
|
|
|
296
316
|
}).sort((a, b) => a.deletion - b.deletion);
|
|
297
317
|
/** This function tells if a prediction is a window for each deleted replica. */
|
|
298
318
|
const eachOf = (prediction) => prediction.kind === "window" && prediction.each === true;
|
|
319
|
+
/** This function gives the window of the failed requests of a prediction: a window, the window of `ends`, or null. */
|
|
320
|
+
const windowOf = (prediction) => prediction.kind === "window" ? prediction : prediction.kind === "ends" ? prediction.window ?? null : null;
|
|
299
321
|
/**
|
|
300
322
|
* This function gives how long a run observes after the end reference of its last window, or null
|
|
301
323
|
* when its prediction sets no such time. The judge closes a window `end.band.hi` + δ after its end
|
|
302
324
|
* reference. A request that starts by then ends at most the timeout of the workload later: the
|
|
303
|
-
* client cancels it then. A prediction
|
|
325
|
+
* client cancels it then. A prediction with no window, or a window with no end, sets none.
|
|
304
326
|
*/
|
|
305
327
|
const observationAfter = (cell) => {
|
|
306
|
-
const
|
|
307
|
-
if (
|
|
308
|
-
return seconds(sec(valueIn(
|
|
328
|
+
const window = windowOf(cell.prediction);
|
|
329
|
+
if (window === null || window.end === null) return null;
|
|
330
|
+
return seconds(sec(valueIn(window.end.band.hi, "ms") + valueIn(cell.workload.timeout, "ms") + NOISE));
|
|
309
331
|
};
|
|
310
332
|
/**
|
|
311
333
|
* This function gives the earliest end of the observation of a run, as a stamp, or null when its
|
|
@@ -314,17 +336,27 @@ const observationAfter = (cell) => {
|
|
|
314
336
|
* it can miss its own failures. In a rolling update of gRPC, a call waits in the queue of its channel
|
|
315
337
|
* for a stalled connect. It fails only at its timeout (kind run 40128267098). A reference that is not
|
|
316
338
|
* in the lines yet does not count.
|
|
339
|
+
*
|
|
340
|
+
* The band of `ends` also sets an end: its reference, the end of the band and δ.
|
|
317
341
|
*/
|
|
318
342
|
const observationEnd = (cell, run) => {
|
|
319
|
-
const
|
|
343
|
+
const ends = [];
|
|
320
344
|
const prediction = cell.prediction;
|
|
321
|
-
if (
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
345
|
+
if (prediction.kind === "ends" && prediction.band !== null) {
|
|
346
|
+
const at = referenceOf(run, prediction.from);
|
|
347
|
+
if (at !== null) ends.push(at + valueIn(prediction.band.hi, "ms") + NOISE);
|
|
348
|
+
}
|
|
349
|
+
const after = observationAfter(cell);
|
|
350
|
+
const window = windowOf(prediction);
|
|
351
|
+
if (after !== null && window !== null && window.end !== null) {
|
|
352
|
+
const from = window.end.from;
|
|
353
|
+
const references = (window.each === true ? run.replicas.map((replica) => replica.pod) : [run.target?.pod]).flatMap((pod) => {
|
|
354
|
+
const at = referenceOf(run, from, pod);
|
|
355
|
+
return at === null ? [] : [at];
|
|
356
|
+
});
|
|
357
|
+
if (references.length > 0) ends.push(Math.max(...references) + valueIn(after, "ms"));
|
|
358
|
+
}
|
|
359
|
+
return ends.length === 0 ? null : Math.max(...ends);
|
|
328
360
|
};
|
|
329
361
|
/** This function gives the facts of a run that each table reads: its failures by outcome and node, and the times of its target. */
|
|
330
362
|
const factsOf = (cell, run) => {
|
|
@@ -599,9 +631,83 @@ const checkRun = (cell, run) => {
|
|
|
599
631
|
if (problems.length === 0) return done(true, `the rollout waited for Redis, and its first new pod was ready ${sec(firstReady.t - heal)} s after Redis came back`, sec(firstReady.t - heal));
|
|
600
632
|
return done(false, problems.join("; "), null);
|
|
601
633
|
}
|
|
634
|
+
case "ends": return checkEnds(cell, run, prediction, done);
|
|
602
635
|
case "measured": return done(true, "measured", facts.failures);
|
|
603
636
|
}
|
|
604
637
|
};
|
|
638
|
+
/**
|
|
639
|
+
* This function judges a run against a prediction `ends`. Each client with an answer before the
|
|
640
|
+
* reference is a client of the run. Its socket is the port of that answer (`lp`).
|
|
641
|
+
* - With a band, the first close of a socket of each client after the reference must be in the band
|
|
642
|
+
* ± δ. It must be the close of that socket. The main value is the middle of these times.
|
|
643
|
+
* - With no band, no socket of a client can close after the reference. The first request of each client
|
|
644
|
+
* after the heal must be `ok` on that socket. The main value is the count of the closes.
|
|
645
|
+
*
|
|
646
|
+
* With a window, the failed requests of the run must also agree with that window (`checkRun`).
|
|
647
|
+
*/
|
|
648
|
+
const checkEnds = (cell, run, prediction, done) => {
|
|
649
|
+
const from = referenceOf(run, prediction.from);
|
|
650
|
+
if (from === null) return done(false, `the run has no ${prediction.from} reference`, null);
|
|
651
|
+
const lines = run.lines;
|
|
652
|
+
const calls = requestsOf(lines).filter((request) => request.t >= run.start && request.t <= run.end).sort((a, b) => a.t - b.t);
|
|
653
|
+
const last = /* @__PURE__ */ new Map();
|
|
654
|
+
for (const request of calls) if (request.t < from && request.o === "ok") last.set(request.p, request);
|
|
655
|
+
if (last.size === 0) return done(false, `no client had an answer before the ${prediction.from}`, null);
|
|
656
|
+
const clients = [...last.keys()].sort();
|
|
657
|
+
const shut = socketsOf(lines).filter((line) => line.what === "close" && line.t >= from && line.t <= run.end);
|
|
658
|
+
const closes = {};
|
|
659
|
+
const times = [];
|
|
660
|
+
const problems = [];
|
|
661
|
+
for (const client of clients) {
|
|
662
|
+
const close = shut.find((line) => line.p === client);
|
|
663
|
+
closes[client] = close === void 0 ? null : seconds(sec(close.t - from));
|
|
664
|
+
if (close !== void 0) times.push(sec(close.t - from));
|
|
665
|
+
}
|
|
666
|
+
let agrees;
|
|
667
|
+
let value;
|
|
668
|
+
if (prediction.band !== null) {
|
|
669
|
+
const band = secondsOf(prediction.band);
|
|
670
|
+
for (const client of clients) {
|
|
671
|
+
const close = shut.find((line) => line.p === client);
|
|
672
|
+
const port = last.get(client).lp ?? null;
|
|
673
|
+
if (close === void 0) problems.push(`the socket of ${client} did not end`);
|
|
674
|
+
else if (port !== null && close.lp !== null && close.lp !== port) problems.push(`the first socket of ${client} to end had the port ${close.lp}, not the port ${port} of its last answer`);
|
|
675
|
+
else if (!inBand(sec(close.t - from), band)) problems.push(`the socket of ${client} ended at ${sec(close.t - from)} s`);
|
|
676
|
+
}
|
|
677
|
+
const sorted = [...times].sort((a, b) => a - b);
|
|
678
|
+
value = sorted.length === 0 ? null : middle(sorted);
|
|
679
|
+
agrees = `the socket of each of ${clients.length} clients ended, at ${sorted.join(", ")} s`;
|
|
680
|
+
} else {
|
|
681
|
+
for (const client of clients) {
|
|
682
|
+
const close = shut.find((line) => line.p === client);
|
|
683
|
+
if (close !== void 0) problems.push(`the socket of ${client} ended ${sec(close.t - from)} s after the ${prediction.from}`);
|
|
684
|
+
}
|
|
685
|
+
const resumed = run.acts.heal ?? from;
|
|
686
|
+
for (const client of clients) {
|
|
687
|
+
const port = last.get(client).lp ?? null;
|
|
688
|
+
const next = calls.find((request) => request.p === client && request.t >= resumed);
|
|
689
|
+
if (next === void 0) problems.push(`${client} sent no call after the heal`);
|
|
690
|
+
else if (next.o !== "ok") problems.push(`the first call of ${client} after the heal ended ${next.o}`);
|
|
691
|
+
else if ((next.lp ?? null) !== port) problems.push(`the first call of ${client} after the heal went on the port ${next.lp ?? "none"}, not ${port ?? "none"}`);
|
|
692
|
+
}
|
|
693
|
+
value = times.length;
|
|
694
|
+
agrees = `no socket of ${clients.length} clients ended, and the first call of each one after the heal went on its socket`;
|
|
695
|
+
}
|
|
696
|
+
const inner = prediction.window === void 0 ? null : checkRun({
|
|
697
|
+
...cell,
|
|
698
|
+
prediction: prediction.window
|
|
699
|
+
}, run);
|
|
700
|
+
if (inner !== null && !inner.ok) problems.push(inner.why);
|
|
701
|
+
const why = problems.length > 0 ? problems.join("; ") : inner === null ? agrees : `${agrees}; ${inner.why}`;
|
|
702
|
+
const checked = done(problems.length === 0, why, value, {
|
|
703
|
+
...inner === null ? {} : inner.facts,
|
|
704
|
+
closes
|
|
705
|
+
});
|
|
706
|
+
return inner?.tail === void 0 ? checked : {
|
|
707
|
+
...checked,
|
|
708
|
+
tail: inner.tail
|
|
709
|
+
};
|
|
710
|
+
};
|
|
605
711
|
/** This function judges a cell on its runs: each counted run must agree, and a median in a band must be equivalent to it. */
|
|
606
712
|
const judgeKind = (cell, runs) => {
|
|
607
713
|
const counted = runs.filter((run) => !run.warmup && run.void === null);
|
|
@@ -641,7 +747,7 @@ const judgeKind = (cell, runs) => {
|
|
|
641
747
|
checks
|
|
642
748
|
};
|
|
643
749
|
const prediction = cell.prediction;
|
|
644
|
-
const band = prediction.kind === "removal" || prediction.kind === "leaves" || prediction.kind === "restarts" ? prediction.band : null;
|
|
750
|
+
const band = prediction.kind === "removal" || prediction.kind === "leaves" || prediction.kind === "restarts" || prediction.kind === "ends" ? prediction.band : null;
|
|
645
751
|
if (failed > 0) {
|
|
646
752
|
const first = checks.find((check) => !check.ok);
|
|
647
753
|
return {
|
|
@@ -731,25 +837,34 @@ const upgradeTrigger = (trigger) => {
|
|
|
731
837
|
}
|
|
732
838
|
};
|
|
733
839
|
};
|
|
840
|
+
const upgradeWindow = (window) => {
|
|
841
|
+
const { afterReset, propagation, ...rest } = window;
|
|
842
|
+
return {
|
|
843
|
+
...rest,
|
|
844
|
+
start: {
|
|
845
|
+
from: window.start.from,
|
|
846
|
+
band: band(window.start.band)
|
|
847
|
+
},
|
|
848
|
+
end: window.end === null ? null : {
|
|
849
|
+
from: window.end.from,
|
|
850
|
+
band: band(window.end.band)
|
|
851
|
+
},
|
|
852
|
+
...afterReset === void 0 ? {} : { afterReset: {
|
|
853
|
+
life: seconds(afterReset.life),
|
|
854
|
+
outcome: afterReset.outcome
|
|
855
|
+
} },
|
|
856
|
+
...propagation === void 0 ? {} : { propagation: band(propagation) }
|
|
857
|
+
};
|
|
858
|
+
};
|
|
734
859
|
const upgradePrediction = (prediction) => {
|
|
735
860
|
switch (prediction.kind) {
|
|
736
|
-
case "window":
|
|
737
|
-
|
|
861
|
+
case "window": return upgradeWindow(prediction);
|
|
862
|
+
case "ends": {
|
|
863
|
+
const { window, ...rest } = prediction;
|
|
738
864
|
return {
|
|
739
865
|
...rest,
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
band: band(prediction.start.band)
|
|
743
|
-
},
|
|
744
|
-
end: prediction.end === null ? null : {
|
|
745
|
-
from: prediction.end.from,
|
|
746
|
-
band: band(prediction.end.band)
|
|
747
|
-
},
|
|
748
|
-
...afterReset === void 0 ? {} : { afterReset: {
|
|
749
|
-
life: seconds(afterReset.life),
|
|
750
|
-
outcome: afterReset.outcome
|
|
751
|
-
} },
|
|
752
|
-
...propagation === void 0 ? {} : { propagation: band(propagation) }
|
|
866
|
+
band: prediction.band === null ? null : band(prediction.band),
|
|
867
|
+
...window === void 0 ? {} : { window: upgradeWindow(window) }
|
|
753
868
|
};
|
|
754
869
|
}
|
|
755
870
|
case "removal":
|
|
@@ -776,7 +891,7 @@ const upgradeFacts = (facts) => {
|
|
|
776
891
|
by: seconds(old.by)
|
|
777
892
|
};
|
|
778
893
|
const byKey = (record) => Object.fromEntries(Object.entries(record).map(([key, old]) => [key, propagated(old)]));
|
|
779
|
-
const { sliceEmpty, rules, left, signals, updates, healUpdates, ...rest } = facts;
|
|
894
|
+
const { sliceEmpty, rules, left, signals, updates, healUpdates, closes, ...rest } = facts;
|
|
780
895
|
return {
|
|
781
896
|
...rest,
|
|
782
897
|
intakeAfterDeletion: time$1(facts.intakeAfterDeletion),
|
|
@@ -787,7 +902,8 @@ const upgradeFacts = (facts) => {
|
|
|
787
902
|
...sliceEmpty === void 0 ? {} : { sliceEmpty: seconds(sliceEmpty) },
|
|
788
903
|
...rules === void 0 ? {} : { rules: Object.fromEntries(Object.entries(rules).map(([node, old]) => [node, time$1(old)])) },
|
|
789
904
|
...left === void 0 ? {} : { left: seconds(left) },
|
|
790
|
-
...signals === void 0 ? {} : { signals: signals.map((old) => time$1(old)) }
|
|
905
|
+
...signals === void 0 ? {} : { signals: signals.map((old) => time$1(old)) },
|
|
906
|
+
...closes === void 0 ? {} : { closes: Object.fromEntries(Object.entries(closes).map(([client, old]) => [client, time$1(old)])) }
|
|
791
907
|
};
|
|
792
908
|
};
|
|
793
909
|
/**
|
|
@@ -853,7 +969,7 @@ const atBand = (from, band) => {
|
|
|
853
969
|
const { lo, hi } = secondsOf(band);
|
|
854
970
|
return lo === 0 && hi === 0 ? `the ${from}` : `the ${from} + ${showBand(band)}`;
|
|
855
971
|
};
|
|
856
|
-
/** This function gives the prediction of a cell in a short form for a table. */
|
|
972
|
+
/** This function gives the prediction of a cell in a short form for a table. A prediction `ends` gives its window after its band. */
|
|
857
973
|
const predictionText = (prediction) => {
|
|
858
974
|
switch (prediction.kind) {
|
|
859
975
|
case "none": return prediction.outcomes === void 0 ? "no failure" : `no ${prediction.outcomes.join(", ")}`;
|
|
@@ -863,6 +979,10 @@ const predictionText = (prediction) => {
|
|
|
863
979
|
case "restarts": return prediction.band === null ? "no restart" : `each replica SIGTERM in ${showBand(prediction.band)}`;
|
|
864
980
|
case "stays": return "stays ready, no refusal";
|
|
865
981
|
case "stalls": return "the rollout waits for Redis";
|
|
982
|
+
case "ends": {
|
|
983
|
+
const closes = prediction.band === null ? "no socket of a client ends, and each one calls again on its socket" : `the socket of each client ends in ${atBand(prediction.from, prediction.band)}`;
|
|
984
|
+
return prediction.window === void 0 ? closes : `${closes}; ${predictionText(prediction.window)}`;
|
|
985
|
+
}
|
|
866
986
|
case "measured": return "measured";
|
|
867
987
|
}
|
|
868
988
|
};
|
|
@@ -964,7 +1084,9 @@ const TRIGGERS = [
|
|
|
964
1084
|
"app-stall",
|
|
965
1085
|
"rollout",
|
|
966
1086
|
"cpu-throttle",
|
|
967
|
-
"authorization-policy"
|
|
1087
|
+
"authorization-policy",
|
|
1088
|
+
"idle",
|
|
1089
|
+
"ztunnel-restart"
|
|
968
1090
|
];
|
|
969
1091
|
const MODES = [
|
|
970
1092
|
"iptables",
|
|
@@ -980,7 +1102,8 @@ const reference = z.enum([
|
|
|
980
1102
|
"intake",
|
|
981
1103
|
"fault",
|
|
982
1104
|
"resume",
|
|
983
|
-
"heal"
|
|
1105
|
+
"heal",
|
|
1106
|
+
"restart"
|
|
984
1107
|
]);
|
|
985
1108
|
const probe = z.object({
|
|
986
1109
|
period: time,
|
|
@@ -1114,33 +1237,34 @@ const recordOf = (a, main) => {
|
|
|
1114
1237
|
lo: a,
|
|
1115
1238
|
hi: a
|
|
1116
1239
|
});
|
|
1240
|
+
const window = z.object({
|
|
1241
|
+
kind: z.literal("window"),
|
|
1242
|
+
nodes: z.enum(["all", "paused"]),
|
|
1243
|
+
start: z.object({
|
|
1244
|
+
from: reference,
|
|
1245
|
+
band
|
|
1246
|
+
}),
|
|
1247
|
+
end: z.object({
|
|
1248
|
+
from: reference,
|
|
1249
|
+
band
|
|
1250
|
+
}).nullable(),
|
|
1251
|
+
certain: z.boolean(),
|
|
1252
|
+
afterReset: z.object({
|
|
1253
|
+
life: a,
|
|
1254
|
+
outcome: z.string()
|
|
1255
|
+
}).optional(),
|
|
1256
|
+
each: z.boolean().optional(),
|
|
1257
|
+
propagation: band.optional(),
|
|
1258
|
+
outcomes: z.array(z.string()).optional(),
|
|
1259
|
+
why: z.string()
|
|
1260
|
+
});
|
|
1117
1261
|
const prediction = z.discriminatedUnion("kind", [
|
|
1118
1262
|
z.object({
|
|
1119
1263
|
kind: z.literal("none"),
|
|
1120
1264
|
outcomes: z.array(z.string()).optional(),
|
|
1121
1265
|
why: z.string()
|
|
1122
1266
|
}),
|
|
1123
|
-
|
|
1124
|
-
kind: z.literal("window"),
|
|
1125
|
-
nodes: z.enum(["all", "paused"]),
|
|
1126
|
-
start: z.object({
|
|
1127
|
-
from: reference,
|
|
1128
|
-
band
|
|
1129
|
-
}),
|
|
1130
|
-
end: z.object({
|
|
1131
|
-
from: reference,
|
|
1132
|
-
band
|
|
1133
|
-
}).nullable(),
|
|
1134
|
-
certain: z.boolean(),
|
|
1135
|
-
afterReset: z.object({
|
|
1136
|
-
life: a,
|
|
1137
|
-
outcome: z.string()
|
|
1138
|
-
}).optional(),
|
|
1139
|
-
each: z.boolean().optional(),
|
|
1140
|
-
propagation: band.optional(),
|
|
1141
|
-
outcomes: z.array(z.string()).optional(),
|
|
1142
|
-
why: z.string()
|
|
1143
|
-
}),
|
|
1267
|
+
window,
|
|
1144
1268
|
z.object({
|
|
1145
1269
|
kind: z.literal("removal"),
|
|
1146
1270
|
band,
|
|
@@ -1164,6 +1288,13 @@ const recordOf = (a, main) => {
|
|
|
1164
1288
|
kind: z.literal("stalls"),
|
|
1165
1289
|
why: z.string()
|
|
1166
1290
|
}),
|
|
1291
|
+
z.object({
|
|
1292
|
+
kind: z.literal("ends"),
|
|
1293
|
+
from: reference,
|
|
1294
|
+
band: band.nullable(),
|
|
1295
|
+
window: window.optional(),
|
|
1296
|
+
why: z.string()
|
|
1297
|
+
}),
|
|
1167
1298
|
z.object({
|
|
1168
1299
|
kind: z.literal("measured"),
|
|
1169
1300
|
why: z.string()
|
|
@@ -1180,7 +1311,8 @@ const recordOf = (a, main) => {
|
|
|
1180
1311
|
stack: z.enum(["service", "grpc"]).optional(),
|
|
1181
1312
|
mesh: z.object({
|
|
1182
1313
|
flavor: z.enum(["sidecar", "ambient"]),
|
|
1183
|
-
mode: z.enum(["sidecar", "ambient"])
|
|
1314
|
+
mode: z.enum(["sidecar", "ambient"]),
|
|
1315
|
+
ztunnel: z.record(z.string(), z.string()).optional()
|
|
1184
1316
|
}).optional(),
|
|
1185
1317
|
substitutes: z.string().nullable(),
|
|
1186
1318
|
mode: z.enum(MODES),
|
|
@@ -1241,7 +1373,8 @@ const recordOf = (a, main) => {
|
|
|
1241
1373
|
healUpdates: z.record(z.string(), z.object({
|
|
1242
1374
|
after: a.nullable(),
|
|
1243
1375
|
by: a
|
|
1244
|
-
}).nullable()).optional()
|
|
1376
|
+
}).nullable()).optional(),
|
|
1377
|
+
closes: z.record(z.string(), a.nullable()).optional()
|
|
1245
1378
|
});
|
|
1246
1379
|
return {
|
|
1247
1380
|
cell,
|
|
@@ -1303,6 +1436,6 @@ const readRecord = (data) => {
|
|
|
1303
1436
|
throw new Error(`a kind cell record has the format ${FORMAT} or ${FORMAT_V1}, not ${String(format)}`);
|
|
1304
1437
|
};
|
|
1305
1438
|
//#endregion
|
|
1306
|
-
export {
|
|
1439
|
+
export { sliceEmptyOf as $, medianInterval as A, intakeOf as B, propagationOf as C, unitOf as D, reusesReset as E, byNode as F, recordsOf as G, outcomesOf as H, deletionOf as I, restartsAt as J, requestsOf as K, dropOf as L, quantileOf as M, backendsAt as N, updatesOf as O, byClients as P, sliceDropOf as Q, emptyOf as R, observationEnd as S, referenceOf as T, podsAt as U, kernelCounts as V, podsOf as W, signalOf as X, rulesOf as Y, signalsOf as Z, checkRun as _, calibrationOf as a, ztunnelReplaced as at, judgeKind as b, predictionText as c, LOG_LABEL as ct, FORMAT_V1 as d, SERVER as dt, slicesOf as et, judge as f, bandOf as ft, LAST_SLACK as g, KIND_NOISE as h, showBand as ht, readRecord as i, ztunnelPodsOf as it, middle as j, RELATIVE as k, timeSpread as l, PORT as lt, upgradeCell as m, secondsOf as mt, kindCellRecordSchema as n, stoppingOf as nt, cellsTable as o, CLIENT as ot, upgrade as p, seconds as pt, restartOf as q, kindCellRecordV1Schema as r, transitionOf as rt, medianText as s, HEALTH_PORT as st, STAGES as t, socketsOf as tt, FORMAT as u, REDIS as ut, deletionsOf as v, propagationSource as w, observationAfter as x, factsOf as y, failuresOf as z };
|
|
1307
1440
|
|
|
1308
|
-
//# sourceMappingURL=judge-
|
|
1441
|
+
//# sourceMappingURL=judge-CmiF-Fcs.js.map
|