@mark1russell7/kind 0.11.8 → 0.11.9

package/build/{judge-DnB9wEwC.d.ts → judge-BBHSeu4i.d.ts}

@@ -198,6 +198,24 @@ interface Trigger<A = Quantity> {
    * not ready (S2, a failed prober).
    */
   readonly serves?: boolean | undefined;
+  /**
+   * These flaps are the faults that come again after the first one, in their sequence. Each flap
+   * starts `rest` after the heal before it, with the steps of the inject, and its heal comes `hold`
+   * after its own fault. The first fault keeps the hold of the trigger. The steps of the prepare come
+   * one time, before the first fault. Only a trigger with a hold and a heal can flap. Without flaps,
+   * the fault comes one time.
+   */
+  readonly flaps?: readonly Flap[] | undefined;
+}
+/** A flap is one more fault of a trigger: the rest from the heal before it, and its hold. */
+interface Flap {
+  readonly rest: Quantity;
+  readonly hold: Quantity;
+}
+/** A cycle is the stamps of one fault of a run and of its heal. */
+interface Cycle {
+  readonly fault: number;
+  readonly heal: number;
 }
 /** A kubelet probe is the period, the failure threshold and the timeout of one probe of the pod. */
 interface KubeletProbe {
@@ -303,7 +321,9 @@ interface Band<A = Quantity> {
  * - `intake`: the stop of the intake of the target pod, as its service records it.
  * - `fault`: the commit of the fault, as the injector or the agent stamps it.
  * - `resume`: the end of a pause of kube-proxy, as the agent stamps it.
- * - `heal`: the start of the first act that ends the fault, for example the delete of a policy.
+ * - `heal`: the start of the first act that ends the fault, for example the delete of a policy. With
+ *   flaps (`Trigger.flaps`), it is the heal of the last fault. A window or a removal with `flaps`
+ *   takes the fault and the heal of each cycle instead.
  * - `restart`: the first deletion timestamp of a ztunnel pod in the run, as the watcher receives it.
  */
 type Reference = "deletion" | "intake" | "fault" | "resume" | "heal" | "restart";
@@ -327,7 +347,9 @@ interface ResetReuse<A = Quantity> {
  *   With `each`, each replica of the run that gets a deletion has a window of its own, from its own
  *   references. A request is late only when it is outside each of these windows.
  * - `removal`: the Service has no ready endpoint, and the rules of each node have no backend, in the band.
- *   With `window`, the failed requests also start in that window.
+ *   With `window`, the failed requests also start in that window. With `flaps`, the band is after
+ *   the fault of each cycle. At each later fault, the Service must have a ready endpoint, and
+ *   the rules of each node a backend. Thus the Service came back after the heal before it.
  * - `leaves`: the target replica leaves the EndpointSlice and the rules of each node in the band.
  *   With `window`, the failed requests also start in that window.
  * - `restarts`: each replica receives SIGTERM in the band, or no replica does.
@@ -349,6 +371,7 @@ type Prediction<A = Quantity> = {
   readonly kind: "removal";
   readonly band: Band<A>;
   readonly window?: WindowPrediction<A> | undefined;
+  readonly flaps?: boolean | undefined;
   readonly why: string;
 } | {
   readonly kind: "leaves";
@@ -425,6 +448,13 @@ interface WindowPrediction<A = Quantity> {
    * earlier drop only closes the window earlier.
    */
   readonly propagation?: Band<A> | undefined;
+  /**
+   * This flag gives a window to each cycle of the run (`KindRun.acts.cycles`): each fault and its
+   * heal. The references `fault` and `heal` are then those of that cycle. The window has no
+   * other reference. A failure in a rest between two windows fails the run. With `certain`, each
+   * window has a failed request, and its last one comes near the end of that window.
+   */
+  readonly flaps?: boolean | undefined;
   /** These outcomes are the only failures that the window permits. A failure with another outcome fails the run. Without them, each outcome but `ok` can fail. */
   readonly outcomes?: readonly string[] | undefined;
   readonly why: string;
@@ -456,6 +486,8 @@ interface KindRun {
     readonly fault: number | null;
     readonly heal: number | null;
     readonly resume: number | null;
+    /** These cycles are each fault and its heal, the first one too, for a trigger with flaps. A run without flaps has none. */
+    readonly cycles?: readonly Cycle[] | undefined;
   };
   /** This replica is the target of the trigger, or null for a fault on Redis. */
   readonly target: Replica | null;
@@ -730,6 +762,13 @@ declare const dropOf: (lines: readonly Line[], node: string, backend: string, af
 declare const emptyOf: (lines: readonly Line[], node: string, after: number) => RulesLine | null;
 /** This function gives when an EndpointSlice of the Service first had no ready endpoint, after a time. */
 declare const sliceEmptyOf: (lines: readonly Line[], svc: string, after: number) => number | null;
+/** This function gives when an EndpointSlice of the Service first had a ready endpoint, after a time. */
+declare const sliceReadyOf: (lines: readonly Line[], svc: string, after: number) => number | null;
+/**
+ * This function tells if the latest EndpointSlice of the Service at a time had a ready endpoint. A
+ * slice line comes only at a change, thus with no line before the time, it gives null.
+ */
+declare const sliceReadyAt: (lines: readonly Line[], svc: string, at: number) => boolean | null;
 /** This function gives when the endpoint of an address was first not ready in the EndpointSlice of the Service, after a time. */
 declare const sliceDropOf: (lines: readonly Line[], svc: string, ip: string, after: number) => number | null;
 /** This function gives the latest state of each pod at a time, by its name. A deleted pod is not in it. */
@@ -798,6 +837,15 @@ interface RunFacts<A = Quantity> {
   readonly healUpdates?: Readonly<Record<string, Propagated<A> | null>> | undefined;
   /** These are the first close of the socket of each client after the reference (`ends`), by client, or null for none. */
   readonly closes?: Readonly<Record<string, A | null>> | undefined;
+  /**
+   * These are the times of each cycle of a removal with flaps, in the sequence of the cycles. `empty` is
+   * when the EndpointSlice had no ready endpoint, after the fault of the cycle. `ready` is when it had a
+   * ready endpoint again, after the heal of the cycle. A time that the run does not have is null.
+   */
+  readonly cycles?: ReadonlyArray<{
+    readonly empty: A | null;
+    readonly ready: A | null;
+  }> | undefined;
 }
 /** A deleted replica is a replica of a run that got a deletion. It gives its pod, and the stamps of its intake (or null) and of its deletion. */
 interface Deleted {
@@ -856,6 +904,11 @@ declare const reusesReset: (request: Request, requests: readonly Request[], reus
  * The references `deletion` and `intake` are those of the target, or of the replica `pod`.
  */
 declare const referenceOf: (run: Pick<KindRun, "start" | "acts" | "target" | "lines">, from: Reference, pod?: string | undefined) => number | null;
+/**
+ * This function gives the cycles of a run: each fault and its heal (`KindRun.acts.cycles`). A run
+ * without cycles gives its one fault and heal, or none when it does not have both stamps.
+ */
+declare const cyclesOf: (run: Pick<KindRun, "acts">) => readonly Cycle[];
 /**
  * This function gives the source of the propagation of a cell. In a sidecar mesh, the client sidecar
  * picks the backend of each call from its endpoints (EDS), thus the source is `eds`. Each other cell
@@ -1067,5 +1120,5 @@ declare const quantileOf: (sorted: readonly number[], p: number) => number;
  */
 declare const medianInterval: (sorted: readonly number[]) => [number, number];
 //#endregion
-export { updatesOf as $, Declaration as $t, TriggerV1 as A, nodeNames as An, rulesOf as At, RunCheck as B, ztunnelPodsOf as Bt, KindCellResult as C, NODE_IMAGES as Cn, outcomesOf as Ct, PredictionV1 as D, goDuration as Dn, requestsOf as Dt, LoadSample as E, cidrRange as En, recordsOf as Et, KIND_NOISE as F, slicesOf as Ft, judgeKind as G, PORT as Gt, checkRun as H, CLIENT as Ht, KindDecision as I, socketsOf as It, propagationOf as J, bandOf as Jt, observationAfter as K, REDIS as Kt, KindVerdict as L, stoppingOf as Lt, upgrade as M, scenarioProblems as Mn, signalsOf as Mt, upgradeCell as N, sliceDropOf as Nt, RunCheckV1 as O, kindConfig as On, restartOf as Ot, Deleted as P, sliceEmptyOf as Pt, unitOf as Q, Band as Qt, LAST_SLACK as R, transitionOf as Rt, KindCellRecordV1 as S, KubeProxyMode as Sn, kernelCounts as St, KindVerdictV1 as T, cidrOverlap as Tn, podsOf as Tt, deletionsOf as U, HEALTH_PORT as Ut, RunFacts as V, ztunnelReplaced as Vt, factsOf as W, LOG_LABEL as Wt, referenceOf as X, secondsOf as Xt, propagationSource as Y, seconds as Yt, reusesReset as Z, showBand as Zt, ClusterManifest as _, WindowPrediction as _n, dropOf as _t, STAGES as a, MeshFlavor as an, RecordLine as at, FORMAT_V1 as b, KindConfig as bn, intakeOf as bt, readRecord as c, Prediction as cn, SliceLine as ct, calibrationOf as d, Replica as dn, TunnelLine as dt, EdgeProtocol as en, Endpoint as et, cellsTable as f, ResetReuse as fn, ZtunnelPodLine as ft, BandV1 as g, TriggerKind as gn, deletionOf as gt, timeSpread as h, Trigger as hn, byNode as ht, quantileOf as i, Mesh as in, PropagationSource as it, judge as j, scenario as jn, signalOf as jt, RunFactsV1 as k, kubeProxyPatch as kn, restartsAt as kt, Judged as l, RedisDeclaration as ln, SocketLine as lt, predictionText as m, Template as mn, byClients as mt, medianInterval as n, KindRun as nn, Line as nt, kindCellRecordSchema as o, MeshMode as on, Request as ot, medianText as p, Stack as pn, backendsAt as pt, observationEnd as q, SERVER as qt, middle as r, KubeletProbe as rn, PodLine as rt, kindCellRecordV1Schema as s, PodSettings as sn, RulesLine as st, RELATIVE as t, KindCell as tn, HBONE_PORT as tt, TimeSpread as u, Reference as un, TCP_STATE as ut, EnvironmentManifest as v, Workload as vn, emptyOf as vt, KindCellV1 as w, NodeImage as wn, podsAt as wt, KindCellRecord as x, KindScenario as xn, isTunnelState as xt, FORMAT as y, KUBE_PROXY_MODES as yn, failuresOf as yt, Propagated as z, tunnelsOf as zt };
-//# sourceMappingURL=judge-DnB9wEwC.d.ts.map
+export { unitOf as $, secondsOf as $t, TriggerV1 as A, cidrOverlap as An, restartsAt as At, RunCheck as B, stoppingOf as Bt, KindCellResult as C, Workload as Cn, kernelCounts as Ct, PredictionV1 as D, KubeProxyMode as Dn, recordsOf as Dt, LoadSample as E, KindScenario as En, podsOf as Et, KIND_NOISE as F, nodeNames as Fn, sliceEmptyOf as Ft, factsOf as G, CLIENT as Gt, checkRun as H, tunnelsOf as Ht, KindDecision as I, scenario as In, sliceReadyAt as It, observationEnd as J, PORT as Jt, judgeKind as K, HEALTH_PORT as Kt, KindVerdict as L, scenarioProblems as Ln, sliceReadyOf as Lt, upgrade as M, goDuration as Mn, signalOf as Mt, upgradeCell as N, kindConfig as Nn, signalsOf as Nt, RunCheckV1 as O, NODE_IMAGES as On, requestsOf as Ot, Deleted as P, kubeProxyPatch as Pn, sliceDropOf as Pt, reusesReset as Q, seconds as Qt, LAST_SLACK as R, slicesOf as Rt, KindCellRecordV1 as S, WindowPrediction as Sn, isTunnelState as St, KindVerdictV1 as T, KindConfig as Tn, podsAt as Tt, cyclesOf as U, ztunnelPodsOf as Ut, RunFacts as V, transitionOf as Vt, deletionsOf as W, ztunnelReplaced as Wt, propagationSource as X, SERVER as Xt, propagationOf as Y, REDIS as Yt, referenceOf as Z, bandOf as Zt, ClusterManifest as _, ResetReuse as _n, deletionOf as _t, STAGES as a, Flap as an, PropagationSource as at, FORMAT_V1 as b, Trigger as bn, failuresOf as bt, readRecord as c, KubeletProbe as cn, RulesLine as ct, calibrationOf as d, MeshMode as dn, TCP_STATE as dt, showBand as en, updatesOf as et, cellsTable as f, PodSettings as fn, TunnelLine as ft, BandV1 as g, Replica as gn, byNode as gt, timeSpread as h, Reference as hn, byClients as ht, quantileOf as i, EdgeProtocol as in, PodLine as it, judge as j, cidrRange as jn, rulesOf as jt, RunFactsV1 as k, NodeImage as kn, restartOf as kt, Judged as l, Mesh as ln, SliceLine as lt, predictionText as m, RedisDeclaration as mn, backendsAt as mt, medianInterval as n, Cycle as nn, HBONE_PORT as nt, kindCellRecordSchema as o, KindCell as on, RecordLine as ot, medianText as p, Prediction as pn, ZtunnelPodLine as pt, observationAfter as q, LOG_LABEL as qt, middle as r, Declaration as rn, Line as rt, kindCellRecordV1Schema as s, KindRun as sn, Request as st, RELATIVE as t, Band as tn, Endpoint as tt, TimeSpread as u, MeshFlavor as un, SocketLine as ut, EnvironmentManifest as v, Stack as vn, dropOf as vt, KindCellV1 as w, KUBE_PROXY_MODES as wn, outcomesOf as wt, KindCellRecord as x, TriggerKind as xn, intakeOf as xt, FORMAT as y, Template as yn, emptyOf as yt, Propagated as z, socketsOf as zt };
+//# sourceMappingURL=judge-BBHSeu4i.d.ts.map

package/build/{judge-Cs91dOGO.js → judge-ZoLw3AXV.js}

@@ -121,6 +121,16 @@ const dropOf = (lines, node, backend, after, source = "rules") => rulesOf(lines,
 const emptyOf = (lines, node, after) => rulesOf(lines).find((line) => line.node === node && line.t >= after && line.b.length === 0) ?? null;
 /** This function gives when an EndpointSlice of the Service first had no ready endpoint, after a time. */
 const sliceEmptyOf = (lines, svc, after) => slicesOf(lines, svc).find((slice) => slice.t >= after && !slice.eps.some((endpoint) => endpoint.r === true))?.t ?? null;
+/** This function gives when an EndpointSlice of the Service first had a ready endpoint, after a time. */
+const sliceReadyOf = (lines, svc, after) => slicesOf(lines, svc).find((slice) => slice.t >= after && slice.eps.some((endpoint) => endpoint.r === true))?.t ?? null;
+/**
+* This function tells if the latest EndpointSlice of the Service at a time had a ready endpoint. A
+* slice line comes only at a change, thus with no line before the time, it gives null.
+*/
+const sliceReadyAt = (lines, svc, at) => {
+	const latest = slicesOf(lines, svc).filter((slice) => slice.t <= at).at(-1);
+	return latest === void 0 ? null : latest.eps.some((endpoint) => endpoint.r === true);
+};
 /** This function gives when the endpoint of an address was first not ready in the EndpointSlice of the Service, after a time. */
 const sliceDropOf = (lines, svc, ip, after) => slicesOf(lines, svc).find((slice) => slice.t >= after && !slice.eps.some((endpoint) => endpoint.ip === ip && endpoint.r === true))?.t ?? null;
 /** This function gives the latest state of each pod at a time, by its name. A deleted pod is not in it. */
@@ -264,6 +274,18 @@ const referenceOf = (run, from, pod = run.target?.pod) => {
 	return intakeOf(lines, pod, run.start);
 };
 /**
+* This function gives the cycles of a run: each fault and its heal (`KindRun.acts.cycles`). A run
+* without cycles gives its one fault and heal, or none when it does not have both stamps.
+*/
+const cyclesOf = (run) => {
+	if (run.acts.cycles !== void 0) return run.acts.cycles;
+	const { fault, heal } = run.acts;
+	return fault === null || heal === null ? [] : [{
+		fault,
+		heal
+	}];
+};
+/**
 * This function gives the source of the propagation of a cell. In a sidecar mesh, the client sidecar
 * picks the backend of each call from its endpoints (EDS), thus the source is `eds`. Each other cell
 * goes through the rules of kube-proxy (`rules`).
@@ -398,6 +420,74 @@ const factsOf = (cell, run) => {
 		} : {}
 	};
 };
+/**
+* This function judges a removal with flaps. In each cycle, the Service has no ready endpoint, and the
+* rules of each node have no backend, in the band after its fault. Each time comes before the fault of
+* the next cycle. At the fault of each cycle after the first, the Service has a ready endpoint and the
+* rules of each node a backend. After the last heal, the Service has a ready endpoint again. The main
+* value is the empty Service of the first cycle.
+*/
+const removalOfCycles = (cell, run, prediction, done) => {
+	const lines = run.lines;
+	const cycles = cyclesOf(run);
+	if (cycles.length === 0) return done(false, "the run has no cycle of a fault and its heal", null);
+	const band = secondsOf(prediction.band);
+	const nodes = [...new Set(lines.filter((line) => line.ev === "rules").map((line) => String(line["node"])))].sort();
+	const problems = [];
+	const times = [];
+	let rules = {};
+	cycles.forEach((cycle, k) => {
+		const n = k + 1;
+		const next = cycles[k + 1]?.fault ?? run.end;
+		if (k > 0) {
+			if (sliceReadyAt(lines, "lab", cycle.fault) !== true) problems.push(`the Service had no ready endpoint at the fault of the cycle ${n}`);
+			const backends = backendsAt(lines, cycle.fault);
+			for (const node of nodes) if ((backends.get(node) ?? []).length === 0) problems.push(`the rules of ${node} had no backend at the fault of the cycle ${n}`);
+		}
+		const empty = sliceEmptyOf(lines, "lab", cycle.fault);
+		const value = empty === null || empty > next ? null : sec(empty - cycle.fault);
+		if (value === null) problems.push(`the Service kept a ready endpoint in the cycle ${n}`);
+		else if (!inBand(value, band)) problems.push(`the EndpointSlice had no ready endpoint at ${value} s in the cycle ${n}`);
+		const own = {};
+		for (const node of nodes) {
+			const drop = emptyOf(lines, node, cycle.fault);
+			const at = drop === null || drop.t > next ? null : sec(drop.t - cycle.fault);
+			own[node] = at === null ? null : seconds(at);
+			if (at === null) problems.push(`the rules of ${node} kept a backend in the cycle ${n}`);
+			else if (!inBand(at, {
+				lo: band.lo,
+				hi: band.hi + .05
+			})) problems.push(`the rules of ${node} had no backend at ${at} s in the cycle ${n}`);
+		}
+		if (k === 0) rules = own;
+		const ready = sliceReadyOf(lines, "lab", cycle.heal);
+		const back = ready === null || ready > next ? null : sec(ready - cycle.heal);
+		if (k === cycles.length - 1 && back === null) problems.push("the Service had no ready endpoint after the last heal");
+		times.push({
+			empty: value === null ? null : seconds(value),
+			ready: back === null ? null : seconds(back)
+		});
+	});
+	const first = times[0].empty === null ? null : valueIn(times[0].empty, "s");
+	const inner = prediction.window === void 0 ? null : checkRun({
+		...cell,
+		prediction: prediction.window
+	}, run);
+	if (inner !== null && !inner.ok) problems.push(inner.why);
+	const empties = times.map((each) => each.empty === null ? "none" : `${valueIn(each.empty, "s")} s`).join(", ");
+	const agrees = `the Service was empty in each of ${cycles.length} cycles, at ${empties} after its fault${inner === null ? "" : `; ${inner.why}`}`;
+	const more = {
+		...inner === null ? {} : inner.facts,
+		...first === null ? {} : { sliceEmpty: seconds(first) },
+		rules,
+		cycles: times
+	};
+	const checked = done(problems.length === 0, problems.length === 0 ? agrees : problems.join("; "), first, more);
+	return inner?.tail === void 0 ? checked : {
+		...checked,
+		tail: inner.tail
+	};
+};
 /** This function judges one run against the prediction of its cell. */
 const checkRun = (cell, run) => {
 	const lines = run.lines;
@@ -434,10 +524,23 @@ const checkRun = (cell, run) => {
 			};
 			const pods = each ? (facts.deletions ?? []).map((deleted) => deleted.pod) : [run.target?.pod];
 			if (pods.length === 0) return done(false, "no replica of the run got a deletion", null);
+			const flaps = prediction.flaps === true;
+			const cyclic = (from) => from === "fault" || from === "heal";
+			if (flaps && (each || !cyclic(prediction.start.from) || prediction.end !== null && !cyclic(prediction.end.from))) return done(false, "a window with flaps has no each, and its references are the fault and the heal", null);
+			const cycles = flaps ? cyclesOf(run) : [];
+			if (flaps && cycles.length === 0) return done(false, "the run has no cycle of a fault and its heal", null);
+			const sources = flaps ? cycles.map((cycle) => ({
+				pod: run.target?.pod,
+				cycle
+			})) : pods.map((pod) => ({
+				pod,
+				cycle: void 0
+			}));
 			const windows = [];
-			for (const pod of pods) {
-				const start = referenceOf(run, prediction.start.from, pod);
-				const end = prediction.end === null ? null : referenceOf(run, prediction.end.from, pod);
+			for (const { pod, cycle } of sources) {
+				const at = (from) => cycle !== void 0 && cyclic(from) ? cycle[from] : referenceOf(run, from, pod);
+				const start = at(prediction.start.from);
+				const end = prediction.end === null ? null : at(prediction.end.from);
 				if (start === null || prediction.end !== null && end === null) return done(false, `the run has no ${start === null ? prediction.start.from : prediction.end.from} reference${each ? ` for ${pod}` : ""}`, null);
 				const origin = each ? referenceOf(run, "deletion", pod) : facts.deletion ?? (cell.trigger.kind === "crash" ? run.acts.fault : null);
 				const ip = run.replicas.find((replica) => replica.pod === pod)?.ip;
@@ -504,19 +607,27 @@ const checkRun = (cell, run) => {
 			const wrong = allowed === void 0 ? [] : relevant.filter((request) => !late.includes(request) && !allowed.includes(request.o));
 			const bounds = (window) => `[${sec(window.lo - window.start)}, ${sec(window.hi - window.start)}] s`;
 			const problems = [];
-			if (outside.length > 0) problems.push(each ? `${outside.length} failed requests outside the window of each deleted replica: ${windows.map((window) => `${window.pod} ${bounds(window)}`).join(", ")} after its ${prediction.start.from}` : `${outside.length} failed requests outside ${bounds(windows[0])} after ${prediction.start.from}`);
+			if (outside.length > 0) problems.push(each ? `${outside.length} failed requests outside the window of each deleted replica: ${windows.map((window) => `${window.pod} ${bounds(window)}`).join(", ")} after its ${prediction.start.from}` : flaps ? `${outside.length} failed requests outside the window of each cycle: ${windows.map((window, k) => `${k + 1} ${bounds(window)}`).join(", ")} after its ${prediction.start.from}` : `${outside.length} failed requests outside ${bounds(windows[0])} after ${prediction.start.from}`);
 			if (others.length > 0) problems.push(`${others.length} failed requests on nodes whose kube-proxy did not pause`);
 			if (wrong.length > 0) problems.push(`${wrong.length} failed requests in the window ended ${JSON.stringify(outcomesOf(wrong))}, not ${allowed.join(", ")}`);
 			if (prediction.certain && relevant.length === 0) problems.push("no request failed in a window that the law opens");
 			const final = windows.reduce((a, b) => (b.end ?? Number.NEGATIVE_INFINITY) > (a.end ?? Number.NEGATIVE_INFINITY) ? b : a);
-			if (prediction.certain && final.end !== null && relevant.length > 0) {
+			if (flaps && prediction.certain && relevant.length > 0) windows.forEach((window, k) => {
+				const own = relevant.filter((request) => request.t >= window.lo && request.t <= window.hi);
+				if (own.length === 0) problems.push(`no request failed in the window of the cycle ${k + 1}`);
+				else if (window.end !== null) {
+					const earliest = window.end + closes.lo * 1e3 - NOISE - SLACK;
+					if (own.at(-1).t < earliest) problems.push(`the last failed request of the cycle ${k + 1} started ${sec(earliest - own.at(-1).t)} s before its window can end`);
+				}
+			});
+			if (!flaps && prediction.certain && final.end !== null && relevant.length > 0) {
 				const last = relevant.at(-1).t;
 				const earliest = final.end + closes.lo * 1e3 - NOISE - SLACK;
 				if (last < earliest) problems.push(`the last failed request started ${sec(earliest - last)} s before the window can end`);
 			}
 			const inside = relevant.filter((request) => !late.includes(request));
 			const span = inside.length === 0 ? 0 : sec(inside.at(-1).t - inside[0].t);
-			const where = each ? `the windows of ${windows.length} deleted replicas` : "the window";
+			const where = each ? `the windows of ${windows.length} deleted replicas` : flaps ? `the windows of ${windows.length} cycles` : "the window";
 			const agrees = reused.length === 0 ? `${relevant.length} failed requests, all in ${where} (span ${span} s)` : `${inside.length} failed requests in ${where} (span ${span} s), and ${reused.length} on a port that a reset closed`;
 			const checked = done(problems.length === 0, problems.length === 0 ? agrees : problems.join("; "), relevant.length);
 			if (problems.length > 0) return checked;
@@ -546,6 +657,7 @@ const checkRun = (cell, run) => {
 		case "removal": {
 			const fault = run.acts.fault;
 			if (fault === null) return done(false, "the run has no fault stamp", null);
+			if (prediction.flaps === true) return removalOfCycles(cell, run, prediction, done);
 			const empty = sliceEmptyOf(lines, "lab", fault);
 			if (empty === null) return done(false, "the Service kept a ready endpoint", null);
 			const value = sec(empty - fault);
@@ -992,7 +1104,7 @@ const upgradeFacts = (facts) => {
 		by: seconds(old.by)
 	};
 	const byKey = (record) => Object.fromEntries(Object.entries(record).map(([key, old]) => [key, propagated(old)]));
-	const { sliceEmpty, rules, left, signals, updates, healUpdates, closes, ...rest } = facts;
+	const { sliceEmpty, rules, left, signals, updates, healUpdates, closes, cycles, ...rest } = facts;
 	return {
 		...rest,
 		intakeAfterDeletion: time$1(facts.intakeAfterDeletion),
@@ -1004,7 +1116,11 @@ const upgradeFacts = (facts) => {
 		...rules === void 0 ? {} : { rules: Object.fromEntries(Object.entries(rules).map(([node, old]) => [node, time$1(old)])) },
 		...left === void 0 ? {} : { left: seconds(left) },
 		...signals === void 0 ? {} : { signals: signals.map((old) => time$1(old)) },
-		...closes === void 0 ? {} : { closes: Object.fromEntries(Object.entries(closes).map(([client, old]) => [client, time$1(old)])) }
+		...closes === void 0 ? {} : { closes: Object.fromEntries(Object.entries(closes).map(([client, old]) => [client, time$1(old)])) },
+		...cycles === void 0 ? {} : { cycles: cycles.map((old) => ({
+			empty: time$1(old.empty),
+			ready: time$1(old.ready)
+		})) }
 	};
 };
 /**
@@ -1074,8 +1190,8 @@ const atBand = (from, band) => {
 const predictionText = (prediction) => {
 	switch (prediction.kind) {
 		case "none": return prediction.outcomes === void 0 ? "no failure" : `no ${prediction.outcomes.join(", ")}`;
-		case "window": return `failures only from ${atBand(prediction.start.from, prediction.start.band)}${prediction.end === null ? "" : ` to ${atBand(prediction.end.from, prediction.end.band)}`}${prediction.each === true ? ", of each deleted replica" : ""}${prediction.nodes === "paused" ? ", on the paused node" : ""}${prediction.certain ? ", at least one" : ""}${prediction.outcomes === void 0 ? "" : `, each one ${prediction.outcomes.join(" or ")}`}${prediction.afterReset === void 0 ? "" : `, or a ${prediction.afterReset.outcome} up to ${format(prediction.afterReset.life)} after a reset on its port`}${prediction.propagation === void 0 ? "" : `, the rules drop the replica in ${showBand(prediction.propagation)}`}`;
-		case "removal": return `Service empty in ${showBand(prediction.band)}${prediction.window === void 0 ? "" : `; ${predictionText(prediction.window)}`}`;
+		case "window": return `failures only from ${atBand(prediction.start.from, prediction.start.band)}${prediction.end === null ? "" : ` to ${atBand(prediction.end.from, prediction.end.band)}`}${prediction.each === true ? ", of each deleted replica" : ""}${prediction.flaps === true ? ", in each cycle" : ""}${prediction.nodes === "paused" ? ", on the paused node" : ""}${prediction.certain ? ", at least one" : ""}${prediction.outcomes === void 0 ? "" : `, each one ${prediction.outcomes.join(" or ")}`}${prediction.afterReset === void 0 ? "" : `, or a ${prediction.afterReset.outcome} up to ${format(prediction.afterReset.life)} after a reset on its port`}${prediction.propagation === void 0 ? "" : `, the rules drop the replica in ${showBand(prediction.propagation)}`}`;
+		case "removal": return `Service empty in ${showBand(prediction.band)}${prediction.flaps === true ? " after each fault, and ready again before the next" : ""}${prediction.window === void 0 ? "" : `; ${predictionText(prediction.window)}`}`;
 		case "leaves": return `the replica leaves in ${showBand(prediction.band)}${prediction.window === void 0 ? "" : `; ${predictionText(prediction.window)}`}`;
 		case "restarts": return prediction.band === null ? "no restart" : `each replica SIGTERM in ${showBand(prediction.band)}`;
 		case "stays": return "stays ready, no refusal";
@@ -1107,7 +1223,7 @@ const cellsTable = (judged) => [
 	...judged.map((each) => {
 		const { cell, verdict } = each;
 		const runs = verdict.voided === 0 ? `${verdict.counted}` : `${verdict.counted} (+${verdict.voided} void)`;
-		return `| ${cell.name} | ${cell.id} | ${cell.mode} | ${cell.trigger.kind} | ${predictionText(cell.prediction)} | ${verdict.decision} | ${runs} | ${medianText(each)} |`;
+		return `| ${cell.name} | ${cell.id} | ${cell.mode} | ${cell.trigger.kind}${cell.trigger.flaps === void 0 || cell.trigger.flaps.length === 0 ? "" : ` (${cell.trigger.flaps.length + 1} faults)`} | ${predictionText(cell.prediction)} | ${verdict.decision} | ${runs} | ${medianText(each)} |`;
 	})
 ].join("\n");
 /**
@@ -1286,7 +1402,11 @@ const run = z.object({
 	acts: z.object({
 		fault: z.number().nullable(),
 		heal: z.number().nullable(),
-		resume: z.number().nullable()
+		resume: z.number().nullable(),
+		cycles: z.array(z.object({
+			fault: z.number(),
+			heal: z.number()
+		})).optional()
 	}),
 	target: replica.nullable(),
 	paused: z.string().nullable(),
@@ -1360,6 +1480,7 @@ const recordOf = (a, main) => {
 		}).optional(),
 		each: z.boolean().optional(),
 		propagation: band.optional(),
+		flaps: z.boolean().optional(),
 		outcomes: z.array(z.string()).optional(),
 		why: z.string()
 	});
@@ -1374,6 +1495,7 @@ const recordOf = (a, main) => {
 			kind: z.literal("removal"),
 			band,
 			window: window.optional(),
+			flaps: z.boolean().optional(),
 			why: z.string()
 		}),
 		z.object({
@@ -1440,7 +1562,11 @@ const recordOf = (a, main) => {
 			quiet: z.boolean().optional(),
 			close: z.boolean().optional(),
 			partition: z.boolean().optional(),
-			serves: z.boolean().optional()
+			serves: z.boolean().optional(),
+			flaps: z.array(z.object({
+				rest: time,
+				hold: time
+			})).optional()
 		}),
 		pod: z.object({
 			replicas: count,
@@ -1494,7 +1620,11 @@ const recordOf = (a, main) => {
 			after: a.nullable(),
 			by: a
 		}).nullable()).optional(),
-		closes: z.record(z.string(), a.nullable()).optional()
+		closes: z.record(z.string(), a.nullable()).optional(),
+		cycles: z.array(z.object({
+			empty: a.nullable(),
+			ready: a.nullable()
+		})).optional()
 	});
 	return {
 		cell,
@@ -1556,6 +1686,6 @@ const readRecord = (data) => {
 	throw new Error(`a kind cell record has the format ${FORMAT} or ${FORMAT_V1}, not ${String(format)}`);
 };
 //#endregion
-export { signalOf as $, medianInterval as A, emptyOf as B, propagationOf as C, unitOf as D, reusesReset as E, backendsAt as F, outcomesOf as G, intakeOf as H, byClients as I, recordsOf as J, podsAt as K, byNode as L, quantileOf as M, HBONE_PORT as N, updatesOf as O, TCP_STATE as P, rulesOf as Q, deletionOf as R, observationEnd as S, referenceOf as T, isTunnelState as U, failuresOf as V, kernelCounts as W, restartOf as X, requestsOf as Y, restartsAt as Z, checkRun as _, seconds as _t, calibrationOf as a, stoppingOf as at, judgeKind as b, predictionText as c, ztunnelPodsOf as ct, FORMAT_V1 as d, HEALTH_PORT as dt, signalsOf as et, judge as f, LOG_LABEL as ft, LAST_SLACK as g, bandOf as gt, KIND_NOISE as h, SERVER as ht, readRecord as i, socketsOf as it, middle as j, RELATIVE as k, timeSpread as l, ztunnelReplaced as lt, upgradeCell as m, REDIS as mt, kindCellRecordSchema as n, sliceEmptyOf as nt, cellsTable as o, transitionOf as ot, upgrade as p, PORT as pt, podsOf as q, kindCellRecordV1Schema as r, slicesOf as rt, medianText as s, tunnelsOf as st, STAGES as t, sliceDropOf as tt, FORMAT as u, CLIENT as ut, deletionsOf as v, secondsOf as vt, propagationSource as w, observationAfter as x, factsOf as y, showBand as yt, dropOf as z };
+export { rulesOf as $, RELATIVE as A, dropOf as B, observationEnd as C, reusesReset as D, referenceOf as E, TCP_STATE as F, kernelCounts as G, failuresOf as H, backendsAt as I, podsOf as J, outcomesOf as K, byClients as L, middle as M, quantileOf as N, unitOf as O, HBONE_PORT as P, restartsAt as Q, byNode as R, observationAfter as S, showBand as St, propagationSource as T, intakeOf as U, emptyOf as V, isTunnelState as W, requestsOf as X, recordsOf as Y, restartOf as Z, checkRun as _, REDIS as _t, calibrationOf as a, sliceReadyOf as at, factsOf as b, seconds as bt, predictionText as c, stoppingOf as ct, FORMAT_V1 as d, ztunnelPodsOf as dt, signalOf as et, judge as f, ztunnelReplaced as ft, LAST_SLACK as g, PORT as gt, KIND_NOISE as h, LOG_LABEL as ht, readRecord as i, sliceReadyAt as it, medianInterval as j, updatesOf as k, timeSpread as l, transitionOf as lt, upgradeCell as m, HEALTH_PORT as mt, kindCellRecordSchema as n, sliceDropOf as nt, cellsTable as o, slicesOf as ot, upgrade as p, CLIENT as pt, podsAt as q, kindCellRecordV1Schema as r, sliceEmptyOf as rt, medianText as s, socketsOf as st, STAGES as t, signalsOf as tt, FORMAT as u, tunnelsOf as ut, cyclesOf as v, SERVER as vt, propagationOf as w, judgeKind as x, secondsOf as xt, deletionsOf as y, bandOf as yt, deletionOf as z };
 
-//# sourceMappingURL=judge-Cs91dOGO.js.map
+//# sourceMappingURL=judge-ZoLw3AXV.js.map