@mark1russell7/kind 0.11.8 → 0.11.10

package/build/{judge-Cs91dOGO.js → judge-BIXZEX__.js}

@@ -121,6 +121,16 @@ const dropOf = (lines, node, backend, after, source = "rules") => rulesOf(lines,
 const emptyOf = (lines, node, after) => rulesOf(lines).find((line) => line.node === node && line.t >= after && line.b.length === 0) ?? null;
 /** This function gives when an EndpointSlice of the Service first had no ready endpoint, after a time. */
 const sliceEmptyOf = (lines, svc, after) => slicesOf(lines, svc).find((slice) => slice.t >= after && !slice.eps.some((endpoint) => endpoint.r === true))?.t ?? null;
+/** This function gives when an EndpointSlice of the Service first had a ready endpoint, after a time. */
+const sliceReadyOf = (lines, svc, after) => slicesOf(lines, svc).find((slice) => slice.t >= after && slice.eps.some((endpoint) => endpoint.r === true))?.t ?? null;
+/**
+* This function tells if the latest EndpointSlice of the Service at a time had a ready endpoint. A
+* slice line comes only at a change, thus with no line before the time, it gives null.
+*/
+const sliceReadyAt = (lines, svc, at) => {
+	const latest = slicesOf(lines, svc).filter((slice) => slice.t <= at).at(-1);
+	return latest === void 0 ? null : latest.eps.some((endpoint) => endpoint.r === true);
+};
 /** This function gives when the endpoint of an address was first not ready in the EndpointSlice of the Service, after a time. */
 const sliceDropOf = (lines, svc, ip, after) => slicesOf(lines, svc).find((slice) => slice.t >= after && !slice.eps.some((endpoint) => endpoint.ip === ip && endpoint.r === true))?.t ?? null;
 /** This function gives the latest state of each pod at a time, by its name. A deleted pod is not in it. */
@@ -264,6 +274,18 @@ const referenceOf = (run, from, pod = run.target?.pod) => {
 	return intakeOf(lines, pod, run.start);
 };
 /**
+* This function gives the cycles of a run: each fault and its heal (`KindRun.acts.cycles`). A run
+* without cycles gives its one fault and heal, or none when it does not have both stamps.
+*/
+const cyclesOf = (run) => {
+	if (run.acts.cycles !== void 0) return run.acts.cycles;
+	const { fault, heal } = run.acts;
+	return fault === null || heal === null ? [] : [{
+		fault,
+		heal
+	}];
+};
+/**
 * This function gives the source of the propagation of a cell. In a sidecar mesh, the client sidecar
 * picks the backend of each call from its endpoints (EDS), thus the source is `eds`. Each other cell
 * goes through the rules of kube-proxy (`rules`).
@@ -398,6 +420,74 @@ const factsOf = (cell, run) => {
 		} : {}
 	};
 };
+/**
+* This function judges a removal with flaps. In each cycle, the Service has no ready endpoint, and the
+* rules of each node have no backend, in the band after its fault. Each time comes before the fault of
+* the next cycle. At the fault of each cycle after the first, the Service has a ready endpoint and the
+* rules of each node a backend. After the last heal, the Service has a ready endpoint again. The main
+* value is the empty Service of the first cycle.
+*/
+const removalOfCycles = (cell, run, prediction, done) => {
+	const lines = run.lines;
+	const cycles = cyclesOf(run);
+	if (cycles.length === 0) return done(false, "the run has no cycle of a fault and its heal", null);
+	const band = secondsOf(prediction.band);
+	const nodes = [...new Set(lines.filter((line) => line.ev === "rules").map((line) => String(line["node"])))].sort();
+	const problems = [];
+	const times = [];
+	let rules = {};
+	cycles.forEach((cycle, k) => {
+		const n = k + 1;
+		const next = cycles[k + 1]?.fault ?? run.end;
+		if (k > 0) {
+			if (sliceReadyAt(lines, "lab", cycle.fault) !== true) problems.push(`the Service had no ready endpoint at the fault of the cycle ${n}`);
+			const backends = backendsAt(lines, cycle.fault);
+			for (const node of nodes) if ((backends.get(node) ?? []).length === 0) problems.push(`the rules of ${node} had no backend at the fault of the cycle ${n}`);
+		}
+		const empty = sliceEmptyOf(lines, "lab", cycle.fault);
+		const value = empty === null || empty > next ? null : sec(empty - cycle.fault);
+		if (value === null) problems.push(`the Service kept a ready endpoint in the cycle ${n}`);
+		else if (!inBand(value, band)) problems.push(`the EndpointSlice had no ready endpoint at ${value} s in the cycle ${n}`);
+		const own = {};
+		for (const node of nodes) {
+			const drop = emptyOf(lines, node, cycle.fault);
+			const at = drop === null || drop.t > next ? null : sec(drop.t - cycle.fault);
+			own[node] = at === null ? null : seconds(at);
+			if (at === null) problems.push(`the rules of ${node} kept a backend in the cycle ${n}`);
+			else if (!inBand(at, {
+				lo: band.lo,
+				hi: band.hi + .05
+			})) problems.push(`the rules of ${node} had no backend at ${at} s in the cycle ${n}`);
+		}
+		if (k === 0) rules = own;
+		const ready = sliceReadyOf(lines, "lab", cycle.heal);
+		const back = ready === null || ready > next ? null : sec(ready - cycle.heal);
+		if (k === cycles.length - 1 && back === null) problems.push("the Service had no ready endpoint after the last heal");
+		times.push({
+			empty: value === null ? null : seconds(value),
+			ready: back === null ? null : seconds(back)
+		});
+	});
+	const first = times[0].empty === null ? null : valueIn(times[0].empty, "s");
+	const inner = prediction.window === void 0 ? null : checkRun({
+		...cell,
+		prediction: prediction.window
+	}, run);
+	if (inner !== null && !inner.ok) problems.push(inner.why);
+	const empties = times.map((each) => each.empty === null ? "none" : `${valueIn(each.empty, "s")} s`).join(", ");
+	const agrees = `the Service was empty in each of ${cycles.length} cycles, at ${empties} after its fault${inner === null ? "" : `; ${inner.why}`}`;
+	const more = {
+		...inner === null ? {} : inner.facts,
+		...first === null ? {} : { sliceEmpty: seconds(first) },
+		rules,
+		cycles: times
+	};
+	const checked = done(problems.length === 0, problems.length === 0 ? agrees : problems.join("; "), first, more);
+	return inner?.tail === void 0 ? checked : {
+		...checked,
+		tail: inner.tail
+	};
+};
 /** This function judges one run against the prediction of its cell. */
 const checkRun = (cell, run) => {
 	const lines = run.lines;
@@ -434,10 +524,23 @@ const checkRun = (cell, run) => {
 			};
 			const pods = each ? (facts.deletions ?? []).map((deleted) => deleted.pod) : [run.target?.pod];
 			if (pods.length === 0) return done(false, "no replica of the run got a deletion", null);
+			const flaps = prediction.flaps === true;
+			const cyclic = (from) => from === "fault" || from === "heal";
+			if (flaps && (each || !cyclic(prediction.start.from) || prediction.end !== null && !cyclic(prediction.end.from))) return done(false, "a window with flaps has no each, and its references are the fault and the heal", null);
+			const cycles = flaps ? cyclesOf(run) : [];
+			if (flaps && cycles.length === 0) return done(false, "the run has no cycle of a fault and its heal", null);
+			const sources = flaps ? cycles.map((cycle) => ({
+				pod: run.target?.pod,
+				cycle
+			})) : pods.map((pod) => ({
+				pod,
+				cycle: void 0
+			}));
 			const windows = [];
-			for (const pod of pods) {
-				const start = referenceOf(run, prediction.start.from, pod);
-				const end = prediction.end === null ? null : referenceOf(run, prediction.end.from, pod);
+			for (const { pod, cycle } of sources) {
+				const at = (from) => cycle !== void 0 && cyclic(from) ? cycle[from] : referenceOf(run, from, pod);
+				const start = at(prediction.start.from);
+				const end = prediction.end === null ? null : at(prediction.end.from);
 				if (start === null || prediction.end !== null && end === null) return done(false, `the run has no ${start === null ? prediction.start.from : prediction.end.from} reference${each ? ` for ${pod}` : ""}`, null);
 				const origin = each ? referenceOf(run, "deletion", pod) : facts.deletion ?? (cell.trigger.kind === "crash" ? run.acts.fault : null);
 				const ip = run.replicas.find((replica) => replica.pod === pod)?.ip;
@@ -504,19 +607,27 @@ const checkRun = (cell, run) => {
 			const wrong = allowed === void 0 ? [] : relevant.filter((request) => !late.includes(request) && !allowed.includes(request.o));
 			const bounds = (window) => `[${sec(window.lo - window.start)}, ${sec(window.hi - window.start)}] s`;
 			const problems = [];
-			if (outside.length > 0) problems.push(each ? `${outside.length} failed requests outside the window of each deleted replica: ${windows.map((window) => `${window.pod} ${bounds(window)}`).join(", ")} after its ${prediction.start.from}` : `${outside.length} failed requests outside ${bounds(windows[0])} after ${prediction.start.from}`);
+			if (outside.length > 0) problems.push(each ? `${outside.length} failed requests outside the window of each deleted replica: ${windows.map((window) => `${window.pod} ${bounds(window)}`).join(", ")} after its ${prediction.start.from}` : flaps ? `${outside.length} failed requests outside the window of each cycle: ${windows.map((window, k) => `${k + 1} ${bounds(window)}`).join(", ")} after its ${prediction.start.from}` : `${outside.length} failed requests outside ${bounds(windows[0])} after ${prediction.start.from}`);
 			if (others.length > 0) problems.push(`${others.length} failed requests on nodes whose kube-proxy did not pause`);
 			if (wrong.length > 0) problems.push(`${wrong.length} failed requests in the window ended ${JSON.stringify(outcomesOf(wrong))}, not ${allowed.join(", ")}`);
 			if (prediction.certain && relevant.length === 0) problems.push("no request failed in a window that the law opens");
 			const final = windows.reduce((a, b) => (b.end ?? Number.NEGATIVE_INFINITY) > (a.end ?? Number.NEGATIVE_INFINITY) ? b : a);
-			if (prediction.certain && final.end !== null && relevant.length > 0) {
+			if (flaps && prediction.certain && relevant.length > 0) windows.forEach((window, k) => {
+				const own = relevant.filter((request) => request.t >= window.lo && request.t <= window.hi);
+				if (own.length === 0) problems.push(`no request failed in the window of the cycle ${k + 1}`);
+				else if (window.end !== null) {
+					const earliest = window.end + closes.lo * 1e3 - NOISE - SLACK;
+					if (own.at(-1).t < earliest) problems.push(`the last failed request of the cycle ${k + 1} started ${sec(earliest - own.at(-1).t)} s before its window can end`);
+				}
+			});
+			if (!flaps && prediction.certain && final.end !== null && relevant.length > 0) {
 				const last = relevant.at(-1).t;
 				const earliest = final.end + closes.lo * 1e3 - NOISE - SLACK;
 				if (last < earliest) problems.push(`the last failed request started ${sec(earliest - last)} s before the window can end`);
 			}
 			const inside = relevant.filter((request) => !late.includes(request));
 			const span = inside.length === 0 ? 0 : sec(inside.at(-1).t - inside[0].t);
-			const where = each ? `the windows of ${windows.length} deleted replicas` : "the window";
+			const where = each ? `the windows of ${windows.length} deleted replicas` : flaps ? `the windows of ${windows.length} cycles` : "the window";
 			const agrees = reused.length === 0 ? `${relevant.length} failed requests, all in ${where} (span ${span} s)` : `${inside.length} failed requests in ${where} (span ${span} s), and ${reused.length} on a port that a reset closed`;
 			const checked = done(problems.length === 0, problems.length === 0 ? agrees : problems.join("; "), relevant.length);
 			if (problems.length > 0) return checked;
@@ -546,6 +657,7 @@ const checkRun = (cell, run) => {
 		case "removal": {
 			const fault = run.acts.fault;
 			if (fault === null) return done(false, "the run has no fault stamp", null);
+			if (prediction.flaps === true) return removalOfCycles(cell, run, prediction, done);
 			const empty = sliceEmptyOf(lines, "lab", fault);
 			if (empty === null) return done(false, "the Service kept a ready endpoint", null);
 			const value = sec(empty - fault);
@@ -589,7 +701,9 @@ const checkRun = (cell, run) => {
 			const problems = [];
 			if (!inBand(value, band)) problems.push(`${run.target.pod} left the EndpointSlice at ${value} s`);
 			const propagation = propagationOf(run, fault, void 0, propagationSource(cell));
+			const lost = cell.trigger.kind === "zone-loss" ? run.paused : null;
 			for (const [node, drop] of Object.entries(propagation)) {
+				if (node === lost) continue;
 				const by = drop === null ? null : valueIn(drop.by, "s");
 				if (by === null) problems.push(`the rules of ${node} kept ${run.target.pod}`);
 				else if (!inBand(by, {
@@ -992,7 +1106,7 @@ const upgradeFacts = (facts) => {
 		by: seconds(old.by)
 	};
 	const byKey = (record) => Object.fromEntries(Object.entries(record).map(([key, old]) => [key, propagated(old)]));
-	const { sliceEmpty, rules, left, signals, updates, healUpdates, closes, ...rest } = facts;
+	const { sliceEmpty, rules, left, signals, updates, healUpdates, closes, cycles, ...rest } = facts;
 	return {
 		...rest,
 		intakeAfterDeletion: time$1(facts.intakeAfterDeletion),
@@ -1004,7 +1118,11 @@ const upgradeFacts = (facts) => {
 		...rules === void 0 ? {} : { rules: Object.fromEntries(Object.entries(rules).map(([node, old]) => [node, time$1(old)])) },
 		...left === void 0 ? {} : { left: seconds(left) },
 		...signals === void 0 ? {} : { signals: signals.map((old) => time$1(old)) },
-		...closes === void 0 ? {} : { closes: Object.fromEntries(Object.entries(closes).map(([client, old]) => [client, time$1(old)])) }
+		...closes === void 0 ? {} : { closes: Object.fromEntries(Object.entries(closes).map(([client, old]) => [client, time$1(old)])) },
+		...cycles === void 0 ? {} : { cycles: cycles.map((old) => ({
+			empty: time$1(old.empty),
+			ready: time$1(old.ready)
+		})) }
 	};
 };
 /**
@@ -1074,8 +1192,8 @@ const atBand = (from, band) => {
 const predictionText = (prediction) => {
 	switch (prediction.kind) {
 		case "none": return prediction.outcomes === void 0 ? "no failure" : `no ${prediction.outcomes.join(", ")}`;
-		case "window": return `failures only from ${atBand(prediction.start.from, prediction.start.band)}${prediction.end === null ? "" : ` to ${atBand(prediction.end.from, prediction.end.band)}`}${prediction.each === true ? ", of each deleted replica" : ""}${prediction.nodes === "paused" ? ", on the paused node" : ""}${prediction.certain ? ", at least one" : ""}${prediction.outcomes === void 0 ? "" : `, each one ${prediction.outcomes.join(" or ")}`}${prediction.afterReset === void 0 ? "" : `, or a ${prediction.afterReset.outcome} up to ${format(prediction.afterReset.life)} after a reset on its port`}${prediction.propagation === void 0 ? "" : `, the rules drop the replica in ${showBand(prediction.propagation)}`}`;
-		case "removal": return `Service empty in ${showBand(prediction.band)}${prediction.window === void 0 ? "" : `; ${predictionText(prediction.window)}`}`;
+		case "window": return `failures only from ${atBand(prediction.start.from, prediction.start.band)}${prediction.end === null ? "" : ` to ${atBand(prediction.end.from, prediction.end.band)}`}${prediction.each === true ? ", of each deleted replica" : ""}${prediction.flaps === true ? ", in each cycle" : ""}${prediction.nodes === "paused" ? ", on the paused node" : ""}${prediction.certain ? ", at least one" : ""}${prediction.outcomes === void 0 ? "" : `, each one ${prediction.outcomes.join(" or ")}`}${prediction.afterReset === void 0 ? "" : `, or a ${prediction.afterReset.outcome} up to ${format(prediction.afterReset.life)} after a reset on its port`}${prediction.propagation === void 0 ? "" : `, the rules drop the replica in ${showBand(prediction.propagation)}`}`;
+		case "removal": return `Service empty in ${showBand(prediction.band)}${prediction.flaps === true ? " after each fault, and ready again before the next" : ""}${prediction.window === void 0 ? "" : `; ${predictionText(prediction.window)}`}`;
 		case "leaves": return `the replica leaves in ${showBand(prediction.band)}${prediction.window === void 0 ? "" : `; ${predictionText(prediction.window)}`}`;
 		case "restarts": return prediction.band === null ? "no restart" : `each replica SIGTERM in ${showBand(prediction.band)}`;
 		case "stays": return "stays ready, no refusal";
@@ -1107,7 +1225,7 @@ const cellsTable = (judged) => [
 	...judged.map((each) => {
 		const { cell, verdict } = each;
 		const runs = verdict.voided === 0 ? `${verdict.counted}` : `${verdict.counted} (+${verdict.voided} void)`;
-		return `| ${cell.name} | ${cell.id} | ${cell.mode} | ${cell.trigger.kind} | ${predictionText(cell.prediction)} | ${verdict.decision} | ${runs} | ${medianText(each)} |`;
+		return `| ${cell.name} | ${cell.id} | ${cell.mode} | ${cell.trigger.kind}${cell.trigger.flaps === void 0 || cell.trigger.flaps.length === 0 ? "" : ` (${cell.trigger.flaps.length + 1} faults)`} | ${predictionText(cell.prediction)} | ${verdict.decision} | ${runs} | ${medianText(each)} |`;
 	})
 ].join("\n");
 /**
@@ -1189,7 +1307,8 @@ const TRIGGERS = [
 	"idle",
 	"ztunnel-restart",
 	"certificate-expiry",
-	"cannot-serve"
+	"cannot-serve",
+	"zone-loss"
 ];
 const MODES = [
 	"iptables",
@@ -1286,7 +1405,11 @@ const run = z.object({
 	acts: z.object({
 		fault: z.number().nullable(),
 		heal: z.number().nullable(),
-		resume: z.number().nullable()
+		resume: z.number().nullable(),
+		cycles: z.array(z.object({
+			fault: z.number(),
+			heal: z.number()
+		})).optional()
 	}),
 	target: replica.nullable(),
 	paused: z.string().nullable(),
@@ -1360,6 +1483,7 @@ const recordOf = (a, main) => {
 		}).optional(),
 		each: z.boolean().optional(),
 		propagation: band.optional(),
+		flaps: z.boolean().optional(),
 		outcomes: z.array(z.string()).optional(),
 		why: z.string()
 	});
@@ -1374,6 +1498,7 @@ const recordOf = (a, main) => {
 			kind: z.literal("removal"),
 			band,
 			window: window.optional(),
+			flaps: z.boolean().optional(),
 			why: z.string()
 		}),
 		z.object({
@@ -1440,7 +1565,11 @@ const recordOf = (a, main) => {
 			quiet: z.boolean().optional(),
 			close: z.boolean().optional(),
 			partition: z.boolean().optional(),
-			serves: z.boolean().optional()
+			serves: z.boolean().optional(),
+			flaps: z.array(z.object({
+				rest: time,
+				hold: time
+			})).optional()
 		}),
 		pod: z.object({
 			replicas: count,
@@ -1494,7 +1623,11 @@ const recordOf = (a, main) => {
 			after: a.nullable(),
 			by: a
 		}).nullable()).optional(),
-		closes: z.record(z.string(), a.nullable()).optional()
+		closes: z.record(z.string(), a.nullable()).optional(),
+		cycles: z.array(z.object({
+			empty: a.nullable(),
+			ready: a.nullable()
+		})).optional()
 	});
 	return {
 		cell,
@@ -1556,6 +1689,6 @@ const readRecord = (data) => {
 	throw new Error(`a kind cell record has the format ${FORMAT} or ${FORMAT_V1}, not ${String(format)}`);
 };
 //#endregion
-export { signalOf as $, medianInterval as A, emptyOf as B, propagationOf as C, unitOf as D, reusesReset as E, backendsAt as F, outcomesOf as G, intakeOf as H, byClients as I, recordsOf as J, podsAt as K, byNode as L, quantileOf as M, HBONE_PORT as N, updatesOf as O, TCP_STATE as P, rulesOf as Q, deletionOf as R, observationEnd as S, referenceOf as T, isTunnelState as U, failuresOf as V, kernelCounts as W, restartOf as X, requestsOf as Y, restartsAt as Z, checkRun as _, seconds as _t, calibrationOf as a, stoppingOf as at, judgeKind as b, predictionText as c, ztunnelPodsOf as ct, FORMAT_V1 as d, HEALTH_PORT as dt, signalsOf as et, judge as f, LOG_LABEL as ft, LAST_SLACK as g, bandOf as gt, KIND_NOISE as h, SERVER as ht, readRecord as i, socketsOf as it, middle as j, RELATIVE as k, timeSpread as l, ztunnelReplaced as lt, upgradeCell as m, REDIS as mt, kindCellRecordSchema as n, sliceEmptyOf as nt, cellsTable as o, transitionOf as ot, upgrade as p, PORT as pt, podsOf as q, kindCellRecordV1Schema as r, slicesOf as rt, medianText as s, tunnelsOf as st, STAGES as t, sliceDropOf as tt, FORMAT as u, CLIENT as ut, deletionsOf as v, secondsOf as vt, propagationSource as w, observationAfter as x, factsOf as y, showBand as yt, dropOf as z };
+export { rulesOf as $, RELATIVE as A, dropOf as B, observationEnd as C, reusesReset as D, referenceOf as E, TCP_STATE as F, kernelCounts as G, failuresOf as H, backendsAt as I, podsOf as J, outcomesOf as K, byClients as L, middle as M, quantileOf as N, unitOf as O, HBONE_PORT as P, restartsAt as Q, byNode as R, observationAfter as S, showBand as St, propagationSource as T, intakeOf as U, emptyOf as V, isTunnelState as W, requestsOf as X, recordsOf as Y, restartOf as Z, checkRun as _, REDIS as _t, calibrationOf as a, sliceReadyOf as at, factsOf as b, seconds as bt, predictionText as c, stoppingOf as ct, FORMAT_V1 as d, ztunnelPodsOf as dt, signalOf as et, judge as f, ztunnelReplaced as ft, LAST_SLACK as g, PORT as gt, KIND_NOISE as h, LOG_LABEL as ht, readRecord as i, sliceReadyAt as it, medianInterval as j, updatesOf as k, timeSpread as l, transitionOf as lt, upgradeCell as m, HEALTH_PORT as mt, kindCellRecordSchema as n, sliceDropOf as nt, cellsTable as o, slicesOf as ot, upgrade as p, CLIENT as pt, podsAt as q, kindCellRecordV1Schema as r, sliceEmptyOf as rt, medianText as s, socketsOf as st, STAGES as t, signalsOf as tt, FORMAT as u, tunnelsOf as ut, cyclesOf as v, SERVER as vt, propagationOf as w, judgeKind as x, secondsOf as xt, deletionsOf as y, bandOf as yt, deletionOf as z };
 
-//# sourceMappingURL=judge-Cs91dOGO.js.map
+//# sourceMappingURL=judge-BIXZEX__.js.map