@mark1russell7/kind 0.13.4 → 0.14.0

package/build/{judge-DKrgfhAt.js → judge-C6XgY2Pk.js}

@@ -356,6 +356,236 @@ const restartsAt = (lines, at) => {
 	return out;
 };
 //#endregion
+//#region src/reconnects.ts
+const delaysOf = (backoff) => ({
+	initial: valueIn(backoff.initial, "ms"),
+	max: valueIn(backoff.max, "ms"),
+	stop: valueIn(backoff.mandatoryStop, "ms"),
+	jitter: backoff.jitter
+});
+/**
+* This function gives the times of the tries of one path: each delay times `share`, and `latency`
+* from each try to the next call. The tries end with the first one after `until`.
+*/
+const pathOf = (delays, share, latency, until) => {
+	const tries = [];
+	let next = delays.initial;
+	let call = 0;
+	let made = false;
+	for (let count = 0; count < 1e3; count += 1) {
+		let current = next;
+		next = Math.min(2 * next, delays.max);
+		if (!made && call + current > delays.stop) {
+			current = Math.max(delays.initial, delays.stop - call);
+			made = true;
+		}
+		const at = call + Math.max(delays.initial, current * share);
+		tries.push(at);
+		if (at > until) break;
+		call = at + latency;
+	}
+	return tries;
+};
+/**
+* This function gives the band of each try after the first delay starts, up to the first try whose
+* earliest time is after `until`. The earliest path has the most jitter and no time for a try. The
+* latest path has no jitter and `attempt` for each failed try. A later call gets a delay that ends no
+* earlier, thus each try of a path between them is in the band.
+*/
+const triesOf = (backoff, until) => {
+	const delays = delaysOf(backoff);
+	const late = pathOf(delays, 1, valueIn(backoff.attempt, "ms"), Number.POSITIVE_INFINITY);
+	return pathOf(delays, 1 - delays.jitter, 0, until).map((lo, index) => ({
+		lo,
+		hi: late[index] ?? Number.POSITIVE_INFINITY
+	}));
+};
+/**
+* This function gives the latest return of a producer after the fault, from the bands of the
+* prediction. It adds three terms. The first is the first delay at its latest. The second is the
+* latest time of the first try that cannot come before the latest change of the owner. The third
+* is the latest setup.
+*/
+const latestReturn = (prediction) => {
+	const disconnect = {
+		lo: valueIn(prediction.disconnect.lo, "ms"),
+		hi: valueIn(prediction.disconnect.hi, "ms")
+	};
+	const flip = valueIn(prediction.flip.hi, "ms");
+	const tries = triesOf(prediction.backoff, flip - disconnect.lo);
+	const after = tries.find((each) => each.lo >= flip - disconnect.lo) ?? tries.at(-1);
+	return disconnect.hi + after.hi + valueIn(prediction.setup.hi, "ms");
+};
+/** This function gives the delay of each call of a backoff, as an interval, with `elapsed` from the first call to this one. */
+const delayBand = (delays, index, elapsed, made, noise) => {
+	const nominal = Math.min(delays.initial * 2 ** index, delays.max);
+	const band = (current) => ({
+		lo: Math.max(delays.initial, current * (1 - delays.jitter)),
+		hi: Math.max(delays.initial, current)
+	});
+	if (made) return {
+		...band(nominal),
+		made,
+		either: false
+	};
+	const clamped = band(Math.max(delays.initial, delays.stop - elapsed));
+	const plain = band(nominal);
+	if (Math.abs(elapsed + nominal - delays.stop) <= noise) return {
+		lo: Math.min(clamped.lo, plain.lo),
+		hi: Math.max(clamped.hi, plain.hi),
+		made: false,
+		either: true
+	};
+	return elapsed + nominal > delays.stop ? {
+		...clamped,
+		made: true,
+		either: false
+	} : {
+		...plain,
+		made: false,
+		either: false
+	};
+};
+/** This text is the start of the line of the C++ client that schedules a try: "Schedule reconnection in 0.1 s". */
+const SCHEDULE = /Schedule reconnection in ([0-9.]+) s/;
+const text = (line, field) => typeof line[field] === "string" ? line[field] : null;
+/**
+* This function judges one run: the fault at `fault`, the broker `broker` that owned the topic, and
+* the lines of the run. A lookup line is `{ev: "lookup", t, o, owner}`, a line of the client is
+* `{ev: "pulsar", t, p, m}`, and a send is `{ev: "send", t, d, o, p}`. `timeout` and `every` are the
+* timeout and the interval of the sends. Each check has a margin of `noise`.
+*
+* For each producer, the delays of its backoff follow the law, from the observed time of each call.
+* Its last try comes after the last lookup that named the old owner. The try before it comes no
+* later than the first lookup that named another. Then the setup is in its band. Each failed send
+* starts between the fault and the return, with an outcome of the prediction. At least one fails.
+*
+* With `owner: "either"`, the lookups can name the old owner for the whole run, because it started
+* again and took the topic back. Then the last try comes after the start of the band `flip`. The
+* try before it comes no later than the end of that band.
+*/
+const judgeReconnects = (prediction, run, sends, noise) => {
+	const { fault, broker, lines } = run;
+	const ms = (band) => ({
+		lo: valueIn(band.lo, "ms"),
+		hi: valueIn(band.hi, "ms")
+	});
+	const inBand = (value, band) => value >= band.lo - noise && value <= band.hi + noise;
+	const lookups = lines.filter((line) => line.ev === "lookup" && line.t >= fault).sort((a, b) => a.t - b.t);
+	const changed = lookups.findIndex((line) => line["o"] === "ok" && text(line, "owner") !== null && text(line, "owner") !== broker);
+	const fail = (why, more = {}) => ({
+		ok: false,
+		why,
+		flip: null,
+		gap: null,
+		returns: [],
+		failures: {},
+		value: null,
+		...more
+	});
+	if (changed === -1 && prediction.owner !== "either") return fail(`no lookup after the fault names an owner other than ${broker}`);
+	const band = ms(prediction.flip);
+	const same = changed === -1;
+	const flipAt = same ? fault + band.hi : lookups[changed].t;
+	const before = same ? fault + band.lo : changed === 0 ? fault : lookups[changed - 1].t;
+	const flip = same ? null : flipAt - fault;
+	const gap = same ? null : flipAt - before;
+	if (flip !== null && !inBand(flip, band)) return fail(`the first lookup with another owner is ${flip.toFixed(0)} ms after the fault, outside the band`, {
+		flip,
+		gap
+	});
+	if (gap !== null && gap > valueIn(prediction.probe, "ms") + noise) return fail(`the lookups have a gap of ${gap.toFixed(0)} ms before the change of the owner`, {
+		flip,
+		gap
+	});
+	const delays = delaysOf(prediction.backoff);
+	const setup = ms(prediction.setup);
+	const disconnect = ms(prediction.disconnect);
+	const pods = [...new Set(lines.filter((line) => line.ev === "send").map((line) => String(line["p"])))].sort();
+	if (pods.length === 0) return fail("the run has no sends", {
+		flip,
+		gap
+	});
+	const returns = pods.map((pod) => {
+		const own = lines.filter((line) => line.ev === "pulsar" && line["p"] === pod && line.t >= fault - noise).sort((a, b) => a.t - b.t);
+		const created = own.find((line) => (text(line, "m") ?? "").includes("Created producer on broker"));
+		const back = created === void 0 ? null : created.t - fault;
+		const schedules = own.filter((line) => created === void 0 || line.t <= created.t).flatMap((line) => {
+			const found = SCHEDULE.exec(text(line, "m") ?? "");
+			return found === null ? [] : [{
+				t: line.t,
+				delay: Number(found[1]) * 1e3
+			}];
+		});
+		const first = schedules[0];
+		const facts = {
+			pod,
+			disconnect: first === void 0 ? null : first.t - fault,
+			schedules,
+			back,
+			tries: schedules.length
+		};
+		const no = (why, more = {}) => ({
+			...facts,
+			setup: null,
+			...more,
+			ok: false,
+			why: `${pod}: ${why}`
+		});
+		if (created === void 0) return no("the producer is not back on a broker after the fault");
+		if (first === void 0) return no("the producer schedules no try after the fault");
+		if (!inBand(first.t - fault, disconnect)) return no(`its first try is scheduled ${(first.t - fault).toFixed(0)} ms after the fault, outside the band`);
+		let made = false;
+		for (const [index, schedule] of schedules.entries()) {
+			const band = delayBand(delays, index, schedule.t - first.t, made, noise);
+			if (schedule.delay < band.lo - 1 || schedule.delay > band.hi + 1) return no(`the delay ${index + 1} is ${schedule.delay} ms, not in ${band.lo.toFixed(0)}–${band.hi.toFixed(0)} ms of the backoff`);
+			made = band.either ? schedule.delay < Math.min(delays.initial * 2 ** index, delays.max) * (1 - delays.jitter) - 1 : band.made;
+		}
+		const tries = schedules.map((schedule) => schedule.t + schedule.delay);
+		const last = tries.at(-1);
+		const earlier = tries.at(-2);
+		if (last < before - noise) return no(`its last try at ${(last - fault).toFixed(0)} ms comes before the last lookup of the old owner`);
+		if (earlier !== void 0 && earlier > flipAt + noise) return no(`its try at ${(earlier - fault).toFixed(0)} ms comes after the change of the owner, and it did not succeed`);
+		const took = created.t - last;
+		if (!inBand(took, setup)) return no(`its setup takes ${took.toFixed(0)} ms, outside the band`, { setup: took });
+		return {
+			...facts,
+			setup: took,
+			ok: true,
+			why: `${pod}: back ${(created.t - fault).toFixed(0)} ms after the fault, at try ${tries.length}`
+		};
+	});
+	const failed = lines.filter((line) => line.ev === "send" && line["o"] !== "ok").sort((a, b) => a.t - b.t);
+	const failures = {};
+	for (const line of failed) failures[String(line["o"])] = (failures[String(line["o"])] ?? 0) + 1;
+	const backs = returns.flatMap((each) => each.back === null ? [] : [each.back]).sort((a, b) => a - b);
+	const value = backs.length === 0 ? null : backs.length % 2 === 1 ? backs[(backs.length - 1) / 2] : (backs[backs.length / 2 - 1] + backs[backs.length / 2]) / 2;
+	const done = (ok, why) => ({
+		ok,
+		why,
+		flip,
+		gap,
+		returns,
+		failures,
+		value
+	});
+	const wrong = returns.find((each) => !each.ok);
+	if (wrong !== void 0) return done(false, wrong.why);
+	const other = failed.find((line) => !prediction.outcomes.includes(String(line["o"])));
+	if (other !== void 0) return done(false, `a send of ${String(other["p"])} ended ${String(other["o"])}, not ${prediction.outcomes.join(" or ")}`);
+	if (failed.length === 0) return done(false, "no send failed");
+	for (const each of returns) {
+		const own = failed.filter((line) => line["p"] === each.pod);
+		const back = fault + each.back;
+		const outside = own.find((line) => line.t < fault - sends.timeout - noise || line.t > back + noise);
+		if (outside !== void 0) return done(false, `a send of ${each.pod} at ${(outside.t - fault).toFixed(0)} ms failed outside the time to its return`);
+		const latest = own.at(-1);
+		if (latest === void 0) return done(false, `no send of ${each.pod} failed`);
+		if (latest.t < back - sends.timeout - sends.every - noise) return done(false, `the last failed send of ${each.pod} is ${(back - latest.t).toFixed(0)} ms before its return`);
+	}
+	return done(true, `${flip === null ? "the stopped broker owns the topic again" : `another owner ${flip.toFixed(0)} ms after the fault`}; ${returns.map((each) => `${each.pod} back at try ${each.tries}`).join(", ")}`);
+};
+//#endregion
 //#region src/verdict.ts
 /**
 * This time is δ for kind: each time of a run lands in its band ± δ. Its parts are the interval of
@@ -390,7 +620,8 @@ const unitOf = (prediction) => {
 	switch (prediction.kind) {
 		case "removal":
 		case "leaves":
-		case "stalls": return "s";
+		case "stalls":
+		case "reconnects": return "s";
 		case "restarts": return prediction.band === null ? "{replica}" : "s";
 		case "ends": return prediction.band === null ? "{socket}" : "s";
 		default: return "{request}";
@@ -534,6 +765,7 @@ const observationEnd = (cell, run) => {
 		if (at !== null) ends.push(at + valueIn(prediction.band.hi, "ms") + NOISE);
 	}
 	const after = observationAfter(cell);
+	if (prediction.kind === "reconnects" && run.acts.fault !== null) ends.push(run.acts.fault + latestReturn(prediction) + valueIn(cell.workload.timeout, "ms") + NOISE);
 	const window = windowOf(prediction);
 	if (after !== null && window !== null && window.end !== null) {
 		const from = window.end.from;
@@ -1012,6 +1244,37 @@ const checkRun = (cell, run) => {
 			return done(false, problems.join("; "), null);
 		}
 		case "ends": return checkEnds(cell, run, prediction, done);
+		case "reconnects": {
+			const fault = run.acts.fault;
+			if (fault === null) return done(false, "the run has no stamp of the fault", null);
+			if (run.broker === void 0) return done(false, "the run has no broker that owned the topic at the fault", null);
+			const judged = judgeReconnects(prediction, {
+				fault,
+				broker: run.broker,
+				lines
+			}, {
+				timeout: valueIn(cell.workload.timeout, "ms"),
+				every: valueIn(cell.workload.every, "ms")
+			}, NOISE);
+			const time = (value) => value === null ? null : {
+				value: sec(value),
+				unit: "s"
+			};
+			const reconnects = {
+				flip: time(judged.flip),
+				gap: time(judged.gap),
+				returns: judged.returns.map((each) => ({
+					pod: each.pod,
+					disconnect: time(each.disconnect),
+					back: time(each.back),
+					setup: time(each.setup),
+					tries: each.tries
+				})),
+				failures: judged.failures
+			};
+			if (failures.length > 0) return done(false, `${failures.length} requests of the control failed: ${JSON.stringify(facts.outcomes)}`, null, { reconnects });
+			return done(judged.ok, judged.why, judged.value === null ? null : sec(judged.value), { reconnects });
+		}
 		case "measured": return done(true, "measured", facts.failures);
 	}
 };
@@ -1349,6 +1612,23 @@ const upgradePrediction = (prediction) => {
 				...failures === void 0 ? {} : { failures: failures.kind === "window" ? upgradeWindow(failures) : failures }
 			};
 		}
+		case "reconnects": {
+			const { backoff } = prediction;
+			return {
+				...prediction,
+				flip: band(prediction.flip),
+				probe: seconds(prediction.probe),
+				disconnect: band(prediction.disconnect),
+				backoff: {
+					...backoff,
+					initial: seconds(backoff.initial),
+					max: seconds(backoff.max),
+					mandatoryStop: seconds(backoff.mandatoryStop),
+					attempt: seconds(backoff.attempt)
+				},
+				setup: band(prediction.setup)
+			};
+		}
 		default: return prediction;
 	}
 };
@@ -1364,7 +1644,7 @@ const upgradeFacts = (facts) => {
 		by: seconds(old.by)
 	};
 	const byKey = (record) => Object.fromEntries(Object.entries(record).map(([key, old]) => [key, propagated(old)]));
-	const { sliceEmpty, rules, left, signals, updates, healUpdates, closes, cycles, stall, period, ...rest } = facts;
+	const { sliceEmpty, rules, left, signals, updates, healUpdates, closes, cycles, stall, period, reconnects, ...rest } = facts;
 	return {
 		...rest,
 		intakeAfterDeletion: time$1(facts.intakeAfterDeletion),
@@ -1385,6 +1665,17 @@ const upgradeFacts = (facts) => {
 			start: time$1(stall.start),
 			held: time$1(stall.held)
 		} },
+		...reconnects === void 0 ? {} : { reconnects: {
+			...reconnects,
+			flip: time$1(reconnects.flip),
+			gap: time$1(reconnects.gap),
+			returns: reconnects.returns.map((each) => ({
+				...each,
+				disconnect: time$1(each.disconnect),
+				back: time$1(each.back),
+				setup: time$1(each.setup)
+			}))
+		} },
 		...period === void 0 ? {} : { period: {
 			...period,
 			median: seconds(period.median),
@@ -1471,6 +1762,7 @@ const predictionText = (prediction) => {
 			const closes = prediction.band === null ? "no socket of a client ends, and each one calls again on its socket" : prediction.of === "tunnel" ? `the side of the client ends the tunnel of each client in ${atBand(prediction.from, prediction.band)}${prediction.answer === void 0 ? "" : `, or the peer answers in ${atBand(prediction.from, prediction.answer)}`}, and each one calls again` : `the socket of each client ends in ${atBand(prediction.from, prediction.band)}`;
 			return prediction.window === void 0 ? closes : `${closes}; ${predictionText(prediction.window)}`;
 		}
+		case "reconnects": return `another owner in ${showBand(prediction.flip)}, each producer back at the first try of its backoff after it, failures only before it, each one ${prediction.outcomes.join(" or ")}`;
 		case "measured": return "measured";
 	}
 };
@@ -1686,6 +1978,7 @@ const run = z.object({
 	}),
 	target: replica.nullable(),
 	paused: z.string().nullable(),
+	broker: z.string().optional(),
 	replicas: z.array(replica),
 	void: z.string().nullable(),
 	lines: z.array(line),
@@ -1826,6 +2119,22 @@ const recordOf = (a, main) => {
 			answer: band.optional(),
 			why: z.string()
 		}),
+		z.object({
+			kind: z.literal("reconnects"),
+			flip: band,
+			probe: a,
+			disconnect: band,
+			backoff: z.object({
+				initial: a,
+				max: a,
+				mandatoryStop: a,
+				jitter: z.number(),
+				attempt: a
+			}),
+			setup: band,
+			outcomes: z.array(z.string()),
+			why: z.string()
+		}),
 		z.object({
 			kind: z.literal("measured"),
 			why: z.string()
@@ -1894,7 +2203,16 @@ const recordOf = (a, main) => {
 		runs: count,
 		vip: z.object({
 			clients: count,
-			announcer: z.enum(["kube-vip", "metallb"]).optional()
+			announcer: z.enum(["kube-vip", "metallb"]).optional(),
+			memberlist: z.object({
+				probeInterval: time,
+				probeTimeout: time,
+				suspicionMult: count
+			}).optional()
+		}).optional(),
+		pulsar: z.object({
+			clients: count,
+			probe: time
 		}).optional()
 	});
 	const facts = z.object({
@@ -1948,6 +2266,18 @@ const recordOf = (a, main) => {
 			high: a,
 			intervals: count,
 			byClient: z.record(z.string(), a)
+		}).optional(),
+		reconnects: z.object({
+			flip: a.nullable(),
+			gap: a.nullable(),
+			returns: z.array(z.object({
+				pod: z.string(),
+				disconnect: a.nullable(),
+				back: a.nullable(),
+				setup: a.nullable(),
+				tries: count
+			})),
+			failures: z.record(z.string(), count)
 		}).optional()
 	});
 	return {
@@ -2010,6 +2340,6 @@ const readRecord = (data) => {
 	throw new Error(`a kind cell record has the format ${FORMAT} or ${FORMAT_V1}, not ${String(format)}`);
 };
 //#endregion
-export { signalsOf as $, HBONE_PORT as A, secondsOf as At, isTunnelState as B, observationEnd as C, VIP_ANNOTATION as Ct, reusesReset as D, overshootBand as Dt, referenceOf as E, drainBand as Et, deletionOf as F, podsOf as G, outcomesOf as H, dropOf as I, restartOf as J, recordsOf as K, emptyOf as L, backendsAt as M, byClients as N, unitOf as O, requestBand as Ot, byNode as P, signalOf as Q, failuresOf as R, observationAfter as S, SERVER as St, propagationSource as T, bandOf as Tt, periodOf as U, kernelCounts as V, podsAt as W, rulesOf as X, restartsAt as Y, sessionsAt as Z, checkRun as _, LB_CLASS as _t, calibrationOf as a, socketsOf as at, factsOf as b, PORT as bt, predictionText as c, tunnelsOf as ct, FORMAT_V1 as d, RELATIVE as dt, sliceDropOf as et, judge as f, medianInterval$1 as ft, LAST_SLACK as g, HEALTH_PORT as gt, KIND_NOISE as h, CLIENT as ht, readRecord as i, slicesOf as it, TCP_STATE as j, showBand as jt, updatesOf as k, seconds as kt, timeSpread as l, ztunnelPodsOf as lt, upgradeCell as m, quantileOf as mt, kindCellRecordSchema as n, sliceReadyAt as nt, cellsTable as o, stoppingOf as ot, upgrade as p, middle as pt, requestsOf as q, kindCellRecordV1Schema as r, sliceReadyOf as rt, medianText as s, transitionOf as st, STAGES as t, sliceEmptyOf as tt, FORMAT as u, ztunnelReplaced as ut, cyclesOf as v, LOG_LABEL as vt, propagationOf as w, backlogEnd as wt, judgeKind as x, REDIS as xt, deletionsOf as y, OUTSIDE as yt, intakeOf as z };
+export { rulesOf as $, judgeReconnects as A, overshootBand as At, emptyOf as B, observationEnd as C, PORT as Ct, reusesReset as D, backlogEnd as Dt, referenceOf as E, VIP_ANNOTATION as Et, backendsAt as F, outcomesOf as G, intakeOf as H, byClients as I, podsOf as J, periodOf as K, byNode as L, triesOf as M, seconds as Mt, HBONE_PORT as N, secondsOf as Nt, unitOf as O, bandOf as Ot, TCP_STATE as P, showBand as Pt, restartsAt as Q, deletionOf as R, observationAfter as S, OUTSIDE as St, propagationSource as T, SERVER as Tt, isTunnelState as U, failuresOf as V, kernelCounts as W, requestsOf as X, recordsOf as Y, restartOf as Z, checkRun as _, quantileOf as _t, calibrationOf as a, sliceReadyAt as at, factsOf as b, LB_CLASS as bt, predictionText as c, socketsOf as ct, FORMAT_V1 as d, tunnelsOf as dt, sessionsAt as et, judge as f, ztunnelPodsOf as ft, LAST_SLACK as g, middle as gt, KIND_NOISE as h, medianInterval$1 as ht, readRecord as i, sliceEmptyOf as it, latestReturn as j, requestBand as jt, updatesOf as k, drainBand as kt, timeSpread as l, stoppingOf as lt, upgradeCell as m, RELATIVE as mt, kindCellRecordSchema as n, signalsOf as nt, cellsTable as o, sliceReadyOf as ot, upgrade as p, ztunnelReplaced as pt, podsAt as q, kindCellRecordV1Schema as r, sliceDropOf as rt, medianText as s, slicesOf as st, STAGES as t, signalOf as tt, FORMAT as u, transitionOf as ut, cyclesOf as v, CLIENT as vt, propagationOf as w, REDIS as wt, judgeKind as x, LOG_LABEL as xt, deletionsOf as y, HEALTH_PORT as yt, dropOf as z };
 
-//# sourceMappingURL=judge-DKrgfhAt.js.map
+//# sourceMappingURL=judge-C6XgY2Pk.js.map