@mark1russell7/kind 0.1.8 → 0.2.1

package/build/{judge-a7yCzwZc.d.ts → judge-Dwa6MqP4.d.ts}

@@ -79,6 +79,15 @@ declare const cidrOverlap: (a: string, b: string) => boolean;
 //#region src/cell.d.ts
 /** A template is the topology of a cell: T07, the endpoint lifecycle of a Service, or T18, a health-gated fleet. */
 type Template = "T07" | "T18";
+/**
+ * A stack is the server and the clients of a cell:
+ * - `service`: the service of this repository with its declaration, and the HTTP client, which opens a
+ *   new connection for each request. Thus kube-proxy picks a backend for each request.
+ * - `grpc`: the server of grpc-go v1.84.0 of the harness, and a client of grpc-js 1.14.6 with one channel
+ *   at its defaults. The channel holds one connection. It connects again only when that connection
+ *   ends: at a GOAWAY of the server, or at the end of its socket.
+ */
+type Stack = "service" | "grpc";
 /**
  * A trigger is the event or the fault of a cell, by the name of the plan:
  * - `pod-deletion`, `rolling-update`, `node-drain` and `scale-in` stop a pod with its grace.
@@ -157,11 +166,17 @@ interface Declaration {
     readonly threshold: Quantity;
   } | null;
 }
-/** A workload is the clients: one pod on each node, each with requests on new connections to the ClusterIP. */
+/**
+ * A workload is the clients: one pod on each node, each with requests to the ClusterIP. The HTTP client
+ * opens a new connection for each request. The gRPC client sends each call on its one channel.
+ */
 interface Workload {
   /** This time is the interval between two requests of one client. The clients do not wait for an answer. */
   readonly every: Quantity;
-  /** This time is how long a client waits for an answer before the request is a timeout. */
+  /**
+   * This time is how long a client waits for an answer before the request is a timeout. A call of
+   * gRPC has no deadline at the defaults of grpc-js, thus the client cancels it at this time.
+   */
   readonly timeout: Quantity;
 }
 /**
@@ -197,6 +212,8 @@ interface ResetReuse<A = Quantity> {
  * - `window`: the requests that fail start in a window. The window starts at `start` and ends in
  *   `end`. With `certain`, at least one request fails. With `afterReset`, a request outside the
  *   window can also fail, on a port that a reset closed up to `life` before it (`ResetReuse`).
+ *   With `each`, each replica of the run that gets a deletion has a window of its own, from its own
+ *   references. A request is late only when it is outside each of these windows.
  * - `removal`: the Service has no ready endpoint, and the rules of each node have no backend, in the band.
  * - `leaves`: the target replica leaves the EndpointSlice and the rules of each node in the band.
  * - `restarts`: each replica receives SIGTERM in the band, or no replica does.
@@ -222,6 +239,12 @@ type Prediction<A = Quantity> = {
   } | null;
   readonly certain: boolean;
   readonly afterReset?: ResetReuse<A> | undefined;
+  /**
+   * This flag gives a window to each replica of the run (`replicas`) that gets a deletion after the
+   * start of the run. The references `deletion` and `intake` are then those of that replica. A
+   * rolling update deletes each old replica in turn. Without the flag, the window is that of the target.
+   */
+  readonly each?: boolean | undefined;
   readonly why: string;
 } | {
   readonly kind: "removal";
@@ -295,13 +318,16 @@ interface KindCell<A = Quantity> {
   readonly id: string;
   readonly name: string;
   readonly template: Template;
+  /** This stack is the server and the clients of the cell. Without it, the cell has the service (`service`). */
+  readonly stack?: Stack | undefined;
   /** This text names a system of the plan that the cell replaces with the service of this repository, or null. */
   readonly substitutes: string | null;
   readonly mode: KubeProxyMode;
   readonly workers: number;
   readonly trigger: Trigger<A>;
   readonly pod: PodSettings;
-  readonly declaration: Declaration;
+  /** This declaration is the settings of the service, or null for a server that is not the service (the stack `grpc`). */
+  readonly declaration: Declaration | null;
   readonly workload: Workload;
   readonly prediction: Prediction<A>;
   /** This flag marks a calibration: its runs measure the propagation of the endpoints, and nothing fails. */
@@ -342,7 +368,7 @@ declare const REDIS = "redis";
  */
 /**
  * A line is one JSON line of a source of a run, with its stamp `t` in milliseconds on
- * CLOCK_MONOTONIC. It is a request of a client, a record of the service or a probe answer. It is
+ * CLOCK_MONOTONIC. It is a request of a client, a record of the service, a line of grpc-go or a probe answer. It is
  * also a pod, an EndpointSlice or an event of the API. Or it is the backends of the rules of a
  * node, a kernel record, or an act of the lab. A record keeps each line as its source wrote it.
  */
@@ -430,15 +456,32 @@ declare const recordsOf: (lines: readonly Line[]) => RecordLine[];
 declare const failuresOf: (lines: readonly Line[]) => Request[];
 /** This function counts the outcomes of the requests, by outcome. */
 declare const outcomesOf: (requests: readonly Request[]) => Record<string, number>;
+/**
+ * This function orders the replicas by how many clients each one answered last, the most first,
+ * then by name. `answers` gives the backend of the last answer of each client, by client. A client
+ * of gRPC holds its connection to one replica, thus the first replica is the one whose stop the
+ * most clients see.
+ */
+declare const byClients: <T extends {
+  readonly pod: string;
+}>(replicas: readonly T[], answers: ReadonlyMap<string, string>) => T[];
 /** This function gives the requests of each node, by the name of the node. */
 declare const byNode: (requests: readonly Request[]) => Map<string, Request[]>;
 /** This function gives the time that the watcher first received a deletion timestamp of one of the pods, after a time. */
 declare const deletionOf: (lines: readonly Line[], pods: readonly string[], after: number) => number | null;
 /** This function gives the stamp of a transition of the service in a pod: the first matching record of a machine, after a time. */
 declare const transitionOf: (lines: readonly Line[], pod: string, machine: string, match: (record: RecordLine) => boolean, after: number) => number | null;
-/** This function gives when the service of a pod stopped its intake: the start of the phase `stop intake`. */
+/**
+ * This function gives the line `stopping` of grpc-go in a pod, after a time. The server writes it at
+ * SIGTERM. Then GracefulStop closes its listener and sends GOAWAY on each connection.
+ */
+declare const stoppingOf: (lines: readonly Line[], pod: string, after: number) => number | null;
+/**
+ * This function gives when the server of a pod stopped its intake. That is the start of the phase
+ * `stop intake` of the service, or the line `stopping` of grpc-go.
+ */
 declare const intakeOf: (lines: readonly Line[], pod: string, after: number) => number | null;
-/** This function gives when the service of a pod received its exit signal. */
+/** This function gives when the server of a pod received its exit signal: a record of the service, or the line `stopping` of grpc-go. */
 declare const signalOf: (lines: readonly Line[], pod: string, after: number) => number | null;
 /**
  * This function gives when the rules of a node stopped the selection of a backend: the first poll
@@ -506,6 +549,17 @@ interface RunFacts<A = Quantity> {
   readonly left?: A | undefined;
   /** These times are the SIGTERM of each replica, after the fault (`restarts`). */
   readonly signals?: ReadonlyArray<A | null> | undefined;
+  /**
+   * These are the replicas of a window for each deleted replica (`each`). Each one is a replica of the
+   * run with a deletion after its start, with the stamps of its intake and of its deletion.
+   */
+  readonly deletions?: readonly Deleted[] | undefined;
+}
+/** A deleted replica is a replica of a run that got a deletion. It gives its pod, and the stamps of its intake (or null) and of its deletion. */
+interface Deleted {
+  readonly pod: string;
+  readonly intake: number | null;
+  readonly deletion: number;
 }
 /**
  * A run check is the judgement of one run: if it agrees with the prediction, why, its main value,
@@ -552,14 +606,22 @@ declare const unitOf: (prediction: {
  * `outcome`.
  */
 declare const reusesReset: (request: Request, requests: readonly Request[], reuse: ResetReuse) => boolean;
-/** This function gives a reference event of a run, as a stamp, or null when the run does not have it. */
-declare const referenceOf: (run: KindRun, from: Reference) => number | null;
 /**
- * This function gives the propagation of each node for the target of a run. It is the time from
- * its deletion (or its crash) to the first poll of the rules of the node without it. The change
- * came between the poll before (`after`) and that poll (`by`).
+ * This function gives a reference event of a run, as a stamp, or null when the run does not have it.
+ * The references `deletion` and `intake` are those of the target, or of the replica `pod`.
+ */
+declare const referenceOf: (run: KindRun, from: Reference, pod?: string | undefined) => number | null;
+/**
+ * This function gives the propagation of each node for the target of a run, or for the replica of
+ * the address `ip`. It is the time from its deletion (or its crash) to the first poll of the rules
+ * of the node without it. The change came between the poll before (`after`) and that poll (`by`).
+ */
+declare const propagationOf: (run: KindRun, from: number | null, ip?: string | undefined) => Record<string, Propagated | null>;
+/**
+ * This function gives each replica of a run that got a deletion after the start of the run, in the
+ * sequence of the deletions. A rolling update deletes each old replica in turn.
  */
-declare const propagationOf: (run: KindRun, from: number | null) => Record<string, Propagated | null>;
+declare const deletionsOf: (run: KindRun) => Deleted[];
 /** This function gives the facts of a run that each table reads: its failures by outcome and node, and the times of its target. */
 declare const factsOf: (cell: KindCell, run: KindRun) => RunFacts;
 /** This function judges one run against the prediction of its cell. */
@@ -722,5 +784,5 @@ declare const quantileOf: (sorted: readonly number[], p: number) => number;
  */
 declare const medianInterval: (sorted: readonly number[]) => [number, number];
 //#endregion
-export { RulesLine as $, cidrOverlap as $t, TriggerV1 as A, secondsOf as At, RunFacts as B, Reference as Bt, KindCellResult as C, HEALTH_PORT as Ct, PredictionV1 as D, SERVER as Dt, LoadSample as E, REDIS as Et, KindDecision as F, KindRun as Ft, referenceOf as G, TriggerKind as Gt, factsOf as H, ResetReuse as Ht, KindVerdict as I, KubeletProbe as It, Endpoint as J, KindConfig as Jt, reusesReset as K, Workload as Kt, LAST_SLACK as L, PodSettings as Lt, upgrade as M, Band as Mt, upgradeCell as N, Declaration as Nt, RunCheckV1 as O, bandOf as Ot, KIND_NOISE as P, KindCell as Pt, Request as Q, NodeImage as Qt, Propagated as R, Prediction as Rt, KindCellRecordV1 as S, CLIENT as St, KindVerdictV1 as T, PORT as Tt, judgeKind as U, Template as Ut, checkRun as V, Replica as Vt, propagationOf as W, Trigger as Wt, PodLine as X, KubeProxyMode as Xt, Line as Y, KindScenario as Yt, RecordLine as Z, NODE_IMAGES as Zt, ClusterManifest as _, signalsOf as _t, STAGES as a, scenario as an, emptyOf as at, FORMAT_V1 as b, slicesOf as bt, readRecord as c, kernelCounts as ct, calibrationOf as d, podsOf as dt, cidrRange as en, SliceLine as et, cellsTable as f, recordsOf as ft, BandV1 as g, signalOf as gt, timeSpread as h, rulesOf as ht, quantileOf as i, nodeNames as in, dropOf as it, judge as j, showBand as jt, RunFactsV1 as k, seconds as kt, Judged as l, outcomesOf as lt, predictionText as m, restartsAt as mt, medianInterval as n, kindConfig as nn, byNode as nt, kindCellRecordSchema as o, scenarioProblems as on, failuresOf as ot, medianText as p, requestsOf as pt, unitOf as q, KUBE_PROXY_MODES as qt, middle as r, kubeProxyPatch as rn, deletionOf as rt, kindCellRecordV1Schema as s, intakeOf as st, RELATIVE as t, goDuration as tn, backendsAt as tt, TimeSpread as u, podsAt as ut, EnvironmentManifest as v, sliceDropOf as vt, KindCellV1 as w, LOG_LABEL as wt, KindCellRecord as x, transitionOf as xt, FORMAT as y, sliceEmptyOf as yt, RunCheck as z, RedisDeclaration as zt };
-//# sourceMappingURL=judge-a7yCzwZc.d.ts.map
+export { RecordLine as $, KindConfig as $t, TriggerV1 as A, REDIS as At, RunCheck as B, KubeletProbe as Bt, KindCellResult as C, slicesOf as Ct, PredictionV1 as D, HEALTH_PORT as Dt, LoadSample as E, CLIENT as Et, KIND_NOISE as F, showBand as Ft, judgeKind as G, Replica as Gt, checkRun as H, Prediction as Ht, KindDecision as I, Band as It, reusesReset as J, Template as Jt, propagationOf as K, ResetReuse as Kt, KindVerdict as L, Declaration as Lt, upgrade as M, bandOf as Mt, upgradeCell as N, seconds as Nt, RunCheckV1 as O, LOG_LABEL as Ot, Deleted as P, secondsOf as Pt, PodLine as Q, KUBE_PROXY_MODES as Qt, LAST_SLACK as R, KindCell as Rt, KindCellRecordV1 as S, sliceEmptyOf as St, KindVerdictV1 as T, transitionOf as Tt, deletionsOf as U, RedisDeclaration as Ut, RunFacts as V, PodSettings as Vt, factsOf as W, Reference as Wt, Endpoint as X, TriggerKind as Xt, unitOf as Y, Trigger as Yt, Line as Z, Workload as Zt, ClusterManifest as _, restartsAt as _t, STAGES as a, cidrRange as an, byNode as at, FORMAT_V1 as b, signalsOf as bt, readRecord as c, kubeProxyPatch as cn, emptyOf as ct, calibrationOf as d, scenarioProblems as dn, kernelCounts as dt, KindScenario as en, Request as et, cellsTable as f, outcomesOf as ft, BandV1 as g, requestsOf as gt, timeSpread as h, recordsOf as ht, quantileOf as i, cidrOverlap as in, byClients as it, judge as j, SERVER as jt, RunFactsV1 as k, PORT as kt, Judged as l, nodeNames as ln, failuresOf as lt, predictionText as m, podsOf as mt, medianInterval as n, NODE_IMAGES as nn, SliceLine as nt, kindCellRecordSchema as o, goDuration as on, deletionOf as ot, medianText as p, podsAt as pt, referenceOf as q, Stack as qt, middle as r, NodeImage as rn, backendsAt as rt, kindCellRecordV1Schema as s, kindConfig as sn, dropOf as st, RELATIVE as t, KubeProxyMode as tn, RulesLine as tt, TimeSpread as u, scenario as un, intakeOf as ut, EnvironmentManifest as v, rulesOf as vt, KindCellV1 as w, stoppingOf as wt, KindCellRecord as x, sliceDropOf as xt, FORMAT as y, signalOf as yt, Propagated as z, KindRun as zt };
+//# sourceMappingURL=judge-Dwa6MqP4.d.ts.map

package/build/{judge-BbTAxRSd.js → judge-reWPMVr4.js}

@@ -50,6 +50,16 @@ const outcomesOf = (requests) => {
 	for (const request of requests) out[request.o] = (out[request.o] ?? 0) + 1;
 	return out;
 };
+/**
+* This function orders the replicas by how many clients each one answered last, the most first,
+* then by name. `answers` gives the backend of the last answer of each client, by client. A client
+* of gRPC holds its connection to one replica, thus the first replica is the one whose stop the
+* most clients see.
+*/
+const byClients = (replicas, answers) => {
+	const count = (pod) => [...answers.values()].filter((backend) => backend === pod).length;
+	return [...replicas].sort((a, b) => count(b.pod) - count(a.pod) || a.pod.localeCompare(b.pod));
+};
 /** This function gives the requests of each node, by the name of the node. */
 const byNode = (requests) => {
 	const out = /* @__PURE__ */ new Map();
@@ -60,10 +70,18 @@ const byNode = (requests) => {
 const deletionOf = (lines, pods, after) => podsOf(lines).find((pod) => pods.includes(pod.name) && pod.del && pod.t >= after)?.t ?? null;
 /** This function gives the stamp of a transition of the service in a pod: the first matching record of a machine, after a time. */
 const transitionOf = (lines, pod, machine, match, after) => recordsOf(lines).find((record) => record.p === pod && record.m === machine && record.at >= after && match(record))?.at ?? null;
-/** This function gives when the service of a pod stopped its intake: the start of the phase `stop intake`. */
-const intakeOf = (lines, pod, after) => transitionOf(lines, pod, "drain", (record) => record.to === "stop intake", after);
-/** This function gives when the service of a pod received its exit signal. */
-const signalOf = (lines, pod, after) => transitionOf(lines, pod, "exit", (record) => record.on === "exit.request", after);
+/**
+* This function gives the line `stopping` of grpc-go in a pod, after a time. The server writes it at
+* SIGTERM. Then GracefulStop closes its listener and sends GOAWAY on each connection.
+*/
+const stoppingOf = (lines, pod, after) => lines.find((line) => line.ev === "stopping" && line["p"] === pod && line.t >= after)?.t ?? null;
+/**
+* This function gives when the server of a pod stopped its intake. That is the start of the phase
+* `stop intake` of the service, or the line `stopping` of grpc-go.
+*/
+const intakeOf = (lines, pod, after) => transitionOf(lines, pod, "drain", (record) => record.to === "stop intake", after) ?? stoppingOf(lines, pod, after);
+/** This function gives when the server of a pod received its exit signal: a record of the service, or the line `stopping` of grpc-go. */
+const signalOf = (lines, pod, after) => transitionOf(lines, pod, "exit", (record) => record.on === "exit.request", after) ?? stoppingOf(lines, pod, after);
 /**
 * This function gives when the rules of a node stopped the selection of a backend: the first poll
 * after a time without it. The change came after `prev`, the poll before it, and at the latest at `t`.
@@ -199,27 +217,29 @@ const reusesReset = (request, requests, reuse) => {
 	const life = valueIn(reuse.life, "ms") + NOISE;
 	return requests.some((earlier) => earlier.o === "reset" && earlier.p === request.p && earlier.lp === request.lp && earlier.t + earlier.d <= request.t && request.t - (earlier.t + earlier.d) <= life);
 };
-/** This function gives a reference event of a run, as a stamp, or null when the run does not have it. */
-const referenceOf = (run, from) => {
+/**
+* This function gives a reference event of a run, as a stamp, or null when the run does not have it.
+* The references `deletion` and `intake` are those of the target, or of the replica `pod`.
+*/
+const referenceOf = (run, from, pod = run.target?.pod) => {
 	const lines = run.lines;
 	if (from === "fault") return run.acts.fault;
 	if (from === "resume") return run.acts.resume;
-	if (run.target === null) return null;
-	const deletion = deletionOf(lines, [run.target.pod], run.start);
-	if (from === "deletion") return deletion;
-	return intakeOf(lines, run.target.pod, run.start);
+	if (pod === void 0) return null;
+	if (from === "deletion") return deletionOf(lines, [pod], run.start);
+	return intakeOf(lines, pod, run.start);
 };
 /**
-* This function gives the propagation of each node for the target of a run. It is the time from
-* its deletion (or its crash) to the first poll of the rules of the node without it. The change
-* came between the poll before (`after`) and that poll (`by`).
+* This function gives the propagation of each node for the target of a run, or for the replica of
+* the address `ip`. It is the time from its deletion (or its crash) to the first poll of the rules
+* of the node without it. The change came between the poll before (`after`) and that poll (`by`).
 */
-const propagationOf = (run, from) => {
+const propagationOf = (run, from, ip = run.target?.ip) => {
 	const out = {};
-	if (run.target === null || from === null) return out;
+	if (ip === void 0 || from === null) return out;
 	const nodes = [...new Set(run.lines.filter((line) => line.ev === "rules").map((line) => String(line["node"])))].sort();
 	for (const node of nodes) {
-		const drop = dropOf(run.lines, node, `${run.target.ip}:${PORT}`, from);
+		const drop = dropOf(run.lines, node, `${ip}:${PORT}`, from);
 		out[node] = drop === null ? null : {
 			after: drop.prev === null ? null : seconds(sec(Math.max(0, drop.prev - from))),
 			by: seconds(sec(drop.t - from))
@@ -227,6 +247,20 @@ const propagationOf = (run, from) => {
 	}
 	return out;
 };
+/**
+* This function gives each replica of a run that got a deletion after the start of the run, in the
+* sequence of the deletions. A rolling update deletes each old replica in turn.
+*/
+const deletionsOf = (run) => run.replicas.flatMap((replica) => {
+	const deletion = referenceOf(run, "deletion", replica.pod);
+	return deletion === null ? [] : [{
+		pod: replica.pod,
+		intake: referenceOf(run, "intake", replica.pod),
+		deletion
+	}];
+}).sort((a, b) => a.deletion - b.deletion);
+/** This function tells if a prediction is a window for each deleted replica. */
+const eachOf = (prediction) => prediction.kind === "window" && prediction.each === true;
 /** This function gives the facts of a run that each table reads: its failures by outcome and node, and the times of its target. */
 const factsOf = (cell, run) => {
 	const lines = run.lines;
@@ -247,7 +281,8 @@ const factsOf = (cell, run) => {
 			const at = sliceDropOf(lines, "lab", run.target.ip, from);
 			return at === null ? null : sec(at - from);
 		})()),
-		propagation: propagationOf(run, from)
+		propagation: propagationOf(run, from),
+		...eachOf(cell.prediction) ? { deletions: deletionsOf(run) } : {}
 	};
 };
 /** This function judges one run against the prediction of its cell. */
@@ -277,46 +312,64 @@ const checkRun = (cell, run) => {
 			return done(counted.length === 0, counted.length === 0 ? `no ${excluded === void 0 ? "request failed" : `request ended ${excluded.join(", ")}`}` : `${counted.length} ${what}: ${JSON.stringify(facts.outcomes)}`, counted.length);
 		}
 		case "window": {
-			const start = referenceOf(run, prediction.start.from);
-			const end = prediction.end === null ? null : referenceOf(run, prediction.end.from);
-			if (start === null || prediction.end !== null && end === null) return done(false, `the run has no ${start === null ? prediction.start.from : prediction.end.from} reference`, null);
+			const each = prediction.each === true;
 			const opens = secondsOf(prediction.start.band);
 			const closes = prediction.end === null ? null : secondsOf(prediction.end.band);
-			const lo = start + opens.lo * 1e3 - NOISE;
-			const hi = end === null ? Number.POSITIVE_INFINITY : end + closes.hi * 1e3 + NOISE;
+			const droppedAfter = (origin, propagation) => (node) => {
+				const propagated = propagation[node];
+				return origin === null || propagated === void 0 || propagated === null ? null : origin + valueIn(propagated.by, "ms");
+			};
+			const pods = each ? (facts.deletions ?? []).map((deleted) => deleted.pod) : [run.target?.pod];
+			if (pods.length === 0) return done(false, "no replica of the run got a deletion", null);
+			const windows = [];
+			for (const pod of pods) {
+				const start = referenceOf(run, prediction.start.from, pod);
+				const end = prediction.end === null ? null : referenceOf(run, prediction.end.from, pod);
+				if (start === null || prediction.end !== null && end === null) return done(false, `the run has no ${start === null ? prediction.start.from : prediction.end.from} reference${each ? ` for ${pod}` : ""}`, null);
+				const origin = each ? referenceOf(run, "deletion", pod) : facts.deletion ?? (cell.trigger.kind === "crash" ? run.acts.fault : null);
+				const ip = run.replicas.find((replica) => replica.pod === pod)?.ip;
+				windows.push({
+					pod,
+					start,
+					end,
+					lo: start + opens.lo * 1e3 - NOISE,
+					hi: end === null ? Number.POSITIVE_INFINITY : end + closes.hi * 1e3 + NOISE,
+					dropped: droppedAfter(origin, each ? propagationOf(run, origin, ip) : facts.propagation)
+				});
+			}
 			const relevant = prediction.nodes === "paused" ? failures.filter((request) => request.n === run.paused) : failures;
 			const others = prediction.nodes === "paused" ? failures.filter((request) => request.n !== run.paused) : [];
 			const reuse = prediction.afterReset;
-			const late = relevant.filter((request) => request.t < lo || request.t > hi);
+			const late = relevant.filter((request) => windows.every((window) => request.t < window.lo || request.t > window.hi));
 			const reused = reuse === void 0 ? [] : late.filter((request) => reusesReset(request, failures, reuse));
-			const origin = facts.deletion ?? (cell.trigger.kind === "crash" ? run.acts.fault : null);
-			const dropped = (node) => {
-				const propagated = facts.propagation[node];
-				return origin === null || propagated === void 0 || propagated === null ? null : origin + valueIn(propagated.by, "ms");
-			};
-			const tail = late.filter((request) => {
-				const drop = request.t > hi && !reused.includes(request) ? dropped(request.n) : null;
+			const tailOf = (request) => reused.includes(request) ? void 0 : windows.find((window) => {
+				const drop = request.t > window.hi ? window.dropped(request.n) : null;
 				return drop !== null && request.t <= drop + NOISE;
 			});
+			const tail = late.filter((request) => tailOf(request) !== void 0);
 			const outside = late.filter((request) => !reused.includes(request) && !tail.includes(request));
+			const bounds = (window) => `[${sec(window.lo - window.start)}, ${sec(window.hi - window.start)}] s`;
 			const problems = [];
-			if (outside.length > 0) problems.push(`${outside.length} failed requests outside [${sec(lo - start)}, ${sec(hi - start)}] s after ${prediction.start.from}`);
+			if (outside.length > 0) problems.push(each ? `${outside.length} failed requests outside the window of each deleted replica: ${windows.map((window) => `${window.pod} ${bounds(window)}`).join(", ")} after its ${prediction.start.from}` : `${outside.length} failed requests outside ${bounds(windows[0])} after ${prediction.start.from}`);
 			if (others.length > 0) problems.push(`${others.length} failed requests on nodes whose kube-proxy did not pause`);
 			if (prediction.certain && relevant.length === 0) problems.push("no request failed in a window that the law opens");
-			if (prediction.certain && end !== null && relevant.length > 0) {
+			const final = windows.reduce((a, b) => (b.end ?? Number.NEGATIVE_INFINITY) > (a.end ?? Number.NEGATIVE_INFINITY) ? b : a);
+			if (prediction.certain && final.end !== null && relevant.length > 0) {
 				const last = relevant.at(-1).t;
-				const earliest = end + closes.lo * 1e3 - NOISE - SLACK;
+				const earliest = final.end + closes.lo * 1e3 - NOISE - SLACK;
 				if (last < earliest) problems.push(`the last failed request started ${sec(earliest - last)} s before the window can end`);
 			}
 			const inside = relevant.filter((request) => !late.includes(request));
 			const span = inside.length === 0 ? 0 : sec(inside.at(-1).t - inside[0].t);
-			const agrees = reused.length === 0 ? `${relevant.length} failed requests, all in the window (span ${span} s)` : `${inside.length} failed requests in the window (span ${span} s), and ${reused.length} on a port that a reset closed`;
+			const where = each ? `the windows of ${windows.length} deleted replicas` : "the window";
+			const agrees = reused.length === 0 ? `${relevant.length} failed requests, all in ${where} (span ${span} s)` : `${inside.length} failed requests in ${where} (span ${span} s), and ${reused.length} on a port that a reset closed`;
 			const checked = done(problems.length === 0, problems.length === 0 ? agrees : problems.join("; "), relevant.length);
 			if (problems.length > 0 || tail.length === 0) return checked;
 			const last = tail.at(-1);
+			const after = tailOf(last);
 			return {
 				...checked,
-				tail: `${tail.length} failed requests after the window, the last ${sec(last.t - hi + NOISE)} s after it on ${last.n}, before the rules of ${last.n} dropped the target`
+				tail: each ? `${tail.length} failed requests after a window, the last ${sec(last.t - after.hi + NOISE)} s after the window of ${after.pod} on ${last.n}, before the rules of ${last.n} dropped ${after.pod}` : `${tail.length} failed requests after the window, the last ${sec(last.t - after.hi + NOISE)} s after it on ${last.n}, before the rules of ${last.n} dropped the target`
 			};
 		}
 		case "removal": {
@@ -673,7 +726,7 @@ const atBand = (from, band) => {
 const predictionText = (prediction) => {
 	switch (prediction.kind) {
 		case "none": return prediction.outcomes === void 0 ? "no failure" : `no ${prediction.outcomes.join(", ")}`;
-		case "window": return `failures only from ${atBand(prediction.start.from, prediction.start.band)}${prediction.end === null ? "" : ` to ${atBand(prediction.end.from, prediction.end.band)}`}${prediction.nodes === "paused" ? ", on the paused node" : ""}${prediction.certain ? ", at least one" : ""}${prediction.afterReset === void 0 ? "" : `, or a ${prediction.afterReset.outcome} up to ${format(prediction.afterReset.life)} after a reset on its port`}`;
+		case "window": return `failures only from ${atBand(prediction.start.from, prediction.start.band)}${prediction.end === null ? "" : ` to ${atBand(prediction.end.from, prediction.end.band)}`}${prediction.each === true ? ", of each deleted replica" : ""}${prediction.nodes === "paused" ? ", on the paused node" : ""}${prediction.certain ? ", at least one" : ""}${prediction.afterReset === void 0 ? "" : `, or a ${prediction.afterReset.outcome} up to ${format(prediction.afterReset.life)} after a reset on its port`}`;
 		case "removal": return `Service empty in ${showBand(prediction.band)}`;
 		case "leaves": return `the replica leaves in ${showBand(prediction.band)}`;
 		case "restarts": return prediction.band === null ? "no restart" : `each replica SIGTERM in ${showBand(prediction.band)}`;
@@ -930,6 +983,7 @@ const recordOf = (a, main) => {
 				life: a,
 				outcome: z.string()
 			}).optional(),
+			each: z.boolean().optional(),
 			why: z.string()
 		}),
 		z.object({
@@ -964,6 +1018,7 @@ const recordOf = (a, main) => {
 		id: z.string(),
 		name: z.string(),
 		template: z.enum(["T07", "T18"]),
+		stack: z.enum(["service", "grpc"]).optional(),
 		substitutes: z.string().nullable(),
 		mode: z.enum(MODES),
 		workers: count,
@@ -984,7 +1039,7 @@ const recordOf = (a, main) => {
 			liveness: probe,
 			startup: probe.nullable()
 		}),
-		declaration,
+		declaration: declaration.nullable(),
 		workload: z.object({
 			every: time,
 			timeout: time
@@ -1009,7 +1064,12 @@ const recordOf = (a, main) => {
 		sliceEmpty: a.optional(),
 		rules: z.record(z.string(), a.nullable()).optional(),
 		left: a.optional(),
-		signals: z.array(a.nullable()).optional()
+		signals: z.array(a.nullable()).optional(),
+		windows: z.array(z.object({
+			pod: z.string(),
+			intake: z.number().nullable(),
+			deletion: z.number()
+		})).optional()
 	});
 	return {
 		cell,
@@ -1071,6 +1131,6 @@ const readRecord = (data) => {
 	throw new Error(`a kind cell record has the format ${FORMAT} or ${FORMAT_V1}, not ${String(format)}`);
 };
 //#endregion
-export { REDIS as $, deletionOf as A, requestsOf as B, unitOf as C, quantileOf as D, middle as E, kernelCounts as F, sliceDropOf as G, rulesOf as H, outcomesOf as I, transitionOf as J, sliceEmptyOf as K, podsAt as L, emptyOf as M, failuresOf as N, backendsAt as O, intakeOf as P, PORT as Q, podsOf as R, reusesReset as S, medianInterval as T, signalOf as U, restartsAt as V, signalsOf as W, HEALTH_PORT as X, CLIENT as Y, LOG_LABEL as Z, checkRun as _, calibrationOf as a, propagationOf as b, predictionText as c, FORMAT_V1 as d, SERVER as et, judge as f, LAST_SLACK as g, KIND_NOISE as h, readRecord as i, showBand as it, dropOf as j, byNode as k, timeSpread as l, upgradeCell as m, kindCellRecordSchema as n, seconds as nt, cellsTable as o, upgrade as p, slicesOf as q, kindCellRecordV1Schema as r, secondsOf as rt, medianText as s, STAGES as t, bandOf as tt, FORMAT as u, factsOf as v, RELATIVE as w, referenceOf as x, judgeKind as y, recordsOf as z };
+export { HEALTH_PORT as $, byClients as A, podsOf as B, reusesReset as C, middle as D, medianInterval as E, failuresOf as F, signalOf as G, requestsOf as H, intakeOf as I, sliceEmptyOf as J, signalsOf as K, kernelCounts as L, deletionOf as M, dropOf as N, quantileOf as O, emptyOf as P, CLIENT as Q, outcomesOf as R, referenceOf as S, RELATIVE as T, restartsAt as U, recordsOf as V, rulesOf as W, stoppingOf as X, slicesOf as Y, transitionOf as Z, checkRun as _, calibrationOf as a, seconds as at, judgeKind as b, predictionText as c, FORMAT_V1 as d, LOG_LABEL as et, judge as f, LAST_SLACK as g, KIND_NOISE as h, readRecord as i, bandOf as it, byNode as j, backendsAt as k, timeSpread as l, upgradeCell as m, kindCellRecordSchema as n, REDIS as nt, cellsTable as o, secondsOf as ot, upgrade as p, sliceDropOf as q, kindCellRecordV1Schema as r, SERVER as rt, medianText as s, showBand as st, STAGES as t, PORT as tt, FORMAT as u, deletionsOf as v, unitOf as w, propagationOf as x, factsOf as y, podsAt as z };
 
-//# sourceMappingURL=judge-BbTAxRSd.js.map
+//# sourceMappingURL=judge-reWPMVr4.js.map