@mark1russell7/kind 0.2.2 → 0.2.3

package/README.md

@@ -162,6 +162,20 @@ From 0.2.1, a window can be one window for each deleted replica (`each`). A roll
 - The facts of each run give these replicas, with the stamps of their intake and their deletion (`deletions`).
 - `predictionText` adds `of each deleted replica`. A window without `each` stays the window of the target.
 
+From 0.2.3, a run observes until each predicted window can close, and until each request that started in it ended (`observationEnd`).
+
+- The judge closes a window `end.band.hi` + δ after its end reference. A request that starts by then ends at most one timeout of the workload later.
+- Thus a run observes to its latest end reference, plus that band, the timeout and δ (`observationAfter`). With `each`, the end reference of each deleted replica counts.
+- A prediction that is not a window, or a window with no end, keeps the observation of 0.2.2. Such a run observes until the trigger is done and the cell is at rest again.
+- In kind run 40128267098, 10 of 11 runs of a rolling update over gRPC ended too early. The calls in the queue of a channel had not timed out yet, thus only 1 run recorded its failures.
+- The estimate of a cell for the shards adds this observation. The window of a pause of kube-proxy ends up to 122 s after the resume, because of a stale SYN_SENT entry. Thus each run of such a cell observes approximately 2 min more.
+
+From 0.2.3, a window can also give the propagation that it assumes (`propagation`). The judge holds the drop of the replica of each window from the rules of each node to the end of that band ± δ.
+
+- A later drop agrees with the law, but the band is short. The check of the run tells it (`tail`), and the cell is `extend` (D-EXTEND). A node that keeps the replica past the band to the end of the run is such a drop too.
+- An earlier drop only closes the window earlier, thus the judge does not test the start of the band.
+- `predictionText` adds `the rules drop the replica in` and the band.
+
 A preStop sleep that is longer than the grace is a command in the container (`sh -c "sleep 60"`). The API server refuses a sleep action that is longer than the grace.
 
 ## Tests

package/build/index.d.ts

@@ -1,4 +1,4 @@
-import { $ as RecordLine, $t as KindConfig, A as TriggerV1, At as REDIS, B as RunCheck, Bt as KubeletProbe, C as KindCellResult, Ct as slicesOf, D as PredictionV1, Dt as HEALTH_PORT, E as LoadSample, Et as CLIENT, F as KIND_NOISE, Ft as showBand, G as judgeKind, Gt as Replica, H as checkRun, Ht as Prediction, I as KindDecision, It as Band, J as reusesReset, Jt as Template, K as propagationOf, Kt as ResetReuse, L as KindVerdict, Lt as Declaration, M as upgrade, Mt as bandOf, N as upgradeCell, Nt as seconds, O as RunCheckV1, Ot as LOG_LABEL, P as Deleted, Pt as secondsOf, Q as PodLine, Qt as KUBE_PROXY_MODES, R as LAST_SLACK, Rt as KindCell, S as KindCellRecordV1, St as sliceEmptyOf, T as KindVerdictV1, Tt as transitionOf, U as deletionsOf, Ut as RedisDeclaration, V as RunFacts, Vt as PodSettings, W as factsOf, Wt as Reference, X as Endpoint, Xt as TriggerKind, Y as unitOf, Yt as Trigger, Z as Line, Zt as Workload, _ as ClusterManifest, _t as restartsAt, a as STAGES, an as cidrRange, at as byNode, b as FORMAT_V1, bt as signalsOf, c as readRecord, cn as kubeProxyPatch, ct as emptyOf, d as calibrationOf, dn as scenarioProblems, dt as kernelCounts, en as KindScenario, et as Request, f as cellsTable, ft as outcomesOf, g as BandV1, gt as requestsOf, h as timeSpread, ht as recordsOf, i as quantileOf, in as cidrOverlap, it as byClients, j as judge, jt as SERVER, k as RunFactsV1, kt as PORT, l as Judged, ln as nodeNames, lt as failuresOf, m as predictionText, mt as podsOf, n as medianInterval, nn as NODE_IMAGES, nt as SliceLine, o as kindCellRecordSchema, on as goDuration, ot as deletionOf, p as medianText, pt as podsAt, q as referenceOf, qt as Stack, r as middle, rn as NodeImage, rt as backendsAt, s as kindCellRecordV1Schema, sn as kindConfig, st as dropOf, t as RELATIVE, tn as KubeProxyMode, tt as RulesLine, u as TimeSpread, un as scenario, ut as intakeOf, v as EnvironmentManifest, vt as rulesOf, w as KindCellV1, wt as stoppingOf, x as KindCellRecord, xt as sliceDropOf, y as FORMAT, yt as signalOf, z as Propagated, zt as KindRun } from "./judge-Dwa6MqP4.js";
+import { $ as Line, $t as Workload, A as TriggerV1, At as LOG_LABEL, B as RunCheck, Bt as KindCell, C as KindCellResult, Ct as sliceDropOf, D as PredictionV1, Dt as transitionOf, E as LoadSample, Et as stoppingOf, F as KIND_NOISE, Ft as seconds, G as judgeKind, Gt as RedisDeclaration, H as checkRun, Ht as KubeletProbe, I as KindDecision, It as secondsOf, J as propagationOf, Jt as ResetReuse, K as observationAfter, Kt as Reference, L as KindVerdict, Lt as showBand, M as upgrade, Mt as REDIS, N as upgradeCell, Nt as SERVER, O as RunCheckV1, Ot as CLIENT, P as Deleted, Pt as bandOf, Q as Endpoint, Qt as TriggerKind, R as LAST_SLACK, Rt as Band, S as KindCellRecordV1, St as signalsOf, T as KindVerdictV1, Tt as slicesOf, U as deletionsOf, Ut as PodSettings, V as RunFacts, Vt as KindRun, W as factsOf, Wt as Prediction, X as reusesReset, Xt as Template, Y as referenceOf, Yt as Stack, Z as unitOf, Zt as Trigger, _ as ClusterManifest, _t as recordsOf, a as STAGES, an as NodeImage, at as backendsAt, b as FORMAT_V1, bt as rulesOf, c as readRecord, cn as goDuration, ct as deletionOf, d as calibrationOf, dn as nodeNames, dt as failuresOf, en as KUBE_PROXY_MODES, et as PodLine, f as cellsTable, fn as scenario, ft as intakeOf, g as BandV1, gt as podsOf, h as timeSpread, ht as podsAt, i as quantileOf, in as NODE_IMAGES, it as SliceLine, j as judge, jt as PORT, k as RunFactsV1, kt as HEALTH_PORT, l as Judged, ln as kindConfig, lt as dropOf, m as predictionText, mt as outcomesOf, n as medianInterval, nn as KindScenario, nt as Request, o as kindCellRecordSchema, on as cidrOverlap, ot as byClients, p as medianText, pn as scenarioProblems, pt as kernelCounts, q as observationEnd, qt as Replica, r as middle, rn as KubeProxyMode, rt as RulesLine, s as kindCellRecordV1Schema, sn as cidrRange, st as byNode, t as RELATIVE, tn as KindConfig, tt as RecordLine, u as TimeSpread, un as kubeProxyPatch, ut as emptyOf, v as EnvironmentManifest, vt as requestsOf, w as KindCellV1, wt as sliceEmptyOf, x as KindCellRecord, xt as signalOf, y as FORMAT, yt as restartsAt, z as Propagated, zt as Declaration } from "./judge-CctoUEo8.js";
 import { Quantity } from "@mark1russell7/quantity";
 import { CheckFn, Finding, HealthServer, Poll, Rule, Scenario, ServiceBuilder, ServiceDescription, Stage, Timing, TransportImpl, Verdict } from "@mark1russell7/service";
 import { RedisClient } from "@mark1russell7/service/redis";
@@ -416,5 +416,5 @@ export declare const toolPath: (tool: Tool, key?: string, cache?: string) => str
  */
 export declare const ensureTool: (tool: Tool, key?: string, cache?: string) => Promise<string>;
 //#endregion
-export { type Band, type BandV1, CLIENT, type Cluster, type ClusterManifest, type Credentials, type Declaration, type Deleted, type DeletionWindow, Endpoint, type EnvironmentManifest, FORMAT, FORMAT_V1, type FaultPlan, HEALTH_PORT, type Judged, KIND_NOISE, KUBE_PROXY_MODES, type KindCell, type KindCellRecord, type KindCellRecordV1, type KindCellResult, type KindCellV1, type KindConfig, type KindDecision, type KindLab, type KindRun, type KindScenario, type KindVerdict, type KindVerdictV1, type KubeProxyMode, type KubeletProbe, LAST_SLACK, LOG_LABEL, Line, type LoadSample, NODE_IMAGES, type NodeImage, PORT, type Parts, PodLine, type PodSettings, type Prediction, type PredictionV1, type Propagated, type Propagation, REDIS, RELATIVE, RecordLine, type RedisDeclaration, type Reference, type Replica, Request, type ResetReuse, RulesLine, type RunCheck, type RunCheckV1, type RunFacts, type RunFactsV1, type RunOptions, SERVER, STAGES, type ServiceRule, SliceLine, type Stack, type Stamp, type Step, type Target, type Template, type TimeSpread, type Tool, type Trigger, type TriggerKind, type TriggerV1, type Verdicts, type Workload, backendsAt, bandOf, byClients, byNode, calibrationOf, cellsTable, checkRun, cidrOverlap, cidrRange, deletionOf, deletionsOf, dropOf, emptyOf, factsOf, failuresOf, goDuration, intakeOf, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, medianInterval, medianText, middle, nodeNames, outcomesOf, podsAt, podsOf, predictionText, propagationOf, quantileOf, readRecord, recordsOf, referenceOf, requestsOf, restartsAt, reusesReset, rulesOf, scenario, scenarioProblems, seconds, secondsOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, stoppingOf, timeSpread, transitionOf, unitOf, upgrade, upgradeCell };
+export { type Band, type BandV1, CLIENT, type Cluster, type ClusterManifest, type Credentials, type Declaration, type Deleted, type DeletionWindow, Endpoint, type EnvironmentManifest, FORMAT, FORMAT_V1, type FaultPlan, HEALTH_PORT, type Judged, KIND_NOISE, KUBE_PROXY_MODES, type KindCell, type KindCellRecord, type KindCellRecordV1, type KindCellResult, type KindCellV1, type KindConfig, type KindDecision, type KindLab, type KindRun, type KindScenario, type KindVerdict, type KindVerdictV1, type KubeProxyMode, type KubeletProbe, LAST_SLACK, LOG_LABEL, Line, type LoadSample, NODE_IMAGES, type NodeImage, PORT, type Parts, PodLine, type PodSettings, type Prediction, type PredictionV1, type Propagated, type Propagation, REDIS, RELATIVE, RecordLine, type RedisDeclaration, type Reference, type Replica, Request, type ResetReuse, RulesLine, type RunCheck, type RunCheckV1, type RunFacts, type RunFactsV1, type RunOptions, SERVER, STAGES, type ServiceRule, SliceLine, type Stack, type Stamp, type Step, type Target, type Template, type TimeSpread, type Tool, type Trigger, type TriggerKind, type TriggerV1, type Verdicts, type Workload, backendsAt, bandOf, byClients, byNode, calibrationOf, cellsTable, checkRun, cidrOverlap, cidrRange, deletionOf, deletionsOf, dropOf, emptyOf, factsOf, failuresOf, goDuration, intakeOf, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, medianInterval, medianText, middle, nodeNames, observationAfter, observationEnd, outcomesOf, podsAt, podsOf, predictionText, propagationOf, quantileOf, readRecord, recordsOf, referenceOf, requestsOf, restartsAt, reusesReset, rulesOf, scenario, scenarioProblems, seconds, secondsOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, stoppingOf, timeSpread, transitionOf, unitOf, upgrade, upgradeCell };
 //# sourceMappingURL=index.d.ts.map

package/build/index.js

@@ -1,4 +1,4 @@
-import { $ as HEALTH_PORT, A as byClients, B as podsOf, C as reusesReset, D as middle, E as medianInterval, F as failuresOf, G as signalOf, H as requestsOf, I as intakeOf, J as sliceEmptyOf, K as signalsOf, L as kernelCounts, M as deletionOf, N as dropOf, O as quantileOf, P as emptyOf, Q as CLIENT, R as outcomesOf, S as referenceOf, T as RELATIVE, U as restartsAt, V as recordsOf, W as rulesOf, X as stoppingOf, Y as slicesOf, Z as transitionOf, _ as checkRun, a as calibrationOf, at as seconds, b as judgeKind, c as predictionText, d as FORMAT_V1, et as LOG_LABEL, f as judge, g as LAST_SLACK, h as KIND_NOISE, i as readRecord, it as bandOf, j as byNode, k as backendsAt, l as timeSpread, m as upgradeCell, n as kindCellRecordSchema, nt as REDIS, o as cellsTable, ot as secondsOf, p as upgrade, q as sliceDropOf, r as kindCellRecordV1Schema, rt as SERVER, s as medianText, st as showBand, t as STAGES, tt as PORT, u as FORMAT, v as deletionsOf, w as unitOf, x as propagationOf, y as factsOf, z as podsAt } from "./judge-reWPMVr4.js";
+import { $ as transitionOf, A as quantileOf, B as outcomesOf, C as propagationOf, D as RELATIVE, E as unitOf, F as dropOf, G as restartsAt, H as podsOf, I as emptyOf, J as signalsOf, K as rulesOf, L as failuresOf, M as byClients, N as byNode, O as medianInterval, P as deletionOf, Q as stoppingOf, R as intakeOf, S as observationEnd, T as reusesReset, U as recordsOf, V as podsAt, W as requestsOf, X as sliceEmptyOf, Y as sliceDropOf, Z as slicesOf, _ as checkRun, a as calibrationOf, at as SERVER, b as judgeKind, c as predictionText, ct as secondsOf, d as FORMAT_V1, et as CLIENT, f as judge, g as LAST_SLACK, h as KIND_NOISE, i as readRecord, it as REDIS, j as backendsAt, k as middle, l as timeSpread, lt as showBand, m as upgradeCell, n as kindCellRecordSchema, nt as LOG_LABEL, o as cellsTable, ot as bandOf, p as upgrade, q as signalOf, r as kindCellRecordV1Schema, rt as PORT, s as medianText, st as seconds, t as STAGES, tt as HEALTH_PORT, u as FORMAT, v as deletionsOf, w as referenceOf, x as observationAfter, y as factsOf, z as kernelCounts } from "./judge-B3CHoiTW.js";
 import { valueIn } from "@mark1russell7/quantity";
 import { execFile, execFileSync } from "node:child_process";
 import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
@@ -2148,7 +2148,13 @@ const over = (ctx, run, lines) => {
 	if (kind === "app-stall") return lines.some((line) => line.ev === "stall" && line["phase"] === "end");
 	return true;
 };
-/** This function does one run of a cell: at rest, the trigger, the heal, then the observation until the cell is at rest again. */
+/** This function gives the replica that the trigger stopped. For a scale-in or a drain, the controller chooses it. */
+const stoppedOf = (cell, replicas, target, lines) => deletes(cell) ? replicas.find((replica) => lines.some((line) => line.ev === "pod" && line["name"] === replica.pod && line["del"] === true)) ?? target : target;
+/**
+* This function does one run of a cell: at rest, the trigger, the heal, then the observation. The
+* observation lasts until the cell is at rest again, and until each predicted window can close
+* (`observationEnd`).
+*/
 const runOnce = async (ctx, env, index, warmup, ours) => {
 	const { lab, cell, cluster, namespace } = ctx;
 	const plan = planOf(cell);
@@ -2220,6 +2226,21 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
 		replicas,
 		restarts
 	}, lab.lines)) await sleep(250);
+	const until = observationEnd(cell, {
+		start,
+		acts: {
+			fault,
+			heal,
+			resume
+		},
+		target: stoppedOf(cell, replicas, target, lab.lines),
+		replicas,
+		lines: lab.lines
+	});
+	if (until !== null) {
+		const wait = Math.min(until - await lab.now(), limit - Date.now());
+		if (wait > 0) await sleep(wait);
+	}
 	const end = await lab.now();
 	const timedOut = Date.now() >= limit;
 	for (const step of plan.restore) try {
@@ -2230,7 +2251,7 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
 	ctx.expected = cell.pod.replicas;
 	for (const note of notes) ctx.log(`  ${note}`);
 	const lines = lab.take();
-	const stopped = deletes(cell) ? replicas.find((replica) => lines.some((line) => line.ev === "pod" && line["name"] === replica.pod && line["del"] === true)) ?? target : target;
+	const stopped = stoppedOf(cell, replicas, target, lines);
 	if (lines.some((line) => line.ev === "pod" && line["cs"].some((status) => status.rc > 0))) {
 		await cluster.kubectl([
 			"-n",
@@ -2449,13 +2470,16 @@ const runKindCells = async (cells, options) => {
 /** This function gives the estimate of a cell in seconds, for the arithmetic of the shards. */
 const estimate = (cell, environments = 2) => {
 	const hold = cell.trigger.hold === void 0 ? 0 : valueIn(cell.trigger.hold, "s");
-	const each = 25 + ({
+	const trigger = {
 		"liveness-restart": 100,
 		crash: 30,
 		"app-stall": hold + 40,
 		"cpu-throttle": hold + 40,
 		rollout: hold + 50
-	}[cell.trigger.kind] ?? (cell.template === "T18" ? hold + 35 : 15 + valueIn(cell.pod.preStop, "s")));
+	};
+	const after = observationAfter(cell);
+	const observed = after === null ? 0 : hold + valueIn(after, "s");
+	const each = 25 + Math.max(observed, trigger[cell.trigger.kind] ?? (cell.template === "T18" ? hold + 35 : 15 + valueIn(cell.pod.preStop, "s")));
 	return 60 * environments + (cell.runs + environments) * each;
 };
 /**
@@ -2485,6 +2509,6 @@ const shardOf = (cells, index, count) => {
 	return out;
 };
 //#endregion
-export { CLIENT, CONTROLLER, DOCUMENTED, DRAIN_RULES, FLAG_CHECK, FORMAT, FORMAT_V1, HEALTH_PORT, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LOG_LABEL, NODE_IMAGES, O04, O09, O10, O16, PORT, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, STAGES, WATCHDOG, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, checkRun, cidrOverlap, cidrRange, clientObjects, compose, createCluster, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, goDuration, intakeAfter, intakeOf, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, latchBand, lintCell, localPin, medianInterval, medianText, middle, namespaceOf, nodeNames, openKindLab, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, predictionText, probeOf, propagationOf, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverObjects, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, stackOf, stageLabImage, stallRemovalBand, stoppingOf, timeSpread, toolPath, transitionOf, unitOf, upgrade, upgradeCell, verdictsOf, writeResult };
+export { CLIENT, CONTROLLER, DOCUMENTED, DRAIN_RULES, FLAG_CHECK, FORMAT, FORMAT_V1, HEALTH_PORT, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, LAST_SLACK, LOG_LABEL, NODE_IMAGES, O04, O09, O10, O16, PORT, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, STAGES, WATCHDOG, backendsAt, backendsOf, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, checkRun, cidrOverlap, cidrRange, clientObjects, compose, createCluster, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dropOf, emptyOf, ensureTool, estimatedTime, factsOf, failuresOf, goDuration, intakeAfter, intakeOf, judge, judgeKind, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeletBand, labImage, latchBand, lintCell, localPin, medianInterval, medianText, middle, namespaceOf, nodeNames, observationAfter, observationEnd, openKindLab, outcomesOf, pausedWindow, planOf, platformKey, podsAt, podsOf, predictionText, probeOf, propagationOf, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestsOf, restartBand, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverObjects, sha256, shardOf, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, slicesOf, stackOf, stageLabImage, stallRemovalBand, stoppingOf, timeSpread, toolPath, transitionOf, unitOf, upgrade, upgradeCell, verdictsOf, writeResult };
 
 //# sourceMappingURL=index.js.map