@mark1russell7/kind 0.13.3 → 0.14.0

package/build/index.js

@@ -1,4 +1,4 @@
-import { $ as signalsOf, A as HBONE_PORT, At as secondsOf, B as isTunnelState, C as observationEnd, Ct as VIP_ANNOTATION, D as reusesReset, Dt as overshootBand, E as referenceOf, Et as drainBand, F as deletionOf, G as podsOf, H as outcomesOf, I as dropOf, J as restartOf, K as recordsOf, L as emptyOf, M as backendsAt, N as byClients, O as unitOf, Ot as requestBand, P as byNode, Q as signalOf, R as failuresOf, S as observationAfter, St as SERVER, T as propagationSource, Tt as bandOf, U as periodOf, V as kernelCounts, W as podsAt, X as rulesOf, Y as restartsAt, Z as sessionsAt, _ as checkRun, _t as LB_CLASS, a as calibrationOf, at as socketsOf, b as factsOf, bt as PORT, c as predictionText, ct as tunnelsOf, d as FORMAT_V1, dt as RELATIVE, et as sliceDropOf, f as judge, ft as medianInterval, g as LAST_SLACK, gt as HEALTH_PORT, h as KIND_NOISE, ht as CLIENT, i as readRecord, it as slicesOf, j as TCP_STATE, jt as showBand, k as updatesOf, kt as seconds, l as timeSpread, lt as ztunnelPodsOf, m as upgradeCell, mt as quantileOf, n as kindCellRecordSchema, nt as sliceReadyAt, o as cellsTable, ot as stoppingOf, p as upgrade, pt as middle, q as requestsOf, r as kindCellRecordV1Schema, rt as sliceReadyOf, s as medianText, st as transitionOf, t as STAGES, tt as sliceEmptyOf, u as FORMAT, ut as ztunnelReplaced, v as cyclesOf, vt as LOG_LABEL, w as propagationOf, wt as backlogEnd, x as judgeKind, xt as REDIS, y as deletionsOf, yt as OUTSIDE, z as intakeOf } from "./judge-DKrgfhAt.js";
+import { $ as rulesOf, A as judgeReconnects, At as overshootBand, B as emptyOf, C as observationEnd, Ct as PORT, D as reusesReset, Dt as backlogEnd, E as referenceOf, Et as VIP_ANNOTATION, F as backendsAt, G as outcomesOf, H as intakeOf, I as byClients, J as podsOf, K as periodOf, L as byNode, M as triesOf, Mt as seconds, N as HBONE_PORT, Nt as secondsOf, O as unitOf, Ot as bandOf, P as TCP_STATE, Pt as showBand, Q as restartsAt, R as deletionOf, S as observationAfter, St as OUTSIDE, T as propagationSource, Tt as SERVER, U as isTunnelState, V as failuresOf, W as kernelCounts, X as requestsOf, Y as recordsOf, Z as restartOf, _ as checkRun, _t as quantileOf, a as calibrationOf, at as sliceReadyAt, b as factsOf, bt as LB_CLASS, c as predictionText, ct as socketsOf, d as FORMAT_V1, dt as tunnelsOf, et as sessionsAt, f as judge, ft as ztunnelPodsOf, g as LAST_SLACK, gt as middle, h as KIND_NOISE, ht as medianInterval, i as readRecord, it as sliceEmptyOf, j as latestReturn, jt as requestBand, k as updatesOf, kt as drainBand, l as timeSpread, lt as stoppingOf, m as upgradeCell, mt as RELATIVE, n as kindCellRecordSchema, nt as signalsOf, o as cellsTable, ot as sliceReadyOf, p as upgrade, pt as ztunnelReplaced, q as podsAt, r as kindCellRecordV1Schema, rt as sliceDropOf, s as medianText, st as slicesOf, t as STAGES, tt as signalOf, u as FORMAT, ut as transitionOf, v as cyclesOf, vt as CLIENT, w as propagationOf, wt as REDIS, x as judgeKind, xt as LOG_LABEL, y as deletionsOf, yt as HEALTH_PORT, z as dropOf } from "./judge-C6XgY2Pk.js";
 import { valueIn } from "@mark1russell7/quantity";
 import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
@@ -10,7 +10,7 @@ import { gunzipSync } from "node:zlib";
 import { failSpan, lint, lintStop, recoverSpan, service } from "@mark1russell7/service";
 import { createRequire } from "node:module";
 import { tmpdir } from "node:os";
-import { harnessImage, imageOf } from "@mark1russell7/harness";
+import { harnessImage, imageOf, substrateKey, substrateOf, substrateOn } from "@mark1russell7/harness";
 import { redisClient } from "@mark1russell7/service/redis";
 import { DRAIN_RULES, DRAIN_RULES as DRAIN_RULES$1, O04, O09, O10, O16 } from "@mark1russell7/service/rules";
 //#region src/backends.ts
@@ -1375,7 +1375,7 @@ const stepsOf = (cell) => {
 		case "crash": return {
 			inject: [{
 				do: "signal",
-				target: "server",
+				target: cell.pulsar === void 0 ? "server" : "broker",
 				signal: "SIGKILL"
 			}],
 			hold: null,
@@ -1393,6 +1393,24 @@ const stepsOf = (cell) => {
 			restore: []
 		};
 		case "freeze-process":
+			if (cell.vip !== void 0) return {
+				inject: [{
+					do: "signal",
+					target: "speaker",
+					signal: "SIGSTOP"
+				}],
+				hold: needs(trigger.hold, "a hold, the time that the speaker is frozen"),
+				heal: [{
+					do: "signal",
+					target: "speaker",
+					signal: "SIGCONT"
+				}],
+				restore: [{
+					do: "signal",
+					target: "speaker",
+					signal: "SIGCONT"
+				}]
+			};
 			if (cell.template === "T07") return {
 				inject: [
 					{
@@ -1814,7 +1832,7 @@ const deletes = (cell) => [
 	"rolling-update",
 	"node-drain",
 	"scale-in"
-].includes(cell.trigger.kind) || cell.template === "T07" && cell.trigger.kind === "freeze-process";
+].includes(cell.trigger.kind) || cell.template === "T07" && cell.trigger.kind === "freeze-process" && cell.vip === void 0;
 //#endregion
 //#region src/image.ts
 /** This folder is the root of the package, and its `image/` folder has the files of the image. */
@@ -2511,7 +2529,7 @@ const vipOf = (subnet) => {
 		0
 	].map((shift) => Math.floor(address / 2 ** shift) % 256).join(".");
 };
-const sleep$2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const sleep$3 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** This function gives the pods of kube-vip that are not deleted. */
 const kubeVipPods = async (cluster) => {
 	return JSON.parse(await cluster.kubectl([
@@ -2540,7 +2558,7 @@ const until$1 = async (within, check) => {
 	const end = Date.now() + within;
 	let why = await check();
 	while (why !== null && Date.now() < end) {
-		await sleep$2(500);
+		await sleep$3(500);
 		why = await check();
 	}
 	return why;
@@ -2657,8 +2675,8 @@ const vipServed = (cluster, namespace, service, vip, within = 12e4) => until$1(w
 *   a usable speaker (speaker/layer2_controller.go, lines 83–129). A usable speaker is a member of
 *   memberlist that is not dead (internal/speakerlist/speakerlist.go, lines 303–320).
 * - The speakers start memberlist from `DefaultLANConfig` (speakerlist.go, line 72). A file
-*   /etc/metallb/memberlist/config.json replaces that config (lines 77–84). The install makes no
-*   such file, thus each value is the default of hashicorp/memberlist v0.5.1.
+*   /etc/metallb/memberlist/config.json replaces that config (lines 77–84). Without settings of
+*   memberlist, the install makes no such file. Then each value is the default of memberlist v0.5.1.
 * - Each event of memberlist (a join, a leave or an update) calls `ForceSync` (speakerlist.go, lines
 *   339–349). `ForceSync` reconciles each Service at once, with no delay (internal/k8s/k8s.go, lines
 *   181–184, and internal/k8s/controllers/service_controller_reload.go, lines 62–118).
@@ -2719,6 +2737,60 @@ const MEMBERLIST = {
 		unit: "ms"
 	}
 };
+/**
+* These are the values of the config file of memberlist, as `DefaultLANConfig` of v0.5.1 gives them
+* (config.go, lines 302–341). The speaker needs each field of the file, and it reads a time as a
+* duration of Go (speakerlist.go, lines 355–432). The chart mounts the ConfigMap
+* `metallb-memberlist-config` at /etc/metallb/memberlist, if the ConfigMap exists
+* (charts/metallb/templates/speaker.yaml, lines 195–198 and 391–392). The key is `config.json`.
+*/
+const MEMBERLIST_FILE = {
+	TCPTimeout: "10s",
+	IndirectChecks: 3,
+	RetransmitMult: 4,
+	SuspicionMult: 4,
+	SuspicionMaxTimeoutMult: 6,
+	PushPullInterval: "30s",
+	ProbeTimeout: "500ms",
+	ProbeInterval: "1s",
+	DisableTcpPings: false,
+	AwarenessMaxMultiplier: 8,
+	GossipNodes: 3,
+	GossipInterval: "200ms",
+	GossipToTheDeadTime: "30s",
+	GossipVerifyIncoming: true,
+	GossipVerifyOutgoing: true,
+	EnableCompression: true,
+	HandoffQueueDepth: 1024,
+	UDPBufferSize: 1400,
+	QueueCheckInterval: "30s"
+};
+/** This name is the ConfigMap of the config file of memberlist. */
+const MEMBERLIST_CONFIG = "metallb-memberlist-config";
+/** This function gives a time as a duration of Go in whole milliseconds, for example "500ms". */
+const goDuration$1 = (time) => {
+	const ms = valueIn(time, "ms");
+	if (!Number.isInteger(ms) || ms <= 0) throw new RangeError(`a time of memberlist is a whole count of milliseconds above zero, not ${ms} ms`);
+	return `${ms}ms`;
+};
+/** This function gives the config file of memberlist with `settings`. Each other value is the default of `MEMBERLIST_FILE`. */
+const memberlistFileOf = (settings) => {
+	if (!Number.isInteger(settings.suspicionMult) || settings.suspicionMult < 1) throw new RangeError(`SuspicionMult is a whole number of 1 or more, not ${settings.suspicionMult}`);
+	if (valueIn(settings.probeTimeout, "ms") >= valueIn(settings.probeInterval, "ms")) throw new RangeError("the probe timeout of memberlist is less than its probe interval");
+	return {
+		...MEMBERLIST_FILE,
+		ProbeInterval: goDuration$1(settings.probeInterval),
+		ProbeTimeout: goDuration$1(settings.probeTimeout),
+		SuspicionMult: settings.suspicionMult
+	};
+};
+/** This function gives the terms of memberlist with `settings`, or the defaults (`MEMBERLIST`) without them. */
+const memberlistTermsOf = (settings) => settings === void 0 ? MEMBERLIST : {
+	...MEMBERLIST,
+	probeInterval: settings.probeInterval,
+	probeTimeout: settings.probeTimeout,
+	suspicionMult: settings.suspicionMult
+};
 /** These are the gratuitous ARPs of the owner at each reconcile: one at once, then one each `every` until `for` (announcer.go, lines 172–182). */
 const METALLB_SPAM = {
 	every: {
@@ -2829,13 +2901,13 @@ const metallbVipOf = (subnet, owner, nodes) => {
 	}
 	throw new Error(`the subnet ${subnet} has no VIP that ${owner} owns`);
 };
-const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const sleep$2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** This function waits until a condition gives null, and gives null, or the last reason after `within` ms. */
 const until = async (within, check) => {
 	const end = Date.now() + within;
 	let why = await check();
 	while (why !== null && Date.now() < end) {
-		await sleep$1(500);
+		await sleep$2(500);
 		why = await check();
 	}
 	return why;
@@ -2902,8 +2974,12 @@ const holdMetallb = async (cluster, owner, nodes, namespace, service, within = 1
 * and a pool of the VIP. The webhook of the controller checks the pool, thus the function tries the
 * pool again until the webhook answers. A failed install gives the pods of MetalLB and the tail of
 * their logs.
+*
+* With `memberlist`, the ConfigMap of the config file comes before the speakers. Then the function
+* checks that each speaker logged the load of the file (speakerlist.go, line 393). Without it, the
+* function checks that the ConfigMap does not exist.
 */
-const installMetallb = async (cluster, images, nodes, vip) => {
+const installMetallb = async (cluster, images, nodes, vip, memberlist) => {
 	try {
 		for (const node of nodes) await cluster.kubectl([
 			"label",
@@ -2912,24 +2988,66 @@ const installMetallb = async (cluster, images, nodes, vip) => {
 			`${VIP_LABEL}=1`,
 			"--overwrite"
 		]);
+		if (memberlist !== void 0) {
+			await cluster.apply([{
+				apiVersion: "v1",
+				kind: "Namespace",
+				metadata: { name: METALLB_NAMESPACE }
+			}]);
+			await cluster.apply([{
+				apiVersion: "v1",
+				kind: "ConfigMap",
+				metadata: {
+					name: MEMBERLIST_CONFIG,
+					namespace: METALLB_NAMESPACE
+				},
+				data: { "config.json": JSON.stringify(memberlistFileOf(memberlist)) }
+			}]);
+		}
 		await helmInstall(cluster, "metallb", METALLB_CHART, METALLB_NAMESPACE, metallbValues(images), { createNamespace: true });
 		let failure = null;
 		const end = Date.now() + 18e4;
 		do {
 			failure = await cluster.apply(metallbObjects(vip)).then(() => null, (error) => error);
-			if (failure !== null) await sleep$1(2e3);
+			if (failure !== null) await sleep$2(2e3);
 		} while (failure !== null && Date.now() < end);
 		if (failure !== null) throw failure;
-		if ((await cluster.kubectl([
-			"-n",
-			"metallb-system",
-			"get",
-			"configmap",
-			"metallb-memberlist-config",
-			"--ignore-not-found",
-			"-o",
-			"name"
-		])).trim() !== "") throw new Error("the ConfigMap metallb-memberlist-config is there: memberlist would not use its defaults");
+		if (memberlist === void 0) {
+			if ((await cluster.kubectl([
+				"-n",
+				"metallb-system",
+				"get",
+				"configmap",
+				MEMBERLIST_CONFIG,
+				"--ignore-not-found",
+				"-o",
+				"name"
+			])).trim() !== "") throw new Error(`the ConfigMap ${MEMBERLIST_CONFIG} is there: memberlist would not use its defaults`);
+		} else {
+			const loaded = await until(12e4, async () => {
+				const pods = JSON.parse(await cluster.kubectl([
+					"-n",
+					METALLB_NAMESPACE,
+					"get",
+					"pods",
+					"-l",
+					"app.kubernetes.io/component=speaker",
+					"-o",
+					"json"
+				]));
+				if (pods.items.length !== nodes.length) return `${pods.items.length} speakers, not ${nodes.length}`;
+				for (const pod of pods.items) if (!(await cluster.kubectl([
+					"-n",
+					"metallb-system",
+					"logs",
+					pod.metadata.name,
+					"-c",
+					"speaker"
+				]).catch((error) => String(error))).includes("loading memberlist config from file")) return `the speaker ${pod.metadata.name} did not log the load of the config file of memberlist`;
+				return null;
+			});
+			if (loaded !== null) throw new Error(loaded);
+		}
 	} catch (error) {
 		const state = async (args) => (await cluster.kubectl([
 			"-n",
@@ -2954,7 +3072,7 @@ const installMetallb = async (cluster, images, nodes, vip) => {
 	}
 };
 /** These words mark the lines of a speaker that the record keeps: the events of memberlist, the announces and the withdraws. */
-const KEPT = /node event|Marking|Suspect|Refuting|serviceAnnounced|serviceWithdrawn|memberlist/;
+const KEPT$1 = /node event|Marking|Suspect|Refuting|serviceAnnounced|serviceWithdrawn|memberlist/;
 /**
 * This function gives the record of MetalLB after a run: the nodes that announce the Service, and
 * some lines of each speaker. The lines are about memberlist and the announces, with the times of the kubelet.
@@ -2974,7 +3092,7 @@ const metallbRecord = async (cluster, namespace, service) => {
 			"--tail=4000",
 			"--prefix",
 			"--timestamps"
-		]).catch((error) => String(error))).split("\n").filter((line) => KEPT.test(line)).slice(-120).join("\n").slice(-12e3)
+		]).catch((error) => String(error))).split("\n").filter((line) => KEPT$1.test(line)).slice(-600).join("\n").slice(-9e4)
 	};
 };
 //#endregion
@@ -3025,10 +3143,10 @@ const METALLB_ANNOUNCER = {
 	},
 	settle: METALLB_SPAM.for,
 	vipOf: (subnet, owner, nodes) => metallbVipOf(subnet, owner, nodes),
-	install: (cluster, images, _owner, nodes, vip) => installMetallb(cluster, {
+	install: (cluster, images, _owner, nodes, vip, settings) => installMetallb(cluster, {
 		speaker: local(images, "metallb-speaker"),
 		controller: local(images, "metallb-controller")
-	}, nodes, vip),
+	}, nodes, vip, settings?.memberlist),
 	hold: (cluster, owner, nodes, namespace, service) => holdMetallb(cluster, owner, nodes, namespace, service),
 	record: (cluster, namespace, service) => metallbRecord(cluster, namespace, service)
 };
@@ -3041,6 +3159,451 @@ const macsAt = (found, mac, count) => {
 	return wrong.length === 0 ? null : wrong.map((each) => `${each.client} has ${each.mac ?? "no MAC"}`).join(", ");
 };
 //#endregion
+//#region src/pulsar.ts
+/**
+* This module installs Apache Pulsar 5.0.0 in a kind cluster (SYS-56): one ZooKeeper, one bookie and
+* two brokers, from the Helm chart 4.8.0. It also gives the producers of the cell: a Python script
+* in the image of Pulsar, with its client 3.13.0 over the C++ client 4.2.0. Each producer sends to
+* one topic, and a thread of it looks up the owner of that topic at each probe.
+*
+* The fault is a SIGKILL of the JVM of the broker that owns the topic. Each term of the return of a
+* producer comes from a pinned source:
+* - The owner of a bundle is an ephemeral znode of the session of its broker. A lookup gives the
+*   owner from the cache of the ownership, with no check of its life (pulsar-broker
+*   NamespaceService.java at v5.0.0, lines 489–508). Thus each lookup names the stopped broker
+*   until its session expires.
+* - The session timeout is `metadataStoreSessionTimeoutMillis`, 30000 (conf/broker.conf at v5.0.0,
+*   line 187, and the chart, values.yaml line 1474). ZKMetadataStore.java gives it to ZooKeeper (line 105).
+* - The client of ZooKeeper 3.9.6 pings after 9 s with no send (ClientCnxn.java, lines 422, 1184
+*   and 1263–1268). That is half of the read timeout of 2/3 of the session, less 1 s.
+*   The server rounds each expiry up to the next tick of 2000 ms (ExpiryQueue.java, lines 53–54
+*   and 87, and conf/zookeeper.conf of Pulsar, line 21). Thus the session expires 21–32 s after the stop.
+* - The broker that stays then takes the bundle at the next lookup.
+* - The chart starts a broker again only after its old znode is gone (templates/broker-statefulset.yaml,
+*   lines 296–302). Thus the stopped broker cannot take the bundle back first.
+*/
+/** This version is the release of Pulsar of the lab. */
+const PULSAR_VERSION = "5.0.0";
+/** This chart is the Helm chart 4.8.0 of Pulsar. The sha256 is of the file of the release. */
+const PULSAR_CHART = {
+	name: "pulsar",
+	version: "4.8.0",
+	url: "https://downloads.apache.org/pulsar/helm-chart/4.8.0/pulsar-4.8.0.tgz",
+	sha256: "464f4a268463d187c9bc83b365b31de0b4a460533d948a04f846ba9b026068f5"
+};
+/**
+* This image is Pulsar 5.0.0 for linux/amd64, pinned by the digest of the manifest of the platform.
+* It has ZooKeeper 3.9.6, BookKeeper, the broker, Python 3 and the Python client 3.13.0. The lab
+* pulls it by its digest, tags it (`localPin`), and loads it into each node.
+*/
+const PULSAR_IMAGE = "apachepulsar/pulsar:5.0.0@sha256:11e9a885121e16be91da77fdd3cfc3d4cbbece4700271d75e949585dbd3a5a5b";
+/** This namespace holds Pulsar. The release of the chart is `pulsar` too. */
+const PULSAR_NAMESPACE = "pulsar";
+/** This name is the app of the producers of a cell. */
+const PRODUCER = "producer";
+/** This topic is the topic of the producers. */
+const PULSAR_TOPIC = "persistent://public/default/lab";
+/** This URL is the Service of the brokers for the binary protocol. */
+const PULSAR_SERVICE = `pulsar://pulsar-broker.${PULSAR_NAMESPACE}.svc.cluster.local:6650`;
+/** This URL is the lookup of the owner of the topic, on the Service of the brokers. */
+const PULSAR_LOOKUP = `http://pulsar-broker.${PULSAR_NAMESPACE}.svc.cluster.local:8080/lookup/v2/topic/persistent/public/default/lab`;
+/** This time is the session timeout of each broker in ZooKeeper (`metadataStoreSessionTimeoutMillis`). */
+const SESSION = {
+	value: 30,
+	unit: "s"
+};
+/** This time is the tick of ZooKeeper (`tickTime`). The server rounds each expiry up to a tick. */
+const TICK = {
+	value: 2,
+	unit: "s"
+};
+/** This time is the longest idle time of the client of ZooKeeper before its ping: the half of 2/3 of the session, less 1 s. */
+const PING = {
+	value: 9,
+	unit: "s"
+};
+/**
+* This backoff is the backoff of a producer of the C++ client 4.2.0, with the defaults of the Python
+* client 3.13.0:
+* - The initial delay is 100 ms and the most is 60 s (lib/ClientConfigurationImpl.h, lines 39–40).
+* - The mandatory stop is the send timeout less 100 ms, and at least 100 ms (lib/ProducerImpl.cc,
+*   lines 55–57). The send timeout of `create_producer` is 30000 ms (pulsar/__init__.py, line 1015).
+* - The jitter takes 0–9 % of each delay (lib/Backoff.cc, line 54).
+* - A try that fails takes the lookup and the connect, which a reset ends at once. The lab gives it 100 ms.
+*/
+const PRODUCER_BACKOFF = {
+	initial: {
+		value: 100,
+		unit: "ms"
+	},
+	max: {
+		value: 60,
+		unit: "s"
+	},
+	mandatoryStop: {
+		value: 29.9,
+		unit: "s"
+	},
+	jitter: .09,
+	attempt: {
+		value: 100,
+		unit: "ms"
+	}
+};
+/** This function gives the repository and the tag of a local image, for the values of a chart. */
+const repositoryOf = (local) => {
+	const at = local.lastIndexOf(":");
+	if (at <= 0) throw new Error(`the image ${local} has no tag`);
+	return {
+		repository: local.slice(0, at),
+		tag: local.slice(at + 1)
+	};
+};
+/**
+* This function gives the values of the chart: ZooKeeper, one bookie and two brokers, with no
+* proxy, no autorecovery, no toolset and no monitor. The bookie keeps one copy of each entry, and
+* the brokers do not move a bundle by load. Each image is the local image, with no pull.
+*/
+const pulsarValues = (image) => {
+	const { repository, tag } = repositoryOf(image);
+	const noMonitor = { podMonitor: { enabled: false } };
+	return {
+		components: {
+			zookeeper: true,
+			bookkeeper: true,
+			broker: true,
+			autorecovery: false,
+			proxy: false,
+			toolset: false,
+			functions: false,
+			function_worker: false
+		},
+		"victoria-metrics-k8s-stack": { enabled: false },
+		defaultPulsarImageRepository: repository,
+		defaultPulsarImageTag: tag,
+		defaultPullPolicy: "Never",
+		affinity: { anti_affinity: false },
+		volumes: { persistence: false },
+		zookeeper: {
+			replicaCount: 1,
+			statefulsetUpgrade: { enabled: false },
+			...noMonitor
+		},
+		bookkeeper: {
+			replicaCount: 1,
+			...noMonitor
+		},
+		autorecovery: noMonitor,
+		proxy: noMonitor,
+		broker: {
+			replicaCount: 2,
+			statefulsetUpgrade: { enabled: false },
+			...noMonitor,
+			configData: {
+				managedLedgerDefaultEnsembleSize: "1",
+				managedLedgerDefaultWriteQuorum: "1",
+				managedLedgerDefaultAckQuorum: "1",
+				loadBalancerSheddingEnabled: "false",
+				loadBalancerAutoBundleSplitEnabled: "false"
+			}
+		}
+	};
+};
+/**
+* This script is a producer of the cell. Each line that it writes is JSON, with the time of
+* CLOCK_MONOTONIC in milliseconds, the pod and the node:
+* - `send`: one send, at its start, with its time to the answer and its outcome. A send is `ok` within
+*   the timeout, `late` after it, or the name of the result of the client (for example `timeout`).
+* - `lookup`: one lookup of the owner of the topic, with the host of the owner.
+* - `pulsar`: one line of the C++ client about the connection of the producer.
+*/
+const PRODUCER_SCRIPT = `import json, logging, os, sys, threading, time, urllib.request
+import pulsar
+
+POD = os.environ["POD_NAME"]
+NODE = os.environ["NODE_NAME"]
+EVERY = int(os.environ["EVERY_MS"]) / 1000
+TIMEOUT = int(os.environ["TIMEOUT_MS"])
+PROBE = int(os.environ["PROBE_MS"]) / 1000
+LOCK = threading.Lock()
+KEPT = ("Schedule reconnection", "Getting connection from pool", "Created producer", "Connection closed", "Disconnected", "Lookup response", "Failed to", "Finished connecting")
+
+
+def now():
+    return time.monotonic() * 1000
+
+
+def say(**line):
+    text = json.dumps({**line, "p": POD, "n": NODE}, separators=(",", ":"))
+    with LOCK:
+        sys.stdout.write(text + "\\n")
+        sys.stdout.flush()
+
+
+class Relay(logging.Handler):
+    def emit(self, record):
+        message = record.getMessage()
+        if any(word in message for word in KEPT):
+            say(ev="pulsar", t=now(), m=message[:400])
+
+
+def probe():
+    while True:
+        start = now()
+        owner = None
+        outcome = "ok"
+        try:
+            with urllib.request.urlopen(os.environ["LOOKUP"], timeout=1) as answer:
+                data = json.loads(answer.read())
+            url = data.get("brokerUrl") or ""
+            owner = url.split("://", 1)[-1].rsplit(":", 1)[0] or None
+        except Exception as error:
+            outcome = type(error).__name__.lower()
+        say(ev="lookup", t=start, d=round(now() - start, 3), o=outcome, owner=owner)
+        time.sleep(max(0.0, PROBE - (now() - start) / 1000))
+
+
+def answered(start):
+    def done(result, message_id):
+        took = now() - start
+        if result == pulsar.Result.Ok:
+            outcome = "ok" if took <= TIMEOUT else "late"
+        else:
+            outcome = getattr(result, "name", str(result).split(".")[-1]).lower()
+        say(ev="send", t=start, d=round(took, 3), o=outcome)
+    return done
+
+
+logger = logging.getLogger("lab")
+logger.setLevel(logging.INFO)
+logger.propagate = False
+logger.addHandler(Relay())
+threading.Thread(target=probe, daemon=True).start()
+client = pulsar.Client(os.environ["SERVICE"], logger=logger)
+while True:
+    try:
+        producer = client.create_producer(os.environ["TOPIC"])
+        break
+    except Exception as error:
+        say(ev="text", t=now(), text="create_producer: " + str(error)[:200])
+        time.sleep(2)
+at = time.monotonic()
+while True:
+    start = now()
+    try:
+        producer.send_async(b"lab", answered(start))
+    except Exception as error:
+        say(ev="send", t=start, d=0, o="refused-" + type(error).__name__.lower())
+    at += EVERY
+    time.sleep(max(0.0, at - time.monotonic()))
+`;
+/**
+* This function gives the producers of a cell in its namespace. They are the script in a ConfigMap,
+* and a Deployment of `clients` pods in the local image of Pulsar, one on each worker.
+*/
+const pulsarObjects = (cell, namespace, image) => {
+	if (cell.pulsar === void 0) throw new Error(`${cell.name} has no broker`);
+	const env = {
+		SERVICE: PULSAR_SERVICE,
+		TOPIC: PULSAR_TOPIC,
+		LOOKUP: PULSAR_LOOKUP,
+		EVERY_MS: String(valueIn(cell.workload.every, "ms")),
+		TIMEOUT_MS: String(valueIn(cell.workload.timeout, "ms")),
+		PROBE_MS: String(valueIn(cell.pulsar.probe, "ms")),
+		PYTHONUNBUFFERED: "1"
+	};
+	return [{
+		apiVersion: "v1",
+		kind: "ConfigMap",
+		metadata: {
+			name: PRODUCER,
+			namespace
+		},
+		data: { "producer.py": PRODUCER_SCRIPT }
+	}, {
+		apiVersion: "apps/v1",
+		kind: "Deployment",
+		metadata: {
+			name: PRODUCER,
+			namespace,
+			labels: { app: PRODUCER }
+		},
+		spec: {
+			replicas: cell.pulsar.clients,
+			selector: { matchLabels: { app: PRODUCER } },
+			template: {
+				metadata: { labels: {
+					app: PRODUCER,
+					[LOG_LABEL]: "1"
+				} },
+				spec: {
+					terminationGracePeriodSeconds: 0,
+					topologySpreadConstraints: [{
+						maxSkew: 1,
+						topologyKey: "kubernetes.io/hostname",
+						whenUnsatisfiable: "DoNotSchedule",
+						labelSelector: { matchLabels: { app: PRODUCER } }
+					}],
+					volumes: [{
+						name: "script",
+						configMap: { name: PRODUCER }
+					}],
+					containers: [{
+						name: PRODUCER,
+						image,
+						imagePullPolicy: "Never",
+						command: ["python3", "/lab/producer.py"],
+						volumeMounts: [{
+							name: "script",
+							mountPath: "/lab"
+						}],
+						env: [
+							...Object.entries(env).map(([name, value]) => ({
+								name,
+								value
+							})),
+							{
+								name: "POD_NAME",
+								valueFrom: { fieldRef: { fieldPath: "metadata.name" } }
+							},
+							{
+								name: "NODE_NAME",
+								valueFrom: { fieldRef: { fieldPath: "spec.nodeName" } }
+							}
+						]
+					}]
+				}
+			}
+		}
+	}];
+};
+/** This function gives the pods of the brokers, in the sequence of their names. */
+const brokersOf = async (cluster) => {
+	return JSON.parse(await cluster.kubectl([
+		"-n",
+		PULSAR_NAMESPACE,
+		"get",
+		"pods",
+		"-l",
+		"component=broker",
+		"-o",
+		"json"
+	])).items.filter((pod) => pod.metadata.deletionTimestamp === void 0).map((pod) => ({
+		pod: pod.metadata.name,
+		node: pod.spec.nodeName ?? "",
+		ip: pod.status?.podIP ?? "",
+		ready: (pod.status?.containerStatuses ?? []).length > 0 && pod.status.containerStatuses.every((status) => status.ready)
+	})).sort((a, b) => a.pod.localeCompare(b.pod));
+};
+/** This function gives the broker that a host of a lookup names. Its pod is the first label of the host, or its address is the host. */
+const brokerOfHost = (brokers, host) => brokers.find((broker) => host.split(".")[0] === broker.pod || host === broker.ip);
+/**
+* This function tells why the producers are not at rest, or gives the host of the owner of the
+* topic. At rest, each of `clients` producers had an answer in the last second and no failure in
+* the last 2 s. The last lookup of each one is in the last 2 s, and each names the same owner.
+*/
+const ownerAtRest = (state, clients) => {
+	if (state.sends.size !== clients) return `${state.sends.size} producers send, not ${clients}`;
+	for (const [pod, send] of state.sends) {
+		if (state.last - send.ok > 1e3) return `the producer ${pod} has no answer in the last second`;
+		if (state.last - send.failed < 2e3) return `the producer ${pod} failed in the last 2 s`;
+	}
+	const owners = /* @__PURE__ */ new Set();
+	for (const pod of state.sends.keys()) {
+		const seen = state.owners.get(pod);
+		if (seen === void 0 || seen.owner === null || state.last - seen.t > 2e3) return `the producer ${pod} has no lookup with an owner in the last 2 s`;
+		owners.add(seen.owner);
+	}
+	if (owners.size !== 1) return `the lookups name ${[...owners].join(" and ")}`;
+	return { owner: [...owners][0] };
+};
+const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/**
+* This function waits until each broker is ready and the producers are at rest (`ownerAtRest`). It
+* gives the broker that owns the topic and the host that the lookups name, or why it could not in `within` ms.
+*/
+const holdPulsar = async (cluster, state, clients, within = 3e5) => {
+	const end = Date.now() + within;
+	let why = "the hold did not start";
+	do {
+		const brokers = await brokersOf(cluster);
+		const busy = brokers.find((broker) => !broker.ready);
+		const rest = ownerAtRest(state(), clients);
+		if (brokers.length !== 2) why = `${brokers.length} brokers, not 2`;
+		else if (busy !== void 0) why = `the broker ${busy.pod} is not ready`;
+		else if (typeof rest === "string") why = rest;
+		else {
+			const broker = brokerOfHost(brokers, rest.owner);
+			if (broker !== void 0) return {
+				broker,
+				host: rest.owner
+			};
+			why = `the owner ${rest.owner} is no broker`;
+		}
+		await sleep$1(500);
+	} while (Date.now() < end);
+	return why;
+};
+/**
+* This function installs Pulsar in a cluster, with its image on each node. A failed install gives
+* the pods of Pulsar and the tail of their logs.
+*/
+const installPulsar = async (cluster, image) => {
+	try {
+		await helmInstall(cluster, "pulsar", PULSAR_CHART, PULSAR_NAMESPACE, pulsarValues(image), { createNamespace: true });
+	} catch (error) {
+		const state = async (args) => (await cluster.kubectl([
+			"-n",
+			PULSAR_NAMESPACE,
+			...args
+		]).catch((why) => String(why))).slice(-4e3);
+		const pods = await state([
+			"get",
+			"pods",
+			"-o",
+			"wide"
+		]);
+		const logs = await state([
+			"logs",
+			"-l",
+			"app=pulsar",
+			"--tail=30",
+			"--prefix",
+			"--all-containers"
+		]);
+		throw new Error(`the install of Pulsar failed: ${String(error)}\nthe pods of Pulsar:\n${pods}\nthe logs:\n${logs}`);
+	}
+};
+/** These words mark the lines of a broker that the record keeps: the sessions, the leader, the ownership and the topics. */
+const KEPT = /[Ss]ession|[Ee]xpired|[Ll]eader|[Oo]wnership|[Bb]undle|Loaded|[Ll]oading topic|Created new producer|Disconnect|Closed connection|znode|ManagedLedger.*(open|recover)/;
+/**
+* This function gives the record of Pulsar after a run. It has the brokers, and the kept lines of
+* each broker and of its last container, with the times of the kubelet.
+*/
+const pulsarRecord = async (cluster) => {
+	const brokers = await brokersOf(cluster);
+	const logs = {};
+	for (const broker of brokers) for (const previous of [false, true]) {
+		const text = await cluster.kubectl([
+			"-n",
+			PULSAR_NAMESPACE,
+			"logs",
+			broker.pod,
+			"-c",
+			"pulsar-broker",
+			"--since=300s",
+			"--tail=6000",
+			"--timestamps",
+			...previous ? ["--previous"] : []
+		]).catch((error) => String(error));
+		logs[`${broker.pod}${previous ? ", the last container" : ""}`] = text.split("\n").filter((line) => KEPT.test(line)).slice(-80).join("\n").slice(-8e3);
+	}
+	return {
+		brokers,
+		logs
+	};
+};
+//#endregion
 //#region src/lds.ts
 /**
 * This filter of RE2 selects the two counters of the listener manager of Envoy that an update of LDS
@@ -4187,9 +4750,11 @@ const hasRedis = (cell) => cell.declaration !== null && cell.declaration.redis !
 * This function gives the key of the cluster of a cell: its kube-proxy mode, its node count, and the
 * flavor of its mesh. A zone loss has clusters of its own, with a zone on each node.
 */
-const clusterKey = (cell) => (cell.mesh !== void 0 ? `${cell.mode}-${cell.workers}-${cell.mesh.flavor}` : zoned(cell) ? `${cell.mode}-${cell.workers}-zones` : `${cell.mode}-${cell.workers}`) + (cell.vip === void 0 ? "" : cell.vip.announcer === "metallb" ? "-vip-mlb" : "-vip");
-/** This function tells if a cell needs a cluster with zones: a zone loss. */
-const zoned = (cell) => cell.trigger.kind === "zone-loss";
+const clusterKey = (cell) => (cell.mesh !== void 0 ? `${cell.mode}-${cell.workers}-${cell.mesh.flavor}` : zoned(cell) ? `${cell.mode}-${cell.workers}-zones` : `${cell.mode}-${cell.workers}`) + (cell.vip === void 0 ? "" : cell.vip.announcer !== "metallb" ? "-vip" : cell.vip.memberlist === void 0 ? "-vip-mlb" : `-mlb${valueIn(cell.vip.memberlist.probeInterval, "ms")}x${cell.vip.memberlist.suspicionMult}`) + (cell.pulsar === void 0 ? "" : "-pul");
+/** This function gives the name of the cluster of a cell in an environment. Kind permits 32 characters at most: lowercase letters, digits and dashes. */
+const clusterNameOf = (cell, env) => `lab-${clusterKey(cell)}-${env}`;
+/** This function tells if a cell needs a cluster with zones. A zone loss needs one, and so does a cell with settings of memberlist, that shares the cluster of its zone loss. */
+const zoned = (cell) => cell.trigger.kind === "zone-loss" || cell.vip?.memberlist !== void 0;
 /**
 * This tracker keeps the latest state of the cell from the lines: each pod, each slice and each
 * client. It also has, for each node, the backends of its rules and the hosts of its client sidecar.
@@ -4202,7 +4767,13 @@ var Tracker = class {
 	clients = /* @__PURE__ */ new Map();
 	/** This map is the backend of the last answer of each client, by client. */
 	answers = /* @__PURE__ */ new Map();
+	/** This map is the last send of each producer of a broker that got an answer, and the last that failed. */
+	sends = /* @__PURE__ */ new Map();
+	/** This map is the last lookup of the owner of the topic of each producer, and the host that it names. */
+	owners = /* @__PURE__ */ new Map();
 	last = 0;
+	/** This time is the last line of a producer. */
+	lastSend = 0;
 	see = (line) => {
 		if (typeof line.t === "number" && line.t > this.last && line.ev === "req") this.last = line.t;
 		if (line.ev === "pod") {
@@ -4225,6 +4796,23 @@ var Tracker = class {
 			else client.failed = Math.max(client.failed, request.t);
 			this.clients.set(request.p, client);
 			if (request.o === "ok" && request.b !== void 0) this.answers.set(request.p, request.b);
+		} else if (line.ev === "send") {
+			const pod = String(line["p"]);
+			const send = this.sends.get(pod) ?? {
+				ok: 0,
+				failed: 0
+			};
+			if (line["o"] === "ok") send.ok = Math.max(send.ok, line.t);
+			else send.failed = Math.max(send.failed, line.t);
+			this.sends.set(pod, send);
+			this.lastSend = Math.max(this.lastSend, line.t);
+		} else if (line.ev === "lookup") {
+			const pod = String(line["p"]);
+			const owner = line["o"] === "ok" && typeof line["owner"] === "string" ? line["owner"] : null;
+			if (line.t >= (this.owners.get(pod)?.t ?? 0)) this.owners.set(pod, {
+				t: line.t,
+				owner
+			});
 		}
 	};
 	servers() {
@@ -4405,6 +4993,18 @@ const deploy = async (lab, cluster, cell, namespace, images, vip) => {
 		`daemonset/${CLIENT}`,
 		"--timeout=300s"
 	]);
+	if (cell.pulsar !== void 0) {
+		if (images.pulsar === void 0) throw new Error(`${cell.name}: the cluster has no image of Pulsar`);
+		await cluster.apply(pulsarObjects(cell, namespace, images.pulsar));
+		await cluster.kubectl([
+			"-n",
+			namespace,
+			"rollout",
+			"status",
+			`deployment/${PRODUCER}`,
+			"--timeout=300s"
+		]);
+	}
 	if (cell.vip !== void 0) {
 		if (vip === null) throw new Error(`${cell.name}: the cluster has no VIP`);
 		const served = await vipServed(cluster, namespace, "lab", vip);
@@ -4488,7 +5088,7 @@ const probeThePods = async (ctx) => {
 	if (key === ctx.probed) return;
 	const netns = [];
 	for (const pod of pods) {
-		const container = pod.labels["app"] === "lab" ? "app" : pod.labels["app"] === "client" ? CLIENT : REDIS;
+		const container = pod.labels["app"] === "lab" ? "app" : pod.labels["app"] === "client" ? CLIENT : pod.labels["app"] === "producer" ? PRODUCER : REDIS;
 		const pid = await ctx.lab.pidOf(pod.node, ctx.namespace, pod.name, container);
 		netns.push(await ctx.lab.netnsOf(pod.node, pid));
 	}
@@ -4671,6 +5271,35 @@ const doStep = async (ctx, step, replicas, background) => {
 			pid: await lab.pidOf(target.node, namespace, target.pod, "app"),
 			pod: null
 		};
+		if (which === "speaker") {
+			const owner = lostNodeOf(cluster);
+			const pod = JSON.parse(await cluster.kubectl([
+				"-n",
+				METALLB_NAMESPACE,
+				"get",
+				"pods",
+				"-l",
+				"app.kubernetes.io/component=speaker",
+				"--field-selector",
+				`spec.nodeName=${owner}`,
+				"-o",
+				"json"
+			])).items[0];
+			if (pod === void 0) throw new Error(`no speaker of MetalLB is on ${owner}`);
+			return {
+				node: owner,
+				pid: await lab.pidOf(owner, METALLB_NAMESPACE, pod.metadata.name, "speaker"),
+				pod: null
+			};
+		}
+		if (which === "broker") {
+			if (ctx.broker === null) throw new Error(`${ctx.cell.name}: the run has no broker that owns the topic`);
+			return {
+				node: ctx.broker.node,
+				pid: await lab.pidOf(ctx.broker.node, PULSAR_NAMESPACE, ctx.broker.pod, "pulsar-broker"),
+				pod: null
+			};
+		}
 		if (which === "redis") {
 			const redis = [...ctx.tracker.pods.values()].find((pod) => pod.labels["app"] === "redis" && !pod.del);
 			return {
@@ -4719,7 +5348,8 @@ const doStep = async (ctx, step, replicas, background) => {
 		}
 		case "signal": {
 			if (step.target === "clients") return signalClients(ctx, step.signal);
-			const at = await where(step.target);
+			const at = (step.target === "speaker" && step.signal === "SIGCONT" ? ctx.speaker : void 0) ?? await where(step.target);
+			if (step.target === "speaker") ctx.speaker = step.signal === "SIGSTOP" ? at : void 0;
 			return lab.act(at.node, {
 				op: "signal",
 				pid: at.pid,
@@ -4878,6 +5508,19 @@ const recordVip = async (ctx) => {
 		macs
 	});
 };
+/** This function records the brokers after a run, with the kept lines of each broker. It stamps the wall clock and the monotonic clock. */
+const recordPulsar = async (ctx) => {
+	const wall = Date.now();
+	const t = Number(process.hrtime.bigint()) / 1e6;
+	ctx.lab.record({
+		ev: "pulsar-record",
+		t,
+		wall,
+		version: PULSAR_VERSION,
+		owner: ctx.broker?.pod ?? null,
+		...await pulsarRecord(ctx.cluster)
+	});
+};
 /** This function puts the replicas of a node first, and keeps the sequence of the others. */
 const lostFirst = (replicas, node) => [...replicas.filter((replica) => replica.node === node), ...replicas.filter((replica) => replica.node !== node)];
 /** This function tells if a run is over: the trigger is done, and the cell is at rest again. */
@@ -4957,10 +5600,25 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
 		await sleep(valueIn(announcer.settle, "ms") + valueIn(cell.workload.timeout, "ms") + 1e3);
 	}
 	if (await steady(ctx, 3e5) !== null) return empty("the cell was not at rest after the wait for a quiet VM", await lab.now());
+	ctx.broker = null;
+	if (cell.pulsar !== void 0) {
+		const tracker = ctx.tracker;
+		const held = await holdPulsar(cluster, () => ({
+			owners: tracker.owners,
+			sends: tracker.sends,
+			last: tracker.lastSend
+		}), cell.pulsar.clients);
+		if (typeof held === "string") return empty(`the producers were not at rest: ${held}`, await lab.now());
+		ctx.broker = {
+			pod: held.broker.pod,
+			node: held.broker.node,
+			host: held.host
+		};
+	}
 	const replicas = zoneLoss ? lostFirst(replicasOf(ctx), lostNodeOf(cluster)) : replicasOf(ctx);
 	if (zoneLoss && cell.vip === void 0 && replicas[0]?.node !== lostNodeOf(cluster)) return empty(`no replica is on the node of the lost zone, ${lostNodeOf(cluster)}`, await lab.now());
 	const restarts = new Map(ctx.tracker.servers().map((pod) => [pod.name, pod.cs.reduce((sum, status) => sum + status.rc, 0)]));
-	const target = cell.vip !== void 0 || hasRedis(cell) && [
+	const target = cell.vip !== void 0 || cell.pulsar !== void 0 || hasRedis(cell) && [
 		"rst",
 		"blackhole",
 		"freeze-process",
@@ -4968,7 +5626,7 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
 		"rollout",
 		"latency"
 	].includes(cell.trigger.kind) ? null : replicas[0];
-	const paused = cell.template === "T07" && cell.trigger.kind === "freeze-process" ? cluster.nodes[1] ?? cluster.nodes[0] : zoneLoss ? lostNodeOf(cluster) : null;
+	const paused = cell.template === "T07" && cell.trigger.kind === "freeze-process" && cell.vip === void 0 ? cluster.nodes[1] ?? cluster.nodes[0] : zoneLoss ? lostNodeOf(cluster) : null;
 	const cleared = await lab.forgetHandshakes(ctx.clusterIp, 80);
 	lab.take();
 	for (const key of Object.keys(lab.dropped)) delete lab.dropped[key];
@@ -5061,6 +5719,7 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
 	const end = await lab.now();
 	const timedOut = Date.now() >= limit;
 	if (cell.vip !== void 0) await recordVip(ctx).catch((error) => notes.push(`the record of the VIP failed: ${String(error)}`));
+	if (cell.pulsar !== void 0) await recordPulsar(ctx).catch((error) => notes.push(`the record of Pulsar failed: ${String(error)}`));
 	for (const step of plan.restore) try {
 		await doStep(ctx, step, replicas, background);
 	} catch (error) {
@@ -5103,6 +5762,7 @@ const runOnce = async (ctx, env, index, warmup, ours) => {
 		},
 		target: stopped,
 		paused,
+		...ctx.broker === null ? {} : { broker: ctx.broker.host },
 		replicas,
 		void: null,
 		lines,
@@ -5147,7 +5807,8 @@ const writeResult = (out, result) => {
 		voided: verdict.voided,
 		failed: verdict.failed,
 		median: verdict.median,
-		why: verdict.why
+		why: verdict.why,
+		substrate: substrateKey(substrateOf(result.manifests[0] ?? {}, result.at, "E2"))
 	})}\n`, { flag: "a" });
 };
 /**
@@ -5185,19 +5846,21 @@ const runKindCells = async (cells, options) => {
 		for (const [name, ref] of Object.entries(announcer.images)) pinned[name] = await localPin(ref);
 		announcerImages.set(announcer.name, pinned);
 	}
+	const pulsarImage = cells.some((cell) => cell.pulsar !== void 0) ? await localPin(PULSAR_IMAGE) : void 0;
 	const images = {
 		lab: image,
 		redis: redisImage,
 		grpc: grpcImage,
-		mesh: meshImages
+		mesh: meshImages,
+		pulsar: pulsarImage
 	};
 	const groups = /* @__PURE__ */ new Map();
 	for (const cell of cells) groups.set(clusterKey(cell), [...groups.get(clusterKey(cell)) ?? [], cell]);
 	const results = [];
-	for (const [key, group] of groups) {
+	for (const group of groups.values()) {
 		const partial = /* @__PURE__ */ new Map();
 		for (let env = 0; env < environments; env += 1) {
-			const s = scenario(`lab-${key}-${env}`, group[0].mode, group[0].workers, void 0, zoned(group[0]));
+			const s = scenario(clusterNameOf(group[0], env), group[0].mode, group[0].workers, void 0, zoned(group[0]));
 			const dir = join(options.work ?? join(tmpdir(), "systems-kind-work"), s.name);
 			const mesh = group[0].mesh;
 			options.log(`cluster ${s.name}: ${s.dataplane}, ${s.workers} workers${mesh === void 0 ? "" : `, Istio ${mesh.flavor}`}`);
@@ -5238,9 +5901,16 @@ const runKindCells = async (cells, options) => {
 					for (const each of Object.values(pinned)) await cluster.load(each);
 					const began = Date.now();
 					vip = announcer.vipOf((await vipNetworkOf(cluster.network)).subnet, lostNodeOf(cluster), workersOf(cluster));
-					await announcer.install(cluster, pinned, lostNodeOf(cluster), workersOf(cluster), vip);
+					await announcer.install(cluster, pinned, lostNodeOf(cluster), workersOf(cluster), vip, group[0].vip);
 					options.log(`  ${announcer.name} installed in ${Math.round((Date.now() - began) / 100) / 10} s, the VIP ${vip}`);
 				}
+				if (group[0].pulsar !== void 0) {
+					if (pulsarImage === void 0) throw new Error(`${group[0].name}: the run has no image of Pulsar`);
+					await cluster.load(pulsarImage);
+					const began = Date.now();
+					await installPulsar(cluster, pulsarImage);
+					options.log(`  Pulsar ${PULSAR_VERSION} installed in ${Math.round((Date.now() - began) / 100) / 10} s`);
+				}
 				lab = await openKindLab(cluster, image);
 				const { GenericContainer } = await import("testcontainers");
 				const Shared = class extends GenericContainer {
@@ -5279,7 +5949,8 @@ const runKindCells = async (cells, options) => {
 							probed: "",
 							listened: "",
 							expected: cell.pod.replicas,
-							pids: /* @__PURE__ */ new Map()
+							pids: /* @__PURE__ */ new Map(),
+							broker: null
 						};
 						const share = Math.ceil(cell.runs / environments) - (env < cell.runs % environments || cell.runs % environments === 0 ? 0 : 1);
 						const runs = [await runOnce(ctx, env, -1, true, ours)];
@@ -5294,13 +5965,15 @@ const runKindCells = async (cells, options) => {
 							env,
 							scenario: s,
 							cluster: manifest,
+							substrate: substrateOn("E2"),
 							images: {
 								lab: image,
 								redis: redisImage,
 								...stackOf(cell) === "grpc" ? { grpc: grpcImage } : {},
 								...cell.mesh === void 0 ? {} : Object.fromEntries(MESH_IMAGES[cell.mesh.flavor].map((name) => [`istio-${name}`, ISTIO_IMAGES[name]])),
 								...s.cni === "cilium" ? Object.fromEntries(Object.entries(CILIUM_IMAGES).map(([name, ref]) => [`cilium-${name}`, ref])) : {},
-								...cell.vip === void 0 ? {} : announcerOf(cell).images
+								...cell.vip === void 0 ? {} : announcerOf(cell).images,
+								...cell.pulsar === void 0 ? {} : { pulsar: PULSAR_IMAGE }
 							}
 						});
 						entry.loads.push(...loads);
@@ -5368,14 +6041,17 @@ const estimate = (cell, environments = 2) => {
 		"ztunnel-restart": 40,
 		"certificate-expiry": hold + 20,
 		"cannot-serve": hold + 20,
-		"zone-loss": hold + 45
+		"zone-loss": hold + 45,
+		...cell.vip === void 0 ? {} : { "freeze-process": hold + 20 }
 	};
 	const after = observationAfter(cell);
 	const observed = after === null ? 0 : hold + valueIn(after, "s");
 	const closes = cell.prediction.kind === "ends" && cell.prediction.band !== null ? valueIn(cell.prediction.band.hi, "s") + 3 : 0;
 	const quiet = cell.trigger.quiet === true ? hold + 10 : 0;
-	const each = 25 + (cell.vip === void 0 ? 0 : 30) + Math.max(observed, closes, quiet, trigger[cell.trigger.kind] ?? (cell.template === "T18" ? hold + 35 : 15 + valueIn(cell.pod.preStop, "s")));
-	return ((cell.mesh === void 0 ? 60 : 180) + (cell.vip === void 0 ? 0 : 60)) * environments + (cell.runs + environments) * each;
+	const move = cell.vip === void 0 ? 0 : 30;
+	const broker = cell.pulsar === void 0 ? 0 : 150;
+	const each = 25 + move + broker + Math.max(observed, closes, quiet, trigger[cell.trigger.kind] ?? (cell.template === "T18" ? hold + 35 : 15 + valueIn(cell.pod.preStop, "s")));
+	return ((cell.mesh === void 0 ? 60 : 180) + (cell.vip === void 0 ? 0 : 60) + (cell.pulsar === void 0 ? 0 : 240)) * environments + (cell.runs + environments) * each;
 };
 /**
 * This function gives an estimate of the time of a cell: its setup, then each run and each
@@ -5406,6 +6082,6 @@ const shardOf = (cells, index, count) => {
 	return out;
 };
 //#endregion
-export { ACCESS_LOG_FORMAT, ANNOUNCE_ANNOTATION, CHANNEL_USE, CILIUM_CHART, CILIUM_CONFIG_KEYS, CILIUM_IMAGES, CILIUM_VERSION, CILIUM_WORKLOADS, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DATAPLANES, DNS_PORT, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, HELM, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, KUBE_VIP, KUBE_VIP_ANNOUNCER, KUBE_VIP_ARP_RATE, KUBE_VIP_ELECTION, KUBE_VIP_IMAGE, KUBE_VIP_LEASE, KUBE_VIP_VERSION, LAST_SLACK, LB_CLASS, LDS_EVERY, LDS_FILTER, LOG_LABEL, MEMBERLIST, MESH_IMAGES, MESH_WORKLOADS, METALLB_ANNOTATION, METALLB_ANNOUNCER, METALLB_CHART, METALLB_IMAGES, METALLB_NAMESPACE, METALLB_POOL, METALLB_SPAM, METALLB_VERSION, MIDDLEBOX, NETEM_DRAIN, NETNS_DEFAULTS, NODE_IMAGES, NODE_LIFECYCLE, O04, O09, O10, O16, OUTSIDE, POD_LINK, POLICY, PORT, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, SPREAD_LEAD, STAGES, SYSCTL, TCP_STATE, VIP_ANNOTATION, VIP_LABEL, WATCHDOG, ZONE_KEPT, ZONE_LABEL, ZONE_LOST, ZTUNNEL_PODS, announcementsOf, announcerOf, arpMacOf, backendsAt, backendsOf, backlogEnd, bandOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, certificateArgs, checkRun, cidrOverlap, cidrRange, ciliumBackendsOf, ciliumMode, ciliumValues, clientObjects, clusterFilter, compose, createCluster, cyclesOf, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dnsCommands, drainBand, dropOf, emptyOf, ensureChart, ensurePinnedChart, ensureTool, estimatedTime, factsOf, failuresOf, faultSpan, followsZtunnel, goDuration, helmInstall, holdMetallb, holdVip, hostsOf, installCilium, installKubeVip, installMesh, installMetallb, intakeAfter, intakeOf, isKubeProxyMode, isTunnelState, istioOperator, judge, judgeKind, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeVipObjects, kubeVipPods, kubeVipReady, kubeletBand, labImage, labSource, latchBand, leafCertificates, learnWindow, lintCell, listenerCountsOf, localPin, macsAt, medianInterval, medianText, meshKey, metallbObjects, metallbRecord, metallbValues, metallbVipOf, middle, middleboxCommands, namesService, namespaceObject, namespaceOf, nodeLossBand, nodeNames, nodeReturnBand, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, overshootBand, ownerOrder, pausedWindow, periodOf, placementOf, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, sessionsAt, setZtunnelEnv, sha256, shardOf, sharedGate, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, speakersReady, stackOf, stageLabImage, stallRemovalBand, stopOf, stoppingOf, suspicionOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, vipLeaseOf, vipNetworkOf, vipOf, vipServed, vipService, writeResult, zoneOf, ztunnelCertificates, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
+export { ACCESS_LOG_FORMAT, ANNOUNCE_ANNOTATION, CHANNEL_USE, CILIUM_CHART, CILIUM_CONFIG_KEYS, CILIUM_IMAGES, CILIUM_VERSION, CILIUM_WORKLOADS, CLIENT, CLIENT_ACCOUNT, CLOSE_LEAD, CONTROLLER, DATAPLANES, DNS_PORT, DOCUMENTED, DRAIN_RULES, EDS_EVERY, FLAG_CHECK, FORMAT, FORMAT_V1, HBONE_PORT, HEALTH_PORT, HELM, ISTIOCTL, ISTIO_IMAGES, ISTIO_VERSION, KEPT_KERNEL, KIND, KIND_NOISE, KUBECTL, KUBE_PROXY_MODES, KUBE_VIP, KUBE_VIP_ANNOUNCER, KUBE_VIP_ARP_RATE, KUBE_VIP_ELECTION, KUBE_VIP_IMAGE, KUBE_VIP_LEASE, KUBE_VIP_VERSION, LAST_SLACK, LB_CLASS, LDS_EVERY, LDS_FILTER, LOG_LABEL, MEMBERLIST, MEMBERLIST_FILE, MESH_IMAGES, MESH_WORKLOADS, METALLB_ANNOTATION, METALLB_ANNOUNCER, METALLB_CHART, METALLB_IMAGES, METALLB_NAMESPACE, METALLB_POOL, METALLB_SPAM, METALLB_VERSION, MIDDLEBOX, NETEM_DRAIN, NETNS_DEFAULTS, NODE_IMAGES, NODE_LIFECYCLE, O04, O09, O10, O16, OUTSIDE, PING, POD_LINK, POLICY, PORT, PRODUCER, PRODUCER_BACKOFF, PRODUCER_SCRIPT, PULSAR_CHART, PULSAR_IMAGE, PULSAR_LOOKUP, PULSAR_NAMESPACE, PULSAR_SERVICE, PULSAR_TOPIC, PULSAR_VERSION, QUIET_LEAD, REDIS, REDIS_IMAGE, RELATIVE, ROUTE, RULES_EVERY, SERVER, SESSION, SPREAD_LEAD, STAGES, SYSCTL, TCP_STATE, TICK, VIP_ANNOTATION, VIP_LABEL, WATCHDOG, ZONE_KEPT, ZONE_LABEL, ZONE_LOST, ZTUNNEL_PODS, announcementsOf, announcerOf, arpMacOf, backendsAt, backendsOf, backlogEnd, bandOf, brokerOfHost, brokersOf, byClients, byNode, cacheFolder, calibrationOf, capOf, cellsTable, certificateArgs, checkRun, cidrOverlap, cidrRange, ciliumBackendsOf, ciliumMode, ciliumValues, clientObjects, clusterFilter, clusterNameOf, compose, createCluster, cyclesOf, deletes, deletionOf, deletionWindow, deletionsOf, describeCell, dnsCommands, drainBand, dropOf, emptyOf, ensureChart, ensurePinnedChart, ensureTool, estimatedTime, factsOf, failuresOf, faultSpan, followsZtunnel, goDuration, helmInstall, holdMetallb, holdPulsar, holdVip, hostsOf, installCilium, installKubeVip, installMesh, installMetallb, installPulsar, intakeAfter, intakeOf, isKubeProxyMode, isTunnelState, istioOperator, judge, judgeKind, judgeReconnects, keptKernel, kernelCounts, kindCellRecordSchema, kindCellRecordV1Schema, kindConfig, kubeProxyPatch, kubeVipObjects, kubeVipPods, kubeVipReady, kubeletBand, labImage, labSource, latchBand, latestReturn, leafCertificates, learnWindow, lintCell, listenerCountsOf, localPin, macsAt, medianInterval, medianText, memberlistFileOf, memberlistTermsOf, meshKey, metallbObjects, metallbRecord, metallbValues, metallbVipOf, middle, middleboxCommands, namesService, namespaceObject, namespaceOf, nodeLossBand, nodeNames, nodeReturnBand, observationAfter, observationEnd, openKindLab, outboundCluster, outcomesOf, overshootBand, ownerAtRest, ownerOrder, pausedWindow, periodOf, placementOf, planOf, platformKey, podsAt, podsOf, policyObject, pollsListeners, predictionText, probeOf, propagationOf, propagationSource, pulsarObjects, pulsarRecord, pulsarValues, quantileOf, readRecord, recordsOf, redisObjects, referenceOf, removalBand, requestBand, requestsOf, restartBand, restartOf, restartsAt, reusesReset, rulesOf, runKindCells, scenario, scenarioOf, scenarioProblems, seconds, secondsOf, serverCluster, serverObjects, sessionsAt, setZtunnelEnv, sha256, shardOf, sharedGate, showBand, signalOf, signalsOf, sliceDropOf, sliceEmptyOf, sliceReadyAt, sliceReadyOf, slicesOf, socketsOf, speakersReady, stackOf, stageLabImage, stallRemovalBand, stopOf, stoppingOf, suspicionOf, sysctlCommand, tarEntry, timeSpread, toolPath, transitionOf, triesOf, tunnelsOf, unitOf, updatesOf, upgrade, upgradeCell, verdictsOf, vipLeaseOf, vipNetworkOf, vipOf, vipServed, vipService, writeResult, zoneOf, ztunnelCertificates, ztunnelEnvArgs, ztunnelPodsOf, ztunnelReplaced };
 
 //# sourceMappingURL=index.js.map