@rocicorp/zero 1.11.0-canary.35 → 1.11.0-canary.37

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -17,6 +17,7 @@ import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateNativeHisto
 import { rowIDString } from "../../types/row-key.js";
 import "../replicator/schema/replication-state.js";
 import { planWarningMessage } from "./plan-warnings.js";
+import { shouldCutOffPush } from "./push-cutoff.js";
 import { queryShape } from "./query-shape.js";
 import { ResetPipelinesSignal } from "./snapshotter.js";
 import { Subscription } from "../../types/subscription.js";
@@ -56,7 +57,36 @@ function shutdownBeforeInitializationError() {
 * use a timer to update the ttlClock every minute.
 */
 var TTL_CLOCK_INTERVAL = 6e4;
-var slowHydrationLogThrottle = new LogThrottle({ windowMs: 5 * 6e4 });
+/**
+* A slow query is typically slow for every client group that runs it, so
+* slow hydrations are logged at most once per query shape per this window,
+* per process. See {@link LogThrottle}.
+*/
+var SLOW_HYDRATION_LOG_WINDOW_MS = 5 * 6e4;
+/** See `ViewSyncerService.#logRebuilds()`. */
+var REBUILD_LOG_INTERVAL_MS = 6e4;
+var REBUILD_LOG_MIN_MS = 1e3;
+function newRebuildLog() {
+	return {
+		/** The batches that hydrated queries again. */
+		batches: 0,
+		/** How many of them were not read, which hydrates every query again. */
+		batchesNotRead: 0,
+		/** The queries that they abandoned, and why. */
+		queries: 0,
+		estimate: 0,
+		read: 0,
+		cutoff: 0,
+		/** How many of those were hydrated again, and how long that took. */
+		hydrated: 0,
+		hydrationTimeMs: 0,
+		/** How many were removed instead. */
+		timedOut: 0,
+		rejectedByCircuitBreaker: 0,
+		evictedForHydrationBudget: 0
+	};
+}
+var slowHydrationLogThrottle = new LogThrottle({ windowMs: SLOW_HYDRATION_LOG_WINDOW_MS });
 var ViewSyncerService = class {
 	id;
 	createdAtMs = Date.now();
@@ -210,9 +240,65 @@ var ViewSyncerService = class {
 	});
 	#inspectorDelegate;
 	#config;
+	/**
+	* Set when `costBasedSyncEnabled`: advancements are then bounded by
+	* cutting off the queries this says to and hydrating them again in the
+	* same poke, instead of by the advancement time limit that resets every
+	* pipeline of the client group.
+	*/
+	#pushCutoff;
+	/**
+	* Shared by the client groups of the sync worker. Set when
+	* `costBasedSyncEnabled`.
+	*/
+	#pacer;
+	/**
+	* The time that the pipelines took for the batch being processed, if the
+	* pipelines were advanced: reading the changes, pushing them, and
+	* hydrating queries again. Unlike the processing time of the advancement,
+	* this leaves out the time spent awaiting I/O (e.g. the CVR flush), during
+	* which other client groups run.
+	*/
+	#batchTimeMs;
+	#pipelineTime = { ms: 0 };
+	/** What the client group's batches have been taking, for pacing. */
+	#smoothedBatchTimeMs;
+	/** Ends the wait between batches, if the client group is in one. */
+	#endBatchWait;
+	/** The waits between batches since the last time they were logged. */
+	#batchWaitLog = {
+		loggedAt: -Infinity,
+		waits: 0,
+		waitedMs: 0
+	};
+	/** What batches have hydrated again since that was last logged. */
+	#rebuildLog = newRebuildLog();
+	#rebuildsLoggedAt = -Infinity;
+	#queryRebuilds = getOrCreateCounter("sync", "query-rebuilds", {
+		description: "Number of queries hydrated again by an advancement instead of being pushed (all of) its changes, by reason: \"estimate\" if pushing them to it was estimated to cost more, \"read\" if reading them was, for the queries that were left to push them to, or \"cutoff\" if the pushes had taken as long as hydrating the query did",
+		unit: "{query}"
+	});
+	#queryRebuildTime = getOrCreateLatencyHistogram("sync", "query-rebuild-time", "Time to hydrate a query again in an advancement.");
+	#pushEstimateRatio = getOrCreateValueHistogram("sync", "push-estimate-ratio", {
+		description: "The estimated time to push the changes of an advancement to a query, divided by the time it took, for queries that were pushed all of the changes",
+		unit: "1",
+		bucketBoundaries: [
+			.1,
+			.25,
+			.5,
+			.8,
+			1.25,
+			2,
+			4,
+			10
+		]
+	});
+	#batchWaitTime = getOrCreateLatencyHistogram("sync", "batch-wait-time", "Time a client group is made to wait between advancements by pacing.");
 	#runPriorityOp;
-	constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis), now = performance.now.bind(performance)) {
+	constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis), now = performance.now.bind(performance), pushCutoff = shouldCutOffPush, pacer) {
 		this.#config = config;
+		this.#pushCutoff = config.costBasedSyncEnabled ? pushCutoff : void 0;
+		this.#pacer = config.costBasedSyncEnabled ? pacer : void 0;
 		this.id = clientGroupID;
 		this.connContextManager = connContextManager;
 		this.#shard = shard;
@@ -335,6 +421,7 @@ var ViewSyncerService = class {
 				}
 				assert(state === "version-ready", "state should be version-ready");
 				this.#e2eServingLagTracker.onVersionReady(replicaState);
+				this.#batchTimeMs = void 0;
 				await this.#runInLockWithCVR(async (lc, cvr) => {
 					const clientSchema = must(cvr.clientSchema, "cvr.clientSchema missing after initialization");
 					if (!this.#pipelines.initialized()) this.#pipelines.init(clientSchema);
@@ -354,7 +441,7 @@ var ViewSyncerService = class {
 							case "scalar-subquery":
 							case "truncation":
 							case "schema-change":
-								if (!Object.values(cvr.queries).some((q) => q.type === "client")) previousQueries = new Map(this.#pipelines.queries());
+								if (!Object.values(cvr.queries).some((q) => q.type === "client")) previousQueries = new Map([...this.#pipelines.queries(), ...this.#pipelines.abandonedQueries()]);
 								break;
 							case "permissions-change": break;
 							default: unreachable(result.reason);
@@ -366,7 +453,9 @@ var ViewSyncerService = class {
 					const backgroundConnCtx = this.connContextManager.getBackgroundConnectionContext();
 					if (backgroundConnCtx) await this.#maybeHydratePipelines(lc, cvr, backgroundConnCtx, previousQueries);
 					else lc.info?.("No validated background connection; deferring pipeline init");
+					if (this.#batchTimeMs !== void 0) this.#batchTimeMs += this.#totalHydrationTimeMs();
 				});
+				if (this.#pacer && this.#batchTimeMs !== void 0) await this.#paceBatches(this.#pacer, this.#batchTimeMs);
 			}
 			if (this.#drainCoordinator.shouldDrain()) this.#drainCoordinator.drainNextIn(this.#totalHydrationTimeMs());
 			await this.#cleanup();
@@ -406,7 +495,7 @@ var ViewSyncerService = class {
 		if (!this.#pipelines.initialized()) return;
 		let version;
 		try {
-			version = this.#pipelines.advanceWithoutDiff();
+			version = this.#pipelines.advanceWithoutDiff(this.#pushCutoff !== void 0);
 		} catch (e) {
 			if (!(e instanceof ResetPipelinesSignal)) throw e;
 			lc.info?.(`resetting pipelines: ${e.message}${e.breakdown ? ` (${e.breakdown})` : ""}`);
@@ -932,7 +1021,7 @@ var ViewSyncerService = class {
 				coveredHydratedQueries++;
 				firstCoveredQuery ??= covered;
 			}
-			const timer = new TimeSliceTimer(lc);
+			const timer = this.#timer(lc, "hydration");
 			let count = 0;
 			let timedOut = false;
 			await startAsyncSpan(tracer, "vs.#hydrateUnchangedQueries.addQuery", async (span) => {
@@ -1379,7 +1468,7 @@ var ViewSyncerService = class {
 				this.#queryReplacements.delete(q.id);
 			}
 			let totalProcessTime = 0;
-			const timer = new TimeSliceTimer(lc);
+			const timer = this.#timer(lc, "hydration");
 			const pipelines = this.#pipelines;
 			const hydrations = this.#hydrations;
 			const hydrationTime = this.#hydrationTime;
@@ -1393,7 +1482,7 @@ var ViewSyncerService = class {
 			const rejectedQueries = [];
 			const circuitBreaker = this.#hydrationCircuitBreaker;
 			const self = this;
-			await yieldProcess(lc);
+			await (this.#pacer ? this.#pacer.yieldSlice("hydration") : yieldProcess(lc));
 			function* hydrateQueries(queries, optional, slowHydrateThreshold) {
 				for (let i = 0; i < queries.length; i++) {
 					if (optional && hydrationBudget.exhausted()) {
@@ -1634,16 +1723,19 @@ var ViewSyncerService = class {
 			span.setAttribute("clientGroupID", this.id);
 			assert(this.#pipelines.initialized(), "pipelines must be initialized (advancePipelines");
 			const start = performance.now();
-			const timer = new TimeSliceTimer(lc);
+			const timer = this.#timer(lc, "batch");
 			let outcome = { outcome: "error" };
 			let setupTimeMs;
+			let rebuildTimeMs = 0;
+			let evictedQueries = false;
+			this.#pipelineTime.ms = 0;
 			try {
 				let pokers;
 				let updater;
 				let version;
 				let numChanges = 0;
 				try {
-					const advancement = this.#pipelines.advance(timer);
+					const advancement = this.#pipelines.advance(timer, this.#costBasedAdvance(cvr));
 					version = advancement.version;
 					numChanges = advancement.numChanges;
 					lc = lc.withContext("newVersion", version);
@@ -1651,10 +1743,17 @@ var ViewSyncerService = class {
 					pokers = startPoke(lc, this.#getClients(cvr.version), targetVersion);
 					lc.debug?.(`applying ${numChanges} to advance to ${version}`);
 					setupTimeMs = performance.now() - start;
-					await this.#processChanges(lc, await timer.start(), advancement.changes, (rows) => {
+					await this.#processChanges(lc, await timer.start(), this.#timed(advancement.changes), (rows) => {
 						updater ??= new CVRQueryDrivenUpdater(this.#cvrStore, cvr, version, this.#pipelines.replicaVersion, this.#pipelines);
 						return updater.received(lc, rows);
 					}, pokers);
+					for (const { estimatedPushMs, pushTimeMs } of advancement.estimates()) if (pushTimeMs > 0) this.#pushEstimateRatio.record(estimatedPushMs / pushTimeMs);
+					const abandoned = advancement.abandoned();
+					if (abandoned.length > 0) {
+						updater ??= new CVRQueryDrivenUpdater(this.#cvrStore, cvr, version, this.#pipelines.replicaVersion, this.#pipelines);
+						timer.stop();
+						({rebuildTimeMs, evictedQueries} = await this.#rebuildAbandonedQueries(lc, cvr, abandoned, updater, pokers));
+					}
 				} catch (e) {
 					setupTimeMs ??= performance.now() - start;
 					if (e instanceof ResetPipelinesSignal) {
@@ -1669,19 +1768,23 @@ var ViewSyncerService = class {
 					throw e;
 				}
 				assert(pokers && version !== void 0, "advancement state missing");
-				if (updater) this.#cvr = await this.#flushPoked(lc, updater, pokers);
+				if (updater) {
+					this.#cvr = await this.#flushPoked(lc, updater, pokers);
+					if (evictedQueries) this.#scheduleExpireEviction(lc, this.#cvr);
+				}
 				const finalVersion = (this.#cvr ?? cvr).version;
 				await startAsyncSpan(tracer, "vs.#advancePipelines.pokeEnd", () => pokers.end(finalVersion));
 				this.#markVersionServed(version);
 				const wallTime = performance.now() - start;
-				const totalProcessTime = timer.totalElapsed();
+				const totalProcessTime = timer.totalElapsed() + rebuildTimeMs;
 				lc.debug?.(`finished processing advancement of ${numChanges} changes ((process: ${totalProcessTime} ms, wall: ${wallTime} ms))`);
 				this.#transactionAdvanceTime.recordMs(totalProcessTime);
 				outcome = { outcome: "success" };
 				return "success";
 			} finally {
 				const setupMs = must(setupTimeMs);
-				const processingTimeMs = setupMs + timer.totalElapsed();
+				const processingTimeMs = setupMs + timer.totalElapsed() + rebuildTimeMs;
+				this.#batchTimeMs = setupMs + this.#pipelineTime.ms;
 				const wallTimeMs = performance.now() - start;
 				this.#advancementMetrics.record({
 					processingTimeMs,
@@ -1691,6 +1794,233 @@ var ViewSyncerService = class {
 			}
 		});
 	}
+	/**
+	* What a cost-based advancement needs, or `undefined` if advancements are
+	* bounded by the advancement time limit.
+	*/
+	#costBasedAdvance(cvr) {
+		const shouldCutOff = this.#pushCutoff;
+		if (shouldCutOff === void 0) return;
+		if (this.#pipelines.currentVersion() < cvr.version.stateVersion) return;
+		return {
+			shouldCutOff,
+			rebuildMargin: this.#config.syncRebuildMargin ?? 2
+		};
+	}
+	/**
+	* Hydrates the queries whose pipelines an advancement abandoned, at the
+	* snapshot it advanced to, within the advancement's CVR update and poke.
+	* What the queries contributed to the update before they were abandoned
+	* is retracted first.
+	*
+	* The hydrations are protected as they are in a hydration pass (see
+	* {@link #addAndRemoveQueries}). A query that exceeds the query hydration
+	* timeout, or whose circuit breaker is open, is removed and errored to its
+	* clients. Inactive queries are hydrated last, and those not reached
+	* before the hydration budget is spent are evicted.
+	*
+	* @returns the time spent hydrating, and whether inactive queries were
+	*          evicted.
+	*/
+	async #rebuildAbandonedQueries(lc, cvr, abandoned, updater, pokers) {
+		for (const patch of await updater.retractQueries(lc, abandoned.map((q) => q.queryID))) await pokers.addPatch(patch);
+		const hydrationBudget = new HydrationBudget(this.#config.viewSyncerHydrationBudgetMs ?? 0, this.#now);
+		const abandonedByID = new Map(abandoned.map((q) => [q.queryID, q]));
+		const { required, optional } = classifyQueriesForHydration(abandoned.map((q) => must(cvr.queries[q.queryID])));
+		const byRecord = (queries) => queries.map(({ id }) => must(abandonedByID.get(id)));
+		const timer = this.#timer(lc, "batch");
+		const pipelines = this.#pipelines;
+		const circuitBreaker = this.#hydrationCircuitBreaker;
+		const budgetEvicted = [];
+		const timedOut = [];
+		const rejected = [];
+		const rebuilt = {
+			active: 0,
+			inactive: 0
+		};
+		const self = this;
+		let total = 0;
+		function* rebuild(queries, inactive) {
+			for (let i = 0; i < queries.length; i++) {
+				if (inactive && hydrationBudget.exhausted()) {
+					budgetEvicted.push(...queries.slice(i));
+					return;
+				}
+				const q = must(queries[i]);
+				const hydrationQuery = {
+					id: q.queryID,
+					ast: q.originalAst,
+					transformationHash: q.transformationHash,
+					name: q.queryName
+				};
+				const breakable = cvr.queries[q.queryID]?.type !== "internal";
+				if (breakable && circuitBreaker.isOpen(q.transformationHash)) {
+					rejected.push(q);
+					self.#recordCircuitBreakerRejection(lc, hydrationQuery);
+					continue;
+				}
+				let aborted = false;
+				for (const change of pipelines.addQuery(q.transformationHash, q.queryID, q.originalAst, timer.startWithoutYielding(), q.queryName, "unchanged-query-rehydrate")) {
+					if (change === "yield" && breakable && circuitBreaker.exceeded(timer.totalElapsed())) {
+						aborted = true;
+						yield change;
+						break;
+					}
+					yield change;
+				}
+				const elapsed = timer.stop();
+				total += elapsed;
+				if (aborted) {
+					timedOut.push(q);
+					self.#recordHydrationTimeout(lc, hydrationQuery, elapsed);
+					continue;
+				}
+				if (inactive) rebuilt.inactive++;
+				else rebuilt.active++;
+				self.#queryRebuilds.add(1, { reason: q.reason });
+				self.#queryRebuildTime.recordMs(elapsed);
+				self.#addQueryMaterializationServerMetric(q.queryID, elapsed);
+				if (elapsed > self.#slowHydrateThreshold) self.#logSlowHydration(lc, hydrationQuery, elapsed);
+			}
+		}
+		function* rebuildAll() {
+			yield* rebuild(byRecord(required), false);
+			yield* rebuild(byRecord(optional), true);
+		}
+		await this.#processChanges(lc, timer, this.#timed(rebuildAll()), (rows) => updater.received(lc, rows), pokers);
+		const abortedQueries = [...timedOut, ...rejected];
+		if (abortedQueries.length > 0) {
+			for (const patch of await updater.abortExecutedQueries(lc, abortedQueries.map((q) => q.queryID))) await pokers.addPatch(patch);
+			if (timedOut.length > 0) this.#queryEvictions.add(timedOut.length, { reason: "hydration-timeout" });
+			if (rejected.length > 0) this.#queryEvictions.add(rejected.length, { reason: "hydration-circuit-breaker" });
+			this.#sendHydrationTimeoutErrors(cvr, abortedQueries.map((q) => ({
+				id: q.queryID,
+				name: q.queryName
+			})));
+		}
+		const evictedQueryIDs = budgetEvicted.map((q) => q.queryID);
+		for (const patch of updater.removeTrackedQueries(evictedQueryIDs)) await pokers.addPatch(patch);
+		for (const { queryID } of [...abortedQueries, ...budgetEvicted]) {
+			this.#pipelines.removeQuery(queryID);
+			this.#inspectorDelegate.removeQuery(queryID);
+			this.#queryReplacements.delete(queryID);
+		}
+		this.#recordHydrationBudgetExhaustion(lc, hydrationBudget, rebuilt.active, rebuilt.inactive, evictedQueryIDs);
+		for (const patch of await updater.deleteUnreferencedRows(lc)) await pokers.addPatch(patch);
+		this.#logRebuilds(lc, abandoned, {
+			hydrated: rebuilt.active + rebuilt.inactive,
+			hydrationTimeMs: total,
+			timedOut: timedOut.length,
+			rejectedByCircuitBreaker: rejected.length,
+			evictedForHydrationBudget: budgetEvicted.length
+		});
+		return {
+			rebuildTimeMs: total,
+			evictedQueries: evictedQueryIDs.length > 0
+		};
+	}
+	/**
+	* Says in the log that this client group's batches hydrate queries again,
+	* which can be every batch. Nothing else is logged at `info` for a batch,
+	* so this is at most once a minute, for the batches since the last time,
+	* and not before their hydrations add up to a second. Which queries they
+	* were is in the query stats, by query shape (see `QueryStats`).
+	*/
+	#logRebuilds(lc, abandoned, batch) {
+		const log = this.#rebuildLog;
+		log.batches++;
+		log.queries += abandoned.length;
+		for (const { reason } of abandoned) log[reason]++;
+		if (abandoned.some((q) => q.reason === "read")) log.batchesNotRead++;
+		log.hydrated += batch.hydrated;
+		log.hydrationTimeMs += batch.hydrationTimeMs;
+		log.timedOut += batch.timedOut;
+		log.rejectedByCircuitBreaker += batch.rejectedByCircuitBreaker;
+		log.evictedForHydrationBudget += batch.evictedForHydrationBudget;
+		const now = this.#now();
+		if (log.hydrationTimeMs < REBUILD_LOG_MIN_MS || now - this.#rebuildsLoggedAt < REBUILD_LOG_INTERVAL_MS) return;
+		lc.info?.("Client group hydrated queries again in its batches of changes instead of pushing the changes to them", {
+			clientGroupID: this.id,
+			...log
+		});
+		this.#rebuildsLoggedAt = now;
+		this.#rebuildLog = newRebuildLog();
+	}
+	/**
+	* Adds the time that the pipelines take to produce `values` to the time
+	* of the batch being processed (see {@link #batchTimeMs}).
+	*/
+	#timed(values) {
+		return this.#pacer ? timeIteration(values, this.#pipelineTime) : values;
+	}
+	/**
+	* A timer for a batch, or for the hydration of new queries, which has
+	* priority over batches for the worker's time slices (see
+	* `BatchPacer.yieldSlice()`).
+	*/
+	#timer(lc, kind) {
+		const pacer = this.#pacer;
+		return new TimeSliceTimer(lc, pacer && (() => pacer.yieldSlice(kind)));
+	}
+	/**
+	* Waits as long as the worker's pacer says before the next batch, after
+	* one that took `batchTimeMs`. The wait is outside of the
+	* lock, so connections and query changes are served as usual, at the
+	* pipelines' current snapshot. `version-ready` notifications received in
+	* the meantime are coalesced by the subscription, so the next batch covers
+	* all of them.
+	*/
+	async #paceBatches(pacer, batchTimeMs) {
+		this.#smoothedBatchTimeMs = this.#smoothedBatchTimeMs === void 0 ? batchTimeMs : this.#smoothedBatchTimeMs + BATCH_TIME_SMOOTHING * (batchTimeMs - this.#smoothedBatchTimeMs);
+		const processingMs = this.#smoothedBatchTimeMs * (1 + BATCH_WAIT_JITTER * (Math.random() * 2 - 1));
+		const start = performance.now();
+		let waited = false;
+		for (;;) {
+			const remaining = pacer.waitMs(processingMs) - (performance.now() - start);
+			if (remaining <= 0 || !this.#stateChanges.active) break;
+			waited = true;
+			await new Promise((resolve) => {
+				const timer = setTimeout(resolve, Math.min(remaining, BATCH_WAIT_STEP_MS));
+				this.#endBatchWait = () => {
+					clearTimeout(timer);
+					resolve();
+				};
+			});
+			this.#endBatchWait = void 0;
+		}
+		if (waited) {
+			const waitedMs = performance.now() - start;
+			this.#batchWaitTime.recordMs(waitedMs);
+			this.#logBatchWait(pacer, waitedMs);
+		}
+	}
+	/**
+	* Says in the log that this client group is being held back, which the
+	* worker's metrics do not tell one client group from another by. A client
+	* group that is held back is after every batch, so this is at most once
+	* a minute, for the waits since the last time, and not for waits that add
+	* up to less than a second.
+	*/
+	#logBatchWait(pacer, waitedMs) {
+		const log = this.#batchWaitLog;
+		log.waits++;
+		log.waitedMs += waitedMs;
+		const now = this.#now();
+		if (log.waitedMs < BATCH_WAIT_LOG_MIN_MS || now - log.loggedAt < BATCH_WAIT_LOG_INTERVAL_MS) return;
+		this.#lc.info?.("Client group is waiting between batches: its sync worker is over its share target", {
+			clientGroupID: this.id,
+			waits: log.waits,
+			waitedMs: log.waitedMs,
+			lastWaitMs: waitedMs,
+			batchTimeMs: this.#smoothedBatchTimeMs,
+			waitMultiplier: pacer.multiplier
+		});
+		this.#batchWaitLog = {
+			loggedAt: now,
+			waits: 0,
+			waitedMs: 0
+		};
+	}
 	async inspect(selector, msg) {
 		await this.#runInLockForClient(selector, msg, this.#handleInspect);
 	}
@@ -1784,6 +2114,7 @@ var ViewSyncerService = class {
 		this.connContextManager.setSharedRetransformReady(false);
 		this.#initialized.reject(shutdownBeforeInitializationError());
 		this.#stateChanges.cancel();
+		this.#endBatchWait?.();
 		return this.#stopped.promise;
 	}
 	async #cleanup(err) {
@@ -1854,15 +2185,49 @@ function hasExpiredQueries(cvr) {
 	for (const q of Object.values(cvr.queries)) if (expired(ttlClock, q)) return true;
 	return false;
 }
+/**
+* Iterates over `values`, adding the time spent producing them to
+* `total.ms`. The time that the consumer takes between values, including
+* any that it spends awaiting, is not counted.
+*/
+function* timeIteration(values, total) {
+	const iterator = values[Symbol.iterator]();
+	try {
+		for (;;) {
+			const start = performance.now();
+			const result = iterator.next();
+			total.ms += performance.now() - start;
+			if (result.done) return;
+			yield result.value;
+		}
+	} finally {
+		iterator.return?.();
+	}
+}
+/** How often a client group that is waiting between batches asks again. */
+var BATCH_WAIT_STEP_MS = 250;
+/** See `ViewSyncerService.#logBatchWait()`. */
+var BATCH_WAIT_LOG_INTERVAL_MS = 6e4;
+var BATCH_WAIT_LOG_MIN_MS = 1e3;
+/** The weight of the latest batch in a client group's smoothed batch time. */
+var BATCH_TIME_SMOOTHING = .3;
+/** How far, as a fraction, a client group's wait is varied at random. */
+var BATCH_WAIT_JITTER = .3;
 var TimeSliceTimer = class {
 	#total = 0;
 	#start = 0;
 	#lc;
-	constructor(lc) {
+	#yieldSlice;
+	/**
+	* @param yieldSlice How the timer yields the process, if not by taking
+	*        turns with the rest of the worker's work (see `yieldProcess()`).
+	*/
+	constructor(lc, yieldSlice) {
 		this.#lc = lc;
+		this.#yieldSlice = yieldSlice;
 	}
 	async start() {
-		await yieldProcess(this.#lc);
+		await (this.#yieldSlice ? this.#yieldSlice() : yieldProcess(this.#lc));
 		return this.startWithoutYielding();
 	}
 	startWithoutYielding() {
@@ -1872,7 +2237,7 @@ var TimeSliceTimer = class {
 	}
 	async yieldProcess(_msgForTesting) {
 		this.#stopLap();
-		await yieldProcess(this.#lc);
+		await (this.#yieldSlice ? this.#yieldSlice() : yieldProcess(this.#lc));
 		this.#startLap();
 	}
 	#startLap() {