@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.41

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -1,4 +1,6 @@
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
+import { getOrInsertComputed } from "../../../../shared/src/map.js";
+import { newArray } from "../../../../shared/src/arrays.js";
 import { must } from "../../../../shared/src/must.js";
 import { h64 } from "../../../../shared/src/hash.js";
 import { Internal, InvalidConnectionRequest, InvalidConnectionRequestBaseCookie, Rehome } from "../../../../zero-protocol/src/error-kind-enum.js";
@@ -10,24 +12,28 @@ import { manualSpan, startAsyncSpan, startSpan } from "../../../../otel/src/span
 import { ZERO_VERSION_COLUMN_NAME } from "../replicator/schema/constants.js";
 import { transformAndHashQuery } from "../../auth/read-authorizer.js";
 import { randInt } from "../../../../shared/src/rand.js";
+import { LogThrottle } from "../../observability/log-throttle.js";
 import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateNativeHistogram, getOrCreateUpDownCounter, getOrCreateValueHistogram } from "../../observability/metrics.js";
 import { rowIDString } from "../../types/row-key.js";
 import "../replicator/schema/replication-state.js";
-import { parseSignature } from "./row-set-signature.js";
+import { planWarningMessage } from "./plan-warnings.js";
+import { shouldCutOffPush } from "./push-cutoff.js";
+import { queryShape } from "./query-shape.js";
 import { ResetPipelinesSignal } from "./snapshotter.js";
-import "./pipeline-driver.js";
 import { Subscription } from "../../types/subscription.js";
 import { CustomKeyMap } from "../../../../shared/src/custom-key-map.js";
 import { ttlClockAsNumber, ttlClockFromNumber } from "./ttl-clock.js";
 import { EMPTY_CVR_VERSION, cmpVersions, versionFromString, versionString, versionToCookie } from "./schema/types.js";
 import { ProtocolErrorWithLevel, getLogLevel, wrapWithProtocolError } from "../../types/error-with-level.js";
 import { isAuthErrorBody } from "../../auth/auth.js";
+import { AdvancementMetrics } from "./advancement-metrics.js";
 import { ClientHandler, startPoke } from "./client-handler.js";
 import { tracer } from "./tracer.js";
 import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
 import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, classifyQueriesForHydration, nextEvictionTime } from "./cvr.js";
 import { E2EServingLagTracker } from "./e2e-serving-lag.js";
 import { HydrationBudget } from "./hydration-budget.js";
+import { HydrationCircuitBreaker } from "./hydration-circuit-breaker.js";
 import { handleInspect } from "./inspect-handler.js";
 import { QueryCoveringIndex } from "./query-covering.js";
 import { resolver } from "@rocicorp/resolver";
@@ -51,6 +57,36 @@ function shutdownBeforeInitializationError() {
 * use a timer to update the ttlClock every minute.
 */
 var TTL_CLOCK_INTERVAL = 6e4;
+/**
+* A slow query is typically slow for every client group that runs it, so
+* slow hydrations are logged at most once per query shape per this window,
+* per process. See {@link LogThrottle}.
+*/
+var SLOW_HYDRATION_LOG_WINDOW_MS = 5 * 6e4;
+/** See `ViewSyncerService.#logRebuilds()`. */
+var REBUILD_LOG_INTERVAL_MS = 6e4;
+var REBUILD_LOG_MIN_MS = 1e3;
+function newRebuildLog() {
+	return {
+		/** The batches that hydrated queries again. */
+		batches: 0,
+		/** How many of them were not read, which hydrates every query again. */
+		batchesNotRead: 0,
+		/** The queries that they abandoned, and why. */
+		queries: 0,
+		estimate: 0,
+		read: 0,
+		cutoff: 0,
+		/** How many of those were hydrated again, and how long that took. */
+		hydrated: 0,
+		hydrationTimeMs: 0,
+		/** How many were removed instead. */
+		timedOut: 0,
+		rejectedByCircuitBreaker: 0,
+		evictedForHydrationBudget: 0
+	};
+}
+var slowHydrationLogThrottle = new LogThrottle({ windowMs: SLOW_HYDRATION_LOG_WINDOW_MS });
 var ViewSyncerService = class {
 	id;
 	createdAtMs = Date.now();
@@ -62,6 +98,7 @@ var ViewSyncerService = class {
 	#drainCoordinator;
 	#keepaliveMs;
 	#slowHydrateThreshold;
+	#hydrationCircuitBreaker;
 	#lastConnectTime = Date.now();
 	/**
 	* The TTL clock is used to determine the time at which queries are considered
@@ -89,6 +126,13 @@ var ViewSyncerService = class {
 	#lock = new Lock();
 	#cvrStore;
 	#stopped = resolver();
+	/**
+	* Set when {@link #cleanup} begins. Lock tasks that were in flight when the
+	* view-syncer was stopped may still complete after the timers have been
+	* cleared; this flag prevents them from scheduling new timers, which would
+	* otherwise outlive the service (and retain everything it references).
+	*/
+	#shuttingDown = false;
 	#initialized = resolver();
 	#cvr;
 	/**
@@ -103,7 +147,8 @@ var ViewSyncerService = class {
 	*   updates are processed via `#syncQueryPipelineSet`.
 	*
 	* Resets to `false` if `#advancePipelines` returns a `ResetPipelinesSignal`
-	* and pipelines must be reset and rehydrated.
+	* and pipelines must be reset and rehydrated, or if a reloaded CVR is ahead
+	* of the pipelines (see `#resetPipelinesIfBehindCVR`).
 	*/
 	#pipelinesHydrated = false;
 	#servedVersion = null;
@@ -129,6 +174,7 @@ var ViewSyncerService = class {
 		description: "Observations of sync.e2e_serving_lag that came out negative and were clamped to zero. Non-zero means the upstream database clock is running ahead of this pod by more than the entire pipeline latency, so sync.e2e_serving_lag is biased low and reads healthier than reality. See replication.upstream_clock_skew for the magnitude.",
 		unit: "{observation}"
 	});
+	#advancementMetrics = new AdvancementMetrics();
 	#transactionAdvanceTime = getOrCreateLatencyHistogram("sync", "advance-time", "Time to advance all queries for a given client group after applying a new transaction to the replica.");
 	#queryTransformations = getOrCreateCounter("sync", "query.transformations", "Number of query transformations performed");
 	#queryTransformationTime = getOrCreateLatencyHistogram("sync", "query.transformation-time", "Time to transform custom queries via API server.");
@@ -181,14 +227,78 @@ var ViewSyncerService = class {
 		]
 	});
 	#queryEvictions = getOrCreateCounter("sync", "query_evictions", {
-		description: "Number of inactive queries evicted, grouped by reason.",
+		description: "Number of queries evicted from the CVR ahead of their removal by the client, grouped by reason. Inactive queries are evicted by ttl and hydration-budget; hydration-timeout and hydration-circuit-breaker evict active queries as well.",
+		unit: "{query}"
+	});
+	#hydrationTimeouts = getOrCreateCounter("sync", "hydration_timeouts", {
+		description: "Number of query hydrations aborted for exceeding the query hydration timeout.",
+		unit: "{query}"
+	});
+	#hydrationCircuitBreakerRejections = getOrCreateCounter("sync", "hydration_circuit_breaker_rejections", {
+		description: "Number of queries rejected without hydration because their hydration circuit breaker was open.",
 		unit: "{query}"
 	});
 	#inspectorDelegate;
 	#config;
+	/**
+	* Set when `costBasedSyncEnabled`: advancements are then bounded by
+	* cutting off the queries this says to and hydrating them again in the
+	* same poke, instead of by the advancement time limit that resets every
+	* pipeline of the client group.
+	*/
+	#pushCutoff;
+	/**
+	* Shared by the client groups of the sync worker. Set when
+	* `costBasedSyncEnabled`.
+	*/
+	#pacer;
+	/**
+	* The time that the pipelines took for the batch being processed, if the
+	* pipelines were advanced: reading the changes, pushing them, and
+	* hydrating queries again. Unlike the processing time of the advancement,
+	* this leaves out the time spent awaiting I/O (e.g. the CVR flush), during
+	* which other client groups run.
+	*/
+	#batchTimeMs;
+	#pipelineTime = { ms: 0 };
+	/** What the client group's batches have been taking, for pacing. */
+	#smoothedBatchTimeMs;
+	/** Ends the wait between batches, if the client group is in one. */
+	#endBatchWait;
+	/** The waits between batches since the last time they were logged. */
+	#batchWaitLog = {
+		loggedAt: -Infinity,
+		waits: 0,
+		waitedMs: 0
+	};
+	/** What batches have hydrated again since that was last logged. */
+	#rebuildLog = newRebuildLog();
+	#rebuildsLoggedAt = -Infinity;
+	#queryRebuilds = getOrCreateCounter("sync", "query-rebuilds", {
+		description: "Number of queries hydrated again by an advancement instead of being pushed (all of) its changes, by reason: \"estimate\" if pushing them to it was estimated to cost more, \"read\" if reading them was, for the queries that were left to push them to, or \"cutoff\" if the pushes had taken as long as hydrating the query did",
+		unit: "{query}"
+	});
+	#queryRebuildTime = getOrCreateLatencyHistogram("sync", "query-rebuild-time", "Time to hydrate a query again in an advancement.");
+	#pushEstimateRatio = getOrCreateValueHistogram("sync", "push-estimate-ratio", {
+		description: "The estimated time to push the changes of an advancement to a query, divided by the time it took, for queries that were pushed all of the changes",
+		unit: "1",
+		bucketBoundaries: [
+			.1,
+			.25,
+			.5,
+			.8,
+			1.25,
+			2,
+			4,
+			10
+		]
+	});
+	#batchWaitTime = getOrCreateLatencyHistogram("sync", "batch-wait-time", "Time a client group is made to wait between advancements by pacing.");
 	#runPriorityOp;
-	constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis), now = performance.now.bind(performance)) {
+	constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis), now = performance.now.bind(performance), pushCutoff = shouldCutOffPush, pacer) {
 		this.#config = config;
+		this.#pushCutoff = config.costBasedSyncEnabled ? pushCutoff : void 0;
+		this.#pacer = config.costBasedSyncEnabled ? pacer : void 0;
 		this.id = clientGroupID;
 		this.connContextManager = connContextManager;
 		this.#shard = shard;
@@ -203,6 +313,7 @@ var ViewSyncerService = class {
 		this.#cvrStore = new CVRStore(lc, cvrDb, shard, taskID, clientGroupID, () => this.#stateChanges.cancel());
 		this.#setTimeout = setTimeoutFn;
 		this.#now = now;
+		this.#hydrationCircuitBreaker = new HydrationCircuitBreaker(config.viewSyncerQueryHydrationTimeoutMs ?? 0, void 0, now);
 		this.#runPriorityOp = runPriorityOp;
 		this.keepalive();
 	}
@@ -229,11 +340,13 @@ var ViewSyncerService = class {
 				this.#stateChanges.cancel();
 				return;
 			}
+			let reloaded = false;
 			if (!this.#cvr) {
 				this.#lc.debug?.("loading cvr");
 				this.#cvr = await this.#runPriorityOp(lc, "loading cvr", () => this.#cvrStore.load(lc, this.#lastConnectTime));
 				this.#ttlClock = this.#cvr.ttlClock;
 				this.#ttlClockBase = Date.now();
+				reloaded = true;
 			} else {
 				const now = Date.now();
 				this.#cvr = {
@@ -242,6 +355,10 @@ var ViewSyncerService = class {
 				};
 			}
 			try {
+				if (reloaded && this.#resetPipelinesIfBehindCVR(lc, this.#cvr)) {
+					const connCtx = this.connContextManager.getBackgroundConnectionContext();
+					if (connCtx) await this.#maybeHydratePipelines(lc, this.#cvr, connCtx);
+				}
 				await fn(lc, this.#cvr);
 			} catch (e) {
 				this.#cvr = void 0;
@@ -251,11 +368,47 @@ var ViewSyncerService = class {
 			}
 		});
 	}
+	/**
+	* The CVR is reloaded after an error clears the cached copy. By then another
+	* view-syncer may have taken over the client group and flushed the CVR at a
+	* version ahead of this instance's hydrated pipelines (e.g. its replica was
+	* further ahead). The pipelines can no longer be diffed against the CVR, so
+	* reset them and let `#maybeHydratePipelines` rehydrate once the replica has
+	* caught up to the CVR.
+	*
+	* A CVR behind the pipelines is expected (advancements that do not change
+	* the CVR are not flushed) and is handled by the normal update path.
+	*
+	* Returns whether the pipelines were reset.
+	*
+	* Must be called from within the #lock.
+	*/
+	#resetPipelinesIfBehindCVR(lc, cvr) {
+		if (!this.#pipelinesHydrated) return false;
+		const pipelineVersion = this.#pipelines.currentVersion();
+		if (pipelineVersion >= cvr.version.stateVersion) return false;
+		lc.info?.(`resetting pipelines: pipelines@${pipelineVersion} are behind reloaded cvr@${versionString(cvr.version)}`);
+		this.#pipelineResets.add(1, { reason: "behind-cvr" });
+		this.#pipelinesHydrated = false;
+		this.connContextManager.setSharedRetransformReady(false);
+		this.#pipelines.reset(must(cvr.clientSchema, "cvr.clientSchema missing after initialization"));
+		return true;
+	}
 	readyState() {
-		return Promise.race([this.#initialized.promise, this.#drainCoordinator.draining]);
+		return new Promise((resolve, reject) => {
+			const unsubscribe = this.#drainCoordinator.onDraining(() => resolve("draining"));
+			this.#initialized.promise.then((state) => {
+				unsubscribe();
+				resolve(state);
+			}, (err) => {
+				unsubscribe();
+				reject(err);
+			});
+		});
 	}
 	async run() {
 		try {
+			this.#scheduleShutdown(this.#keepaliveMs);
 			if (await this.readyState() === "draining") {
 				this.#lc.debug?.(`draining view-syncer ${this.id} before running`);
 				this.stop();
@@ -268,6 +421,7 @@ var ViewSyncerService = class {
 				}
 				assert(state === "version-ready", "state should be version-ready");
 				this.#e2eServingLagTracker.onVersionReady(replicaState);
+				this.#batchTimeMs = void 0;
 				await this.#runInLockWithCVR(async (lc, cvr) => {
 					const clientSchema = must(cvr.clientSchema, "cvr.clientSchema missing after initialization");
 					if (!this.#pipelines.initialized()) this.#pipelines.init(clientSchema);
@@ -276,19 +430,32 @@ var ViewSyncerService = class {
 						lc.info?.(`resetting CVR: ${message}`);
 						throw new ClientNotFoundError(message);
 					}
+					let previousQueries;
 					if (this.#pipelinesHydrated) {
 						const result = await this.#advancePipelines(lc, cvr);
 						if (result === "success") return;
-						lc.info?.(`resetting pipelines: ${result.message}`);
+						lc.info?.(`resetting pipelines: ${result.message}${result.breakdown ? ` (${result.breakdown})` : ""}`);
 						this.#pipelineResets.add(1, { reason: result.reason });
+						switch (result.reason) {
+							case "advancement-timeout":
+							case "scalar-subquery":
+							case "truncation":
+							case "schema-change":
+								if (!Object.values(cvr.queries).some((q) => q.type === "client")) previousQueries = new Map([...this.#pipelines.queries(), ...this.#pipelines.abandonedQueries()]);
+								break;
+							case "permissions-change": break;
+							default: unreachable(result.reason);
+						}
 						this.#pipelines.reset(clientSchema);
 						this.#pipelinesHydrated = false;
 						this.connContextManager.setSharedRetransformReady(false);
 					}
 					const backgroundConnCtx = this.connContextManager.getBackgroundConnectionContext();
-					if (backgroundConnCtx) await this.#maybeHydratePipelines(lc, cvr, backgroundConnCtx);
+					if (backgroundConnCtx) await this.#maybeHydratePipelines(lc, cvr, backgroundConnCtx, previousQueries);
 					else lc.info?.("No validated background connection; deferring pipeline init");
+					if (this.#batchTimeMs !== void 0) this.#batchTimeMs += this.#totalHydrationTimeMs();
 				});
+				if (this.#pacer && this.#batchTimeMs !== void 0) await this.#paceBatches(this.#pacer, this.#batchTimeMs);
 			}
 			if (this.#drainCoordinator.shouldDrain()) this.#drainCoordinator.drainNextIn(this.#totalHydrationTimeMs());
 			await this.#cleanup();
@@ -324,9 +491,18 @@ var ViewSyncerService = class {
 	* - Syncs missing, errored, or drifted queries via `#syncQueryPipelineSet`.
 	* - Sets `#pipelinesHydrated = true` and enables shared retransformations.
 	*/
-	async #maybeHydratePipelines(lc, cvr, connCtx) {
+	async #maybeHydratePipelines(lc, cvr, connCtx, previousQueries) {
 		if (!this.#pipelines.initialized()) return;
-		const version = this.#pipelines.advanceWithoutDiff();
+		let version;
+		try {
+			version = this.#pipelines.advanceWithoutDiff(this.#pushCutoff !== void 0);
+		} catch (e) {
+			if (!(e instanceof ResetPipelinesSignal)) throw e;
+			lc.info?.(`resetting pipelines: ${e.message}${e.breakdown ? ` (${e.breakdown})` : ""}`);
+			this.#pipelineResets.add(1, { reason: e.reason });
+			this.#pipelines.reset(must(cvr.clientSchema, "cvr.clientSchema missing after initialization"));
+			version = this.#pipelines.currentVersion();
+		}
 		const cvrVer = versionString(cvr.version);
 		if (version < cvr.version.stateVersion) {
 			lc.debug?.(`replica@${version} is behind cvr@${cvrVer}`);
@@ -338,8 +514,8 @@ var ViewSyncerService = class {
 			activeHydratedQueries: 0,
 			inactiveHydratedQueries: 0
 		};
-		const driftedQueryIDs = await this.#hydrateUnchangedQueries(lc, cvr, connCtx, hydrationPassStats, hydrationBudget);
-		await this.#syncQueryPipelineSet(lc, cvr, "missing", connCtx, driftedQueryIDs, hydrationBudget, hydrationPassStats);
+		const driftedQueryIDs = await this.#hydrateUnchangedQueries(lc, cvr, connCtx, hydrationPassStats, hydrationBudget, previousQueries);
+		await this.#syncQueryPipelineSet(lc, cvr, "missing", connCtx, driftedQueryIDs, hydrationBudget, hydrationPassStats, previousQueries);
 		this.#pipelinesHydrated = true;
 		this.connContextManager.setSharedRetransformReady(true);
 	}
@@ -433,7 +609,14 @@ var ViewSyncerService = class {
 		return true;
 	}
 	#shutdownTimer = null;
+	#stopShutdownTimer() {
+		if (this.#shutdownTimer !== null) {
+			clearTimeout(this.#shutdownTimer);
+			this.#shutdownTimer = null;
+		}
+	}
 	#scheduleShutdown(delayMs = 0) {
+		if (this.#shuttingDown) return;
 		this.#shutdownTimer ??= this.#setTimeout(() => {
 			this.#shutdownTimer = null;
 			this.#runInLockWithCVR(() => {}).catch((e) => this.#stateChanges.fail(e));
@@ -480,6 +663,7 @@ var ViewSyncerService = class {
 	*/
 	#scheduleAuthMaintenance(lc) {
 		this.#stopAuthMaintenanceTimer();
+		if (this.#shuttingDown) return;
 		const plan = this.connContextManager.planMaintenance();
 		if (plan.earliestDeadlineAt === void 0) {
 			lc.debug?.("No auth maintenance wakeup scheduled");
@@ -585,17 +769,41 @@ var ViewSyncerService = class {
 		this.#ttlClockBase = now;
 		return ttlClock;
 	}
-	#flushUpdater(lc, updater) {
+	/**
+	* @param patchesPoked Whether patches computed against the updater's CVR
+	*     have already been poked to clients. If so, the CVR is checked to be
+	*     current even when the flush has nothing to write: another
+	*     view-syncer may have already committed identical rows at a newer
+	*     version, which would otherwise leave this one poking from a stale
+	*     CVR whose version it cannot advance.
+	*/
+	#flushUpdater(lc, updater, patchesPoked = false) {
 		return startAsyncSpan(tracer, "vs.#flushUpdater", () => this.#runPriorityOp(lc, "flushing cvr", async () => {
 			const now = Date.now();
 			const ttlClock = this.#getTTLClock(now);
-			const { cvr, flushed } = await updater.flush(lc, this.#lastConnectTime, now, ttlClock);
+			const { cvr, flushed } = await updater.flush(lc, this.#lastConnectTime, now, ttlClock, patchesPoked);
 			if (flushed) this.#startTTLClockInterval(lc);
 			return cvr;
 		}));
 	}
+	/**
+	* Flushes an updater whose changes have already been poked. If the flush
+	* fails (e.g. the CVR was concurrently modified), the poke is cancelled
+	* before the error propagates. The error may only fail the client that
+	* initiated the operation, and any other client left mid-poke would fail
+	* its next pokeStart.
+	*/
+	async #flushPoked(lc, updater, pokers) {
+		try {
+			return await this.#flushUpdater(lc, updater, pokers.patchesSent);
+		} catch (e) {
+			await pokers.cancel();
+			throw e;
+		}
+	}
 	#startTTLClockInterval(lc) {
 		this.#stopTTLClockInterval();
+		if (this.#shuttingDown) return;
 		this.#ttlClockInterval = this.#setTimeout(() => {
 			this.#updateTTLClockInCVRWithoutLock(lc);
 			this.#startTTLClockInterval(lc);
@@ -624,7 +832,7 @@ var ViewSyncerService = class {
 		if (cmpVersions(cvr.version, this.#cvr.version) < 0) {
 			const newCVR = this.#cvr;
 			await startAsyncSpan(tracer, "vs.#updateCVRConfig.pokeClients", async () => {
-				const pokers = startPoke(this.#getClients(cvr.version), newCVR.version);
+				const pokers = startPoke(lc, this.#getClients(cvr.version), newCVR.version);
 				for (const patch of patches) await pokers.addPatch(patch);
 				await pokers.end(newCVR.version);
 			});
@@ -726,6 +934,7 @@ var ViewSyncerService = class {
 	#scheduleExpireEviction(lc, cvr) {
 		const { ttlClock } = cvr;
 		this.#stopExpireTimer();
+		if (this.#shuttingDown) return;
 		const next = nextEvictionTime(cvr);
 		if (next === void 0) {
 			lc.debug?.("no inactive queries with ttl");
@@ -753,7 +962,7 @@ var ViewSyncerService = class {
 	*
 	* This must be called from within the #lock.
 	*/
-	async #hydrateUnchangedQueries(lc, cvr, connCtx, hydrationPassStats, hydrationBudget) {
+	async #hydrateUnchangedQueries(lc, cvr, connCtx, hydrationPassStats, hydrationBudget, previousQueries) {
 		assert(this.#pipelines.initialized(), "pipelines must be initialized");
 		const dbVersion = this.#pipelines.currentVersion();
 		const cvrVersion = cvr.version;
@@ -766,9 +975,16 @@ var ViewSyncerService = class {
 		const customQueries = /* @__PURE__ */ new Map();
 		const otherQueries = [];
 		const inactivatedCount = gotQueries.length - requiredGotQueries.length;
-		for (const query of requiredGotQueries) if (query.type === "custom") customQueries.set(query.id, query);
-		else otherQueries.push(query);
 		const transformedQueries = [];
+		for (const query of requiredGotQueries) if (query.type === "custom") {
+			const previous = previousQueries?.get(query.id);
+			if (previous && previous.transformationHash === query.transformationHash) transformedQueries.push({
+				id: query.id,
+				transformationHash: previous.transformationHash,
+				transformedAst: previous.originalAst ?? previous.transformedAst
+			});
+			else customQueries.set(query.id, query);
+		} else otherQueries.push(query);
 		let customErrorCount = 0;
 		let customHashMismatchCount = 0;
 		let otherHashMismatchCount = 0;
@@ -795,33 +1011,58 @@ var ViewSyncerService = class {
 		for (const { id: queryID, transformationHash, transformedAst } of transformedQueries) {
 			const query = cvr.queries[queryID];
 			const queryName = query.type === "custom" ? query.name : void 0;
+			if (query.type !== "internal" && this.#hydrationCircuitBreaker.isOpen(transformationHash)) {
+				lc.info?.(`skipping hydration of ${queryID}: hydration circuit breaker open`);
+				continue;
+			}
 			const covered = queryCoveringIndex ? this.#findQueryCoverageShadowHit(queryCoveringIndex, queryID, transformationHash, transformedAst, queryName) : void 0;
 			totalHydratedQueries++;
 			if (covered) {
 				coveredHydratedQueries++;
 				firstCoveredQuery ??= covered;
 			}
-			const timer = new TimeSliceTimer(lc);
+			const timer = this.#timer(lc, "hydration");
 			let count = 0;
+			let timedOut = false;
 			await startAsyncSpan(tracer, "vs.#hydrateUnchangedQueries.addQuery", async (span) => {
 				span.setAttribute("queryHash", queryID);
 				span.setAttribute("transformationHash", transformationHash);
 				span.setAttribute("table", transformedAst.table);
 				if (queryName !== void 0) span.setAttribute("queryName", queryName);
-				for (const change of this.#pipelines.addQuery(transformationHash, queryID, transformedAst, await timer.start(), queryName, "unchanged-query-rehydrate")) if (change === "yield") await timer.yieldProcess("yield in hydrateUnchangedQueries");
-				else count++;
+				for (const change of this.#pipelines.addQuery(transformationHash, queryID, transformedAst, await timer.start(), queryName, "unchanged-query-rehydrate")) if (change === "yield") {
+					if (query.type !== "internal" && this.#hydrationCircuitBreaker.exceeded(timer.totalElapsed())) {
+						timedOut = true;
+						await timer.yieldProcess("yield in hydrateUnchangedQueries");
+						break;
+					}
+					await timer.yieldProcess("yield in hydrateUnchangedQueries");
+				} else count++;
 			});
 			const elapsed = timer.totalElapsed();
+			if (timedOut) {
+				this.#recordHydrationTimeout(lc, {
+					id: queryID,
+					ast: transformedAst,
+					transformationHash,
+					name: queryName
+				}, elapsed, count);
+				continue;
+			}
 			hydrationPassStats.activeHydratedQueries++;
 			this.#hydrations.add(1);
 			this.#hydrationTime.recordMs(elapsed);
-			this.#addQueryMaterializationServerMetric(transformationHash, elapsed);
-			this.#inspectorDelegate.addQuery(transformationHash, transformedAst);
+			this.#addQueryMaterializationServerMetric(queryID, elapsed);
+			this.#inspectorDelegate.addQuery(queryID, transformedAst);
 			lc.debug?.(`hydrated ${count} rows for ${queryID} (${elapsed} ms)`);
+			if (elapsed > this.#slowHydrateThreshold) this.#logSlowHydration(lc, {
+				id: queryID,
+				ast: transformedAst,
+				transformationHash,
+				name: queryName
+			}, elapsed);
 			let drifted = false;
-			const storedSigHex = cvr.queries[queryID]?.rowSetSignature;
-			if (storedSigHex !== void 0 && storedSigHex !== null) {
-				const priorSig = parseSignature(storedSigHex);
+			const priorSig = this.#cvrStore.rowSetSignature(queryID);
+			if (priorSig !== void 0) {
 				const candidateSig = this.#pipelines.rowSetSignature(queryID) ?? 0n;
 				if (priorSig !== candidateSig) {
 					lc.warn?.(`rowSetSignature drift for query ${queryID}: prior=${priorSig.toString(16)} new=${candidateSig.toString(16)} (${count} rows). Removing from pipelines for full re-execution.`);
@@ -877,11 +1118,7 @@ var ViewSyncerService = class {
 			return;
 		}
 		const appErrorGroups = /* @__PURE__ */ new Map();
-		for (const err of errorOrErrors) for (const clientId of getAffectedClientIDs([err.id])) {
-			const group = appErrorGroups.get(clientId) ?? [];
-			group.push(err);
-			appErrorGroups.set(clientId, group);
-		}
+		for (const err of errorOrErrors) for (const clientId of getAffectedClientIDs([err.id])) getOrInsertComputed(appErrorGroups, clientId, newArray).push(err);
 		for (const [clientId, errors] of appErrorGroups) this.#clients.get(clientId)?.sendQueryTransformApplicationErrors(errors);
 	}
 	#addQueryMaterializationServerMetric(queryID, elapsed) {
@@ -920,7 +1157,7 @@ var ViewSyncerService = class {
 	#syncQueryPipelineSet(lc, cvr, customQueryTransformMode, connCtx, driftedQueryIDs = /* @__PURE__ */ new Set(), hydrationBudget = new HydrationBudget(this.#config.viewSyncerHydrationBudgetMs ?? 0, this.#now), hydrationPassStats = {
 		activeHydratedQueries: 0,
 		inactiveHydratedQueries: 0
-	}) {
+	}, previousQueries) {
 		return startAsyncSpan(tracer, "vs.#syncQueryPipelineSet", async (span) => {
 			const start = performance.now();
 			span.setAttribute("clientGroupID", this.id);
@@ -951,7 +1188,13 @@ var ViewSyncerService = class {
 					});
 				}
 			};
-			const shouldTransformCustomQuery = (query) => customQueryTransformMode === "all" || customQueryTransformMode && !this.#pipelines.queries().has(query.id);
+			const shouldTransformCustomQuery = (query) => {
+				if (customQueryTransformMode === "all") return true;
+				if (this.#pipelines.queries().has(query.id)) return false;
+				const previous = previousQueries?.get(query.id);
+				if (previous && previous.transformationHash === query.transformationHash) return false;
+				return true;
+			};
 			const transformCustomQueries = async (queries) => {
 				if (!customQueryTransformer || queries.length === 0) return;
 				transformedCustomQueryCount += queries.length;
@@ -999,8 +1242,35 @@ var ViewSyncerService = class {
 			};
 			const queriesToTransform = [...required, ...optionalToHydrate];
 			transformOtherQueries(queriesToTransform);
+			if (customQueryTransformMode === "missing" && previousQueries) {
+				for (const query of queriesToTransform) if (query.type === "custom" && !this.#pipelines.queries().has(query.id)) {
+					const previous = previousQueries.get(query.id);
+					if (previous && previous.transformationHash === query.transformationHash) {
+						this.#queryTransformationNoOps.add(1);
+						transformedQueries.push({
+							id: query.id,
+							origQuery: query,
+							transformed: {
+								id: query.id,
+								transformationHash: previous.transformationHash,
+								transformedAst: previous.originalAst ?? previous.transformedAst
+							}
+						});
+					}
+				}
+			}
 			await transformCustomQueries(queriesToTransform.filter((query) => query.type === "custom" && shouldTransformCustomQuery(query)));
-			const removeQueriesQueryIds = new Set([...naturallyExpiredQueryIDs, ...erroredQueryIDs]);
+			const circuitBrokenQueries = transformedQueries.filter(({ id, origQuery, transformed }) => origQuery.type !== "internal" && this.#pipelines.queries().get(id)?.transformationHash !== transformed.transformationHash && this.#hydrationCircuitBreaker.isOpen(transformed.transformationHash)).map(({ id, origQuery, transformed }) => ({
+				id,
+				transformationHash: transformed.transformationHash,
+				name: origQuery.type === "custom" ? origQuery.name : void 0
+			}));
+			if (circuitBrokenQueries.length > 0) this.#rejectCircuitBrokenQueries(lc, cvr, circuitBrokenQueries);
+			const removeQueriesQueryIds = new Set([
+				...naturallyExpiredQueryIDs,
+				...erroredQueryIDs,
+				...circuitBrokenQueries.map(({ id }) => id)
+			]);
 			const addQueries = transformedQueries.map(({ id, origQuery, transformed }) => ({
 				id,
 				ast: transformed.transformedAst,
@@ -1011,7 +1281,7 @@ var ViewSyncerService = class {
 			addQueries.sort((a, b) => must(hydrationOrder.get(a.id)) - must(hydrationOrder.get(b.id)));
 			const requiredAddQueries = addQueries.filter((query) => requiredQueryIDs.has(query.id));
 			const optionalAddQueries = addQueries.filter((query) => !requiredQueryIDs.has(query.id));
-			lc.info?.(`syncQueryPipelineSet: ${cvrQueryEntires.length} CVR queries, ${transformedCustomQueryCount} custom re-transformed, ${erroredQueryIDs.length} errored, ${removeQueriesQueryIds.size} to remove, ${addQueries.length} to add`);
+			lc.info?.(`syncQueryPipelineSet: ${cvrQueryEntires.length} CVR queries, ${transformedCustomQueryCount} custom re-transformed, ${erroredQueryIDs.length} errored, ${circuitBrokenQueries.length} circuit-broken, ${removeQueriesQueryIds.size} to remove, ${addQueries.length} to add`);
 			for (const q of addQueries) {
 				const orig = cvr.queries[q.id];
 				lc.debug?.("ViewSyncer adding query", q.ast, "transformed from", orig.type === "custom" ? orig.name : orig.ast);
@@ -1050,6 +1320,107 @@ var ViewSyncerService = class {
 		record.count++;
 		if (record.count >= THRASH_THRESHOLD) this.#lc.warn?.(`Query thrashing detected for query ${queryID}. ${record.count} replacements in 60s. This may indicate clients with different auth contexts connecting to the same client group.`);
 	}
+	/**
+	* Records that hydrating `query` was aborted for exceeding the query
+	* hydration timeout, and opens its circuit breaker.
+	*/
+	#recordHydrationTimeout(lc, query, elapsedMs, rowCount) {
+		this.#hydrationCircuitBreaker.trip(query.transformationHash);
+		this.#hydrationTimeouts.add(1);
+		if (!lc.warn) return;
+		const shape = queryShape(query.ast);
+		lc.warn("Query hydration aborted for exceeding the hydration timeout", {
+			clientGroupID: this.id,
+			queryHash: query.id,
+			transformationHash: query.transformationHash,
+			...query.name !== void 0 && { queryName: query.name },
+			queryShape: shape.hash,
+			hydrationTimeoutMs: this.#hydrationCircuitBreaker.timeoutMs,
+			hydrationElapsedMs: elapsedMs,
+			...rowCount !== void 0 && { hydrationRowCount: rowCount },
+			circuitBreakerOpenMs: this.#hydrationCircuitBreaker.openMs,
+			zql: shape.zql
+		});
+	}
+	/**
+	* Logs a hydration that exceeded the slow hydration threshold.
+	*
+	* The query is logged as its {@link queryShape}, with literal values
+	* redacted, and each shape is logged at most once per
+	* {@link SLOW_HYDRATION_LOG_WINDOW_MS}. The next log of a shape reports how
+	* many slow hydrations of it were suppressed in the meantime.
+	*/
+	#logSlowHydration(lc, query, elapsedMs) {
+		if (!lc.warn) return;
+		const stats = this.#pipelines.hydrationStats(query.id);
+		const shape = stats?.shape ?? queryShape(query.ast);
+		const suppressed = slowHydrationLogThrottle.admit(`${query.name ?? ""}:${shape.hash}`);
+		if (suppressed === void 0) return;
+		lc.warn("Slow query materialization", {
+			zeroEvent: "query-slow-hydration",
+			clientGroupID: this.id,
+			queryHash: query.id,
+			transformationHash: query.transformationHash,
+			...query.name !== void 0 && { queryName: query.name },
+			queryShape: shape.hash,
+			hydrationTimeMs: elapsedMs,
+			...stats && {
+				hydrationRowCount: stats.rowCount,
+				hydrationRowsRead: stats.rowsRead
+			},
+			...stats !== void 0 && stats.planWarnings.length > 0 && { planWarnings: stats.planWarnings.map(planWarningMessage) },
+			...suppressed > 0 && { suppressedSinceLastLog: suppressed },
+			zql: shape.zql
+		});
+	}
+	/** Records that `query` was rejected without hydration by its open breaker. */
+	#recordCircuitBreakerRejection(lc, query) {
+		this.#hydrationCircuitBreakerRejections.add(1);
+		lc.warn?.("Query rejected by its open hydration circuit breaker", {
+			clientGroupID: this.id,
+			queryHash: query.id,
+			transformationHash: query.transformationHash,
+			...query.name !== void 0 && { queryName: query.name },
+			hydrationTimeoutMs: this.#hydrationCircuitBreaker.timeoutMs,
+			circuitBreakerOpenMs: this.#hydrationCircuitBreaker.openMs
+		});
+	}
+	/**
+	* Handles queries whose hydration circuit breaker is open: they are counted,
+	* logged, and errored to the affected clients. The caller removes them from
+	* the CVR.
+	*/
+	#rejectCircuitBrokenQueries(lc, cvr, queries) {
+		for (const query of queries) this.#recordCircuitBreakerRejection(lc, query);
+		this.#queryEvictions.add(queries.length, { reason: "hydration-circuit-breaker" });
+		this.#sendHydrationTimeoutErrors(cvr, queries);
+	}
+	/**
+	* Sends a per-query error to every client that desires one of `queries`.
+	* The error goes out as a `transformError` application error, which every
+	* client understands as "this query errored" without affecting the
+	* connection or the client's other queries.
+	*/
+	#sendHydrationTimeoutErrors(cvr, queries) {
+		const timeoutMs = this.#hydrationCircuitBreaker.timeoutMs;
+		const errorsByClient = /* @__PURE__ */ new Map();
+		for (const { id, name } of queries) {
+			const query = cvr.queries[id];
+			if (query === void 0 || query.type === "internal") continue;
+			const error = {
+				error: "app",
+				id,
+				name: name ?? (query.type === "custom" ? query.name : "legacy"),
+				message: `Query hydration exceeded the ${timeoutMs}ms limit (ZERO_VIEW_SYNCER_QUERY_HYDRATION_TIMEOUT_MS) and was aborted`,
+				details: {
+					kind: "HydrationTimeout",
+					timeoutMs
+				}
+			};
+			for (const clientID of Object.keys(query.clientState)) getOrInsertComputed(errorsByClient, clientID, newArray).push(error);
+		}
+		for (const [clientID, errors] of errorsByClient) this.#clients.get(clientID)?.sendQueryTransformApplicationErrors(errors);
+	}
 	#recordHydrationBudgetExhaustion(lc, hydrationBudget, activeHydratedQueries, inactiveHydratedQueries, inactiveEvictedQueryIDs) {
 		const elapsedMs = hydrationBudget.exhaustedAtMs;
 		if (elapsedMs === void 0) return;
@@ -1076,7 +1447,7 @@ var ViewSyncerService = class {
 			const stateVersion = this.#pipelines.currentVersion();
 			lc = lc.withContext("stateVersion", stateVersion);
 			lc.info?.(`hydrating ${addQueries.length} queries`);
-			const updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, stateVersion, this.#pipelines.replicaVersion, (queryID) => this.#pipelines.rowSetSignature(queryID));
+			const updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, stateVersion, this.#pipelines.replicaVersion, this.#pipelines);
 			const sameHashRehydratedQueryIDs = addQueries.filter((q) => cvr.queries[q.id]?.transformationHash === q.transformationHash).map((q) => q.id);
 			const trackQueriesWillBumpVersion = stateVersion > cvr.version.stateVersion || removeQueries.length > 0 || addQueries.some((q) => cvr.queries[q.id]?.transformationHash !== q.transformationHash);
 			if (sameHashRehydratedQueryIDs.length > 0 && !trackQueriesWillBumpVersion) {
@@ -1088,7 +1459,8 @@ var ViewSyncerService = class {
 			}
 			const { queryPatches, newVersion } = updater.trackQueries(lc, addQueries, removeQueries);
 			if (hydrationBudget.limitMs !== 0 && optionalQueries.length > 0) assert(cmpVersions(cvr.version, newVersion) < 0, "Optional hydration requires a final poke version before row processing so a removal is actually poked");
-			const pokers = startPoke(this.#getClients(), newVersion);
+			const clients = this.#getClients();
+			const pokers = startPoke(lc, clients, newVersion);
 			for (const patch of queryPatches) await pokers.addPatch(patch);
 			for (const q of removeQueries) {
 				this.#pipelines.removeQuery(q.id);
@@ -1096,7 +1468,7 @@ var ViewSyncerService = class {
 				this.#queryReplacements.delete(q.id);
 			}
 			let totalProcessTime = 0;
-			const timer = new TimeSliceTimer(lc);
+			const timer = this.#timer(lc, "hydration");
 			const pipelines = this.#pipelines;
 			const hydrations = this.#hydrations;
 			const hydrationTime = this.#hydrationTime;
@@ -1106,8 +1478,11 @@ var ViewSyncerService = class {
 			let firstCoveredQuery;
 			const hydratedQueryIDs = [];
 			const budgetEvictedQueryIDs = [];
+			const timedOutQueries = [];
+			const rejectedQueries = [];
+			const circuitBreaker = this.#hydrationCircuitBreaker;
 			const self = this;
-			await yieldProcess(lc);
+			await (this.#pacer ? this.#pacer.yieldSlice("hydration") : yieldProcess(lc));
 			function* hydrateQueries(queries, optional, slowHydrateThreshold) {
 				for (let i = 0; i < queries.length; i++) {
 					if (optional && hydrationBudget.exhausted()) {
@@ -1118,15 +1493,34 @@ var ViewSyncerService = class {
 					let queryLC = lc.withContext("hash", q.id).withContext("queryHash", q.id).withContext("transformationHash", q.transformationHash);
 					if (q.name !== void 0) queryLC = queryLC.withContext("queryName", q.name);
 					queryLC.debug?.(`adding pipeline for query`, q.ast);
+					const breakable = cvr.queries[q.id]?.type !== "internal";
+					if (breakable && circuitBreaker.isOpen(q.transformationHash)) {
+						rejectedQueries.push(q);
+						self.#recordCircuitBreakerRejection(queryLC, q);
+						continue;
+					}
 					const covered = queryCoveringIndex ? self.#findQueryCoverageShadowHit(queryCoveringIndex, q.id, q.transformationHash, q.ast, q.name) : void 0;
 					totalHydratedQueries++;
 					if (covered) {
 						coveredHydratedQueries++;
 						firstCoveredQuery ??= covered;
 					}
-					yield* pipelines.addQuery(q.transformationHash, q.id, q.ast, timer.startWithoutYielding(), q.name, "query-set-sync");
+					let timedOut = false;
+					for (const change of pipelines.addQuery(q.transformationHash, q.id, q.ast, timer.startWithoutYielding(), q.name, "query-set-sync")) {
+						if (change === "yield" && breakable && circuitBreaker.exceeded(timer.totalElapsed())) {
+							timedOut = true;
+							yield change;
+							break;
+						}
+						yield change;
+					}
 					const elapsed = timer.stop();
 					totalProcessTime += elapsed;
+					if (timedOut) {
+						timedOutQueries.push(q);
+						self.#recordHydrationTimeout(queryLC, q, elapsed);
+						continue;
+					}
 					hydratedQueryIDs.push(q.id);
 					if (optional) hydrationPassStats.inactiveHydratedQueries++;
 					else hydrationPassStats.activeHydratedQueries++;
@@ -1137,7 +1531,7 @@ var ViewSyncerService = class {
 						transformationHash: q.transformationHash,
 						...q.name !== void 0 && { queryName: q.name }
 					});
-					if (elapsed > slowHydrateThreshold) queryLC.warn?.("Slow query materialization", elapsed, q.ast);
+					if (elapsed > slowHydrateThreshold) self.#logSlowHydration(queryLC, q, elapsed);
 					manualSpan(tracer, "vs.addAndConsumeQuery", elapsed, {
 						hash: q.id,
 						transformationHash: q.transformationHash,
@@ -1151,7 +1545,20 @@ var ViewSyncerService = class {
 				yield* hydrateQueries(requiredQueries, false, slowHydrateThreshold);
 				yield* hydrateQueries(optionalQueries, true, slowHydrateThreshold);
 			}
-			await this.#processChanges(lc, timer, generateRowChanges(this.#slowHydrateThreshold), updater, pokers);
+			await this.#processChanges(lc, timer, generateRowChanges(this.#slowHydrateThreshold), (rows) => updater.received(lc, rows), pokers);
+			const abortedQueries = [...timedOutQueries, ...rejectedQueries];
+			if (abortedQueries.length > 0) {
+				const abortedQueryIDs = abortedQueries.map(({ id }) => id);
+				for (const patch of await updater.abortExecutedQueries(lc, abortedQueryIDs)) await pokers.addPatch(patch);
+				for (const queryID of abortedQueryIDs) {
+					this.#pipelines.removeQuery(queryID);
+					this.#inspectorDelegate.removeQuery(queryID);
+					this.#queryReplacements.delete(queryID);
+				}
+				if (timedOutQueries.length > 0) this.#queryEvictions.add(timedOutQueries.length, { reason: "hydration-timeout" });
+				if (rejectedQueries.length > 0) this.#queryEvictions.add(rejectedQueries.length, { reason: "hydration-circuit-breaker" });
+				this.#sendHydrationTimeoutErrors(cvr, abortedQueries);
+			}
 			for (const patch of updater.removeTrackedQueries(budgetEvictedQueryIDs)) await pokers.addPatch(patch);
 			for (const queryID of budgetEvictedQueryIDs) {
 				this.#pipelines.removeQuery(queryID);
@@ -1163,7 +1570,7 @@ var ViewSyncerService = class {
 			await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.deleteUnreferencedRows", async () => {
 				for (const patch of await updater.deleteUnreferencedRows(lc)) await pokers.addPatch(patch);
 			});
-			this.#cvr = await this.#flushUpdater(lc, updater);
+			this.#cvr = await this.#flushPoked(lc, updater, pokers);
 			if (budgetEvictedQueryIDs.length > 0) this.#scheduleExpireEviction(lc, this.#cvr);
 			const finalVersion = this.#cvr.version;
 			await this.#catchupClients(lc, cvr, finalVersion, hydratedQueryIDs, pokers);
@@ -1195,7 +1602,7 @@ var ViewSyncerService = class {
 		return startAsyncSpan(tracer, "vs.#catchupClients", async (span) => {
 			current ??= cvr.version;
 			const clients = this.#getClients();
-			const pokers = usePokers ?? startPoke(clients, cvr.version);
+			const pokers = usePokers ?? startPoke(lc, clients, cvr.version);
 			span.setAttribute("numClients", clients.length);
 			const catchupFrom = clients.map((c) => c.version()).reduce((a, b) => cmpVersions(a, b) < 0 ? a : b, cvr.version);
 			const rowPatches = this.#cvrStore.catchupRowPatches(lc, catchupFrom, cvr, current, excludeQueryHashes);
@@ -1242,7 +1649,7 @@ var ViewSyncerService = class {
 			}
 		});
 	}
-	#processChanges(lc, timer, changes, updater, pokers) {
+	#processChanges(lc, timer, changes, onBatch, pokers) {
 		return startAsyncSpan(tracer, "vs.#processChanges", async () => {
 			const start = performance.now();
 			const rows = new CustomKeyMap(rowIDString);
@@ -1251,7 +1658,7 @@ var ViewSyncerService = class {
 				const wallElapsed = performance.now() - start;
 				total += rows.size;
 				lc.debug?.(`processing ${rows.size} (of ${total}) rows (${wallElapsed} ms)`);
-				const patches = await updater.received(lc, rows);
+				const patches = await onBatch(rows);
 				await startAsyncSpan(tracer, "processBatch.flushToClient", async (span) => {
 					span.setAttribute("patches", patches.length);
 					for (const patch of patches) await pokers.addPatch(patch);
@@ -1316,32 +1723,304 @@ var ViewSyncerService = class {
 			span.setAttribute("clientGroupID", this.id);
 			assert(this.#pipelines.initialized(), "pipelines must be initialized (advancePipelines");
 			const start = performance.now();
-			const timer = new TimeSliceTimer(lc);
-			const { version, numChanges, changes } = this.#pipelines.advance(timer);
-			lc = lc.withContext("newVersion", version);
-			const updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, version, this.#pipelines.replicaVersion, (queryID) => this.#pipelines.rowSetSignature(queryID));
-			const pokers = startPoke(this.#getClients(cvr.version), updater.updatedVersion());
-			lc.debug?.(`applying ${numChanges} to advance to ${version}`);
+			const timer = this.#timer(lc, "batch");
+			let outcome = { outcome: "error" };
+			let setupTimeMs;
+			let rebuildTimeMs = 0;
+			let evictedQueries = false;
+			this.#pipelineTime.ms = 0;
 			try {
-				await this.#processChanges(lc, await timer.start(), changes, updater, pokers);
-			} catch (e) {
-				if (e instanceof ResetPipelinesSignal) {
-					await pokers.cancel();
-					return e;
+				let pokers;
+				let updater;
+				let version;
+				let numChanges = 0;
+				try {
+					const advancement = this.#pipelines.advance(timer, this.#costBasedAdvance(cvr));
+					version = advancement.version;
+					numChanges = advancement.numChanges;
+					lc = lc.withContext("newVersion", version);
+					const targetVersion = cmpVersions(cvr.version, { stateVersion: version }) < 0 ? { stateVersion: version } : cvr.version;
+					pokers = startPoke(lc, this.#getClients(cvr.version), targetVersion);
+					lc.debug?.(`applying ${numChanges} to advance to ${version}`);
+					setupTimeMs = performance.now() - start;
+					await this.#processChanges(lc, await timer.start(), this.#timed(advancement.changes), (rows) => {
+						updater ??= new CVRQueryDrivenUpdater(this.#cvrStore, cvr, version, this.#pipelines.replicaVersion, this.#pipelines);
+						return updater.received(lc, rows);
+					}, pokers);
+					for (const { estimatedPushMs, pushTimeMs } of advancement.estimates()) if (pushTimeMs > 0) this.#pushEstimateRatio.record(estimatedPushMs / pushTimeMs);
+					const abandoned = advancement.abandoned();
+					if (abandoned.length > 0) {
+						updater ??= new CVRQueryDrivenUpdater(this.#cvrStore, cvr, version, this.#pipelines.replicaVersion, this.#pipelines);
+						timer.stop();
+						({rebuildTimeMs, evictedQueries} = await this.#rebuildAbandonedQueries(lc, cvr, abandoned, updater, pokers));
+					}
+				} catch (e) {
+					setupTimeMs ??= performance.now() - start;
+					if (e instanceof ResetPipelinesSignal) {
+						await pokers?.cancel();
+						this.#cvrStore.discardPending();
+						outcome = {
+							outcome: "reset",
+							reason: e.reason
+						};
+						return e;
+					}
+					throw e;
 				}
-				throw e;
+				assert(pokers && version !== void 0, "advancement state missing");
+				if (updater) {
+					this.#cvr = await this.#flushPoked(lc, updater, pokers);
+					if (evictedQueries) this.#scheduleExpireEviction(lc, this.#cvr);
+				}
+				const finalVersion = (this.#cvr ?? cvr).version;
+				await startAsyncSpan(tracer, "vs.#advancePipelines.pokeEnd", () => pokers.end(finalVersion));
+				this.#markVersionServed(version);
+				const wallTime = performance.now() - start;
+				const totalProcessTime = timer.totalElapsed() + rebuildTimeMs;
+				lc.debug?.(`finished processing advancement of ${numChanges} changes ((process: ${totalProcessTime} ms, wall: ${wallTime} ms))`);
+				this.#transactionAdvanceTime.recordMs(totalProcessTime);
+				outcome = { outcome: "success" };
+				return "success";
+			} finally {
+				const setupMs = must(setupTimeMs);
+				const processingTimeMs = setupMs + timer.totalElapsed() + rebuildTimeMs;
+				this.#batchTimeMs = setupMs + this.#pipelineTime.ms;
+				const wallTimeMs = performance.now() - start;
+				this.#advancementMetrics.record({
+					processingTimeMs,
+					setupTimeMs: setupMs,
+					wallTimeMs
+				}, outcome);
 			}
-			this.#cvr = await this.#flushUpdater(lc, updater);
-			const finalVersion = this.#cvr.version;
-			await startAsyncSpan(tracer, "vs.#advancePipelines.pokeEnd", () => pokers.end(finalVersion));
-			this.#markVersionServed(version);
-			const wallTime = performance.now() - start;
-			const totalProcessTime = timer.totalElapsed();
-			lc.debug?.(`finished processing advancement of ${numChanges} changes ((process: ${totalProcessTime} ms, wall: ${wallTime} ms))`);
-			this.#transactionAdvanceTime.recordMs(totalProcessTime);
-			return "success";
 		});
 	}
+	/**
+	* What a cost-based advancement needs, or `undefined` if advancements are
+	* bounded by the advancement time limit.
+	*/
+	#costBasedAdvance(cvr) {
+		const shouldCutOff = this.#pushCutoff;
+		if (shouldCutOff === void 0) return;
+		if (this.#pipelines.currentVersion() < cvr.version.stateVersion) return;
+		return {
+			shouldCutOff,
+			rebuildMargin: this.#config.syncRebuildMargin ?? 2
+		};
+	}
+	/**
+	* Hydrates the queries whose pipelines an advancement abandoned, at the
+	* snapshot it advanced to, within the advancement's CVR update and poke.
+	* What the queries contributed to the update before they were abandoned
+	* is retracted first.
+	*
+	* The hydrations are protected as they are in a hydration pass (see
+	* {@link #addAndRemoveQueries}). A query that exceeds the query hydration
+	* timeout, or whose circuit breaker is open, is removed and errored to its
+	* clients. Inactive queries are hydrated last, and those not reached
+	* before the hydration budget is spent are evicted.
+	*
+	* @returns the time spent hydrating, and whether inactive queries were
+	*          evicted.
+	*/
+	async #rebuildAbandonedQueries(lc, cvr, abandoned, updater, pokers) {
+		for (const patch of await updater.retractQueries(lc, abandoned.map((q) => q.queryID))) await pokers.addPatch(patch);
+		const hydrationBudget = new HydrationBudget(this.#config.viewSyncerHydrationBudgetMs ?? 0, this.#now);
+		const abandonedByID = new Map(abandoned.map((q) => [q.queryID, q]));
+		const { required, optional } = classifyQueriesForHydration(abandoned.map((q) => must(cvr.queries[q.queryID])));
+		const byRecord = (queries) => queries.map(({ id }) => must(abandonedByID.get(id)));
+		const timer = this.#timer(lc, "batch");
+		const pipelines = this.#pipelines;
+		const circuitBreaker = this.#hydrationCircuitBreaker;
+		const budgetEvicted = [];
+		const timedOut = [];
+		const rejected = [];
+		const rebuilt = {
+			active: 0,
+			inactive: 0
+		};
+		const self = this;
+		let total = 0;
+		function* rebuild(queries, inactive) {
+			for (let i = 0; i < queries.length; i++) {
+				if (inactive && hydrationBudget.exhausted()) {
+					budgetEvicted.push(...queries.slice(i));
+					return;
+				}
+				const q = must(queries[i]);
+				const hydrationQuery = {
+					id: q.queryID,
+					ast: q.originalAst,
+					transformationHash: q.transformationHash,
+					name: q.queryName
+				};
+				const breakable = cvr.queries[q.queryID]?.type !== "internal";
+				if (breakable && circuitBreaker.isOpen(q.transformationHash)) {
+					rejected.push(q);
+					self.#recordCircuitBreakerRejection(lc, hydrationQuery);
+					continue;
+				}
+				let aborted = false;
+				for (const change of pipelines.addQuery(q.transformationHash, q.queryID, q.originalAst, timer.startWithoutYielding(), q.queryName, "unchanged-query-rehydrate")) {
+					if (change === "yield" && breakable && circuitBreaker.exceeded(timer.totalElapsed())) {
+						aborted = true;
+						yield change;
+						break;
+					}
+					yield change;
+				}
+				const elapsed = timer.stop();
+				total += elapsed;
+				if (aborted) {
+					timedOut.push(q);
+					self.#recordHydrationTimeout(lc, hydrationQuery, elapsed);
+					continue;
+				}
+				if (inactive) rebuilt.inactive++;
+				else rebuilt.active++;
+				self.#queryRebuilds.add(1, { reason: q.reason });
+				self.#queryRebuildTime.recordMs(elapsed);
+				self.#addQueryMaterializationServerMetric(q.queryID, elapsed);
+				if (elapsed > self.#slowHydrateThreshold) self.#logSlowHydration(lc, hydrationQuery, elapsed);
+			}
+		}
+		function* rebuildAll() {
+			yield* rebuild(byRecord(required), false);
+			yield* rebuild(byRecord(optional), true);
+		}
+		await this.#processChanges(lc, timer, this.#timed(rebuildAll()), (rows) => updater.received(lc, rows), pokers);
+		const abortedQueries = [...timedOut, ...rejected];
+		if (abortedQueries.length > 0) {
+			for (const patch of await updater.abortExecutedQueries(lc, abortedQueries.map((q) => q.queryID))) await pokers.addPatch(patch);
+			if (timedOut.length > 0) this.#queryEvictions.add(timedOut.length, { reason: "hydration-timeout" });
+			if (rejected.length > 0) this.#queryEvictions.add(rejected.length, { reason: "hydration-circuit-breaker" });
+			this.#sendHydrationTimeoutErrors(cvr, abortedQueries.map((q) => ({
+				id: q.queryID,
+				name: q.queryName
+			})));
+		}
+		const evictedQueryIDs = budgetEvicted.map((q) => q.queryID);
+		for (const patch of updater.removeTrackedQueries(evictedQueryIDs)) await pokers.addPatch(patch);
+		for (const { queryID } of [...abortedQueries, ...budgetEvicted]) {
+			this.#pipelines.removeQuery(queryID);
+			this.#inspectorDelegate.removeQuery(queryID);
+			this.#queryReplacements.delete(queryID);
+		}
+		this.#recordHydrationBudgetExhaustion(lc, hydrationBudget, rebuilt.active, rebuilt.inactive, evictedQueryIDs);
+		for (const patch of await updater.deleteUnreferencedRows(lc)) await pokers.addPatch(patch);
+		this.#logRebuilds(lc, abandoned, {
+			hydrated: rebuilt.active + rebuilt.inactive,
+			hydrationTimeMs: total,
+			timedOut: timedOut.length,
+			rejectedByCircuitBreaker: rejected.length,
+			evictedForHydrationBudget: budgetEvicted.length
+		});
+		return {
+			rebuildTimeMs: total,
+			evictedQueries: evictedQueryIDs.length > 0
+		};
+	}
+	/**
+	* Says in the log that this client group's batches hydrate queries again,
+	* which can be every batch. Nothing else is logged at `info` for a batch,
+	* so this is at most once a minute, for the batches since the last time,
+	* and not before their hydrations add up to a second. Which queries they
+	* were is in the query stats, by query shape (see `QueryStats`).
+	*/
+	#logRebuilds(lc, abandoned, batch) {
+		const log = this.#rebuildLog;
+		log.batches++;
+		log.queries += abandoned.length;
+		for (const { reason } of abandoned) log[reason]++;
+		if (abandoned.some((q) => q.reason === "read")) log.batchesNotRead++;
+		log.hydrated += batch.hydrated;
+		log.hydrationTimeMs += batch.hydrationTimeMs;
+		log.timedOut += batch.timedOut;
+		log.rejectedByCircuitBreaker += batch.rejectedByCircuitBreaker;
+		log.evictedForHydrationBudget += batch.evictedForHydrationBudget;
+		const now = this.#now();
+		if (log.hydrationTimeMs < REBUILD_LOG_MIN_MS || now - this.#rebuildsLoggedAt < REBUILD_LOG_INTERVAL_MS) return;
+		lc.info?.("Client group hydrated queries again in its batches of changes instead of pushing the changes to them", {
+			clientGroupID: this.id,
+			...log
+		});
+		this.#rebuildsLoggedAt = now;
+		this.#rebuildLog = newRebuildLog();
+	}
+	/**
+	* Adds the time that the pipelines take to produce `values` to the time
+	* of the batch being processed (see {@link #batchTimeMs}).
+	*/
+	#timed(values) {
+		return this.#pacer ? timeIteration(values, this.#pipelineTime) : values;
+	}
+	/**
+	* A timer for a batch, or for the hydration of new queries, which has
+	* priority over batches for the worker's time slices (see
+	* `BatchPacer.yieldSlice()`).
+	*/
+	#timer(lc, kind) {
+		const pacer = this.#pacer;
+		return new TimeSliceTimer(lc, pacer && (() => pacer.yieldSlice(kind)));
+	}
+	/**
+	* Waits as long as the worker's pacer says before the next batch, after
+	* one that took `batchTimeMs`. The wait is outside of the
+	* lock, so connections and query changes are served as usual, at the
+	* pipelines' current snapshot. `version-ready` notifications received in
+	* the meantime are coalesced by the subscription, so the next batch covers
+	* all of them.
+	*/
+	async #paceBatches(pacer, batchTimeMs) {
+		this.#smoothedBatchTimeMs = this.#smoothedBatchTimeMs === void 0 ? batchTimeMs : this.#smoothedBatchTimeMs + BATCH_TIME_SMOOTHING * (batchTimeMs - this.#smoothedBatchTimeMs);
+		const processingMs = this.#smoothedBatchTimeMs * (1 + BATCH_WAIT_JITTER * (Math.random() * 2 - 1));
+		const start = performance.now();
+		let waited = false;
+		for (;;) {
+			const remaining = pacer.waitMs(processingMs) - (performance.now() - start);
+			if (remaining <= 0 || !this.#stateChanges.active) break;
+			waited = true;
+			await new Promise((resolve) => {
+				const timer = setTimeout(resolve, Math.min(remaining, BATCH_WAIT_STEP_MS));
+				this.#endBatchWait = () => {
+					clearTimeout(timer);
+					resolve();
+				};
+			});
+			this.#endBatchWait = void 0;
+		}
+		if (waited) {
+			const waitedMs = performance.now() - start;
+			this.#batchWaitTime.recordMs(waitedMs);
+			this.#logBatchWait(pacer, waitedMs);
+		}
+	}
+	/**
+	* Says in the log that this client group is being held back, which the
+	* worker's metrics do not tell one client group from another by. A client
+	* group that is held back is after every batch, so this is at most once
+	* a minute, for the waits since the last time, and not for waits that add
+	* up to less than a second.
+	*/
+	#logBatchWait(pacer, waitedMs) {
+		const log = this.#batchWaitLog;
+		log.waits++;
+		log.waitedMs += waitedMs;
+		const now = this.#now();
+		if (log.waitedMs < BATCH_WAIT_LOG_MIN_MS || now - log.loggedAt < BATCH_WAIT_LOG_INTERVAL_MS) return;
+		this.#lc.info?.("Client group is waiting between batches: its sync worker is over its share target", {
+			clientGroupID: this.id,
+			waits: log.waits,
+			waitedMs: log.waitedMs,
+			lastWaitMs: waitedMs,
+			batchTimeMs: this.#smoothedBatchTimeMs,
+			waitMultiplier: pacer.multiplier
+		});
+		this.#batchWaitLog = {
+			loggedAt: now,
+			waits: 0,
+			waitedMs: 0
+		};
+	}
 	async inspect(selector, msg) {
 		await this.#runInLockForClient(selector, msg, this.#handleInspect);
 	}
@@ -1351,7 +2030,7 @@ var ViewSyncerService = class {
 			clientID,
 			wsID: client.wsID
 		});
-		return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this.#cvrStore, this.#config, connCtx);
+		return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this, this.#cvrStore, this.#config, connCtx);
 	};
 	async #runBackgroundRetransform(lc) {
 		const attemptRetransform = async (connCtx) => {
@@ -1435,18 +2114,22 @@ var ViewSyncerService = class {
 		this.connContextManager.setSharedRetransformReady(false);
 		this.#initialized.reject(shutdownBeforeInitializationError());
 		this.#stateChanges.cancel();
+		this.#endBatchWait?.();
 		return this.#stopped.promise;
 	}
 	async #cleanup(err) {
+		this.#shuttingDown = true;
 		this.connContextManager.setSharedRetransformReady(false);
 		this.#stopTTLClockInterval();
 		this.#stopExpireTimer();
 		this.#stopAuthMaintenanceTimer();
+		this.#stopShutdownTimer();
 		this.#customQueryTransformer?.destroy();
 		for (const client of this.#clients.values()) if (err) client.fail(err);
 		else client.close(`closed clientGroupID=${this.id}`);
 		await this.#lock.withLock(() => {});
 		this.#pipelines.destroy();
+		this.#inspectorDelegate.clearAuthenticated(this.id, this);
 	}
 	/**
 	* Test helper: Manually mark initialization as complete.
@@ -1502,15 +2185,49 @@ function hasExpiredQueries(cvr) {
 	for (const q of Object.values(cvr.queries)) if (expired(ttlClock, q)) return true;
 	return false;
 }
+/**
+* Iterates over `values`, adding the time spent producing them to
+* `total.ms`. The time that the consumer takes between values, including
+* any that it spends awaiting, is not counted.
+*/
+function* timeIteration(values, total) {
+	const iterator = values[Symbol.iterator]();
+	try {
+		for (;;) {
+			const start = performance.now();
+			const result = iterator.next();
+			total.ms += performance.now() - start;
+			if (result.done) return;
+			yield result.value;
+		}
+	} finally {
+		iterator.return?.();
+	}
+}
+/** How often a client group that is waiting between batches asks again. */
+var BATCH_WAIT_STEP_MS = 250;
+/** See `ViewSyncerService.#logBatchWait()`. */
+var BATCH_WAIT_LOG_INTERVAL_MS = 6e4;
+var BATCH_WAIT_LOG_MIN_MS = 1e3;
+/** The weight of the latest batch in a client group's smoothed batch time. */
+var BATCH_TIME_SMOOTHING = .3;
+/** How far, as a fraction, a client group's wait is varied at random. */
+var BATCH_WAIT_JITTER = .3;
 var TimeSliceTimer = class {
 	#total = 0;
 	#start = 0;
 	#lc;
-	constructor(lc) {
+	#yieldSlice;
+	/**
+	* @param yieldSlice How the timer yields the process, if not by taking
+	*        turns with the rest of the worker's work (see `yieldProcess()`).
+	*/
+	constructor(lc, yieldSlice) {
 		this.#lc = lc;
+		this.#yieldSlice = yieldSlice;
 	}
 	async start() {
-		await yieldProcess(this.#lc);
+		await (this.#yieldSlice ? this.#yieldSlice() : yieldProcess(this.#lc));
 		return this.startWithoutYielding();
 	}
 	startWithoutYielding() {
@@ -1520,7 +2237,7 @@ var TimeSliceTimer = class {
 	}
 	async yieldProcess(_msgForTesting) {
 		this.#stopLap();
-		await yieldProcess(this.#lc);
+		await (this.#yieldSlice ? this.#yieldSlice() : yieldProcess(this.#lc));
 		this.#startLap();
 	}
 	#startLap() {