@rocicorp/zero 1.11.0-canary.3 → 1.11.0-canary.30

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -1,4 +1,6 @@
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
+import { getOrInsertComputed } from "../../../../shared/src/map.js";
+import { newArray } from "../../../../shared/src/arrays.js";
 import { must } from "../../../../shared/src/must.js";
 import { h64 } from "../../../../shared/src/hash.js";
 import { Internal, InvalidConnectionRequest, InvalidConnectionRequestBaseCookie, Rehome } from "../../../../zero-protocol/src/error-kind-enum.js";
@@ -10,23 +12,27 @@ import { manualSpan, startAsyncSpan, startSpan } from "../../../../otel/src/span
 import { ZERO_VERSION_COLUMN_NAME } from "../replicator/schema/constants.js";
 import { transformAndHashQuery } from "../../auth/read-authorizer.js";
 import { randInt } from "../../../../shared/src/rand.js";
-import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateNativeHistogram, getOrCreateUpDownCounter } from "../../observability/metrics.js";
+import { LogThrottle } from "../../observability/log-throttle.js";
+import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateNativeHistogram, getOrCreateUpDownCounter, getOrCreateValueHistogram } from "../../observability/metrics.js";
 import { rowIDString } from "../../types/row-key.js";
 import "../replicator/schema/replication-state.js";
-import { parseSignature } from "./row-set-signature.js";
+import { planWarningMessage } from "./plan-warnings.js";
+import { queryShape } from "./query-shape.js";
 import { ResetPipelinesSignal } from "./snapshotter.js";
-import "./pipeline-driver.js";
 import { Subscription } from "../../types/subscription.js";
 import { CustomKeyMap } from "../../../../shared/src/custom-key-map.js";
 import { ttlClockAsNumber, ttlClockFromNumber } from "./ttl-clock.js";
 import { EMPTY_CVR_VERSION, cmpVersions, versionFromString, versionString, versionToCookie } from "./schema/types.js";
 import { ProtocolErrorWithLevel, getLogLevel, wrapWithProtocolError } from "../../types/error-with-level.js";
 import { isAuthErrorBody } from "../../auth/auth.js";
+import { AdvancementMetrics } from "./advancement-metrics.js";
 import { ClientHandler, startPoke } from "./client-handler.js";
 import { tracer } from "./tracer.js";
 import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
-import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, nextEvictionTime } from "./cvr.js";
+import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, classifyQueriesForHydration, nextEvictionTime } from "./cvr.js";
 import { E2EServingLagTracker } from "./e2e-serving-lag.js";
+import { HydrationBudget } from "./hydration-budget.js";
+import { HydrationCircuitBreaker } from "./hydration-circuit-breaker.js";
 import { handleInspect } from "./inspect-handler.js";
 import { QueryCoveringIndex } from "./query-covering.js";
 import { resolver } from "@rocicorp/resolver";
@@ -50,6 +56,7 @@ function shutdownBeforeInitializationError() {
 * use a timer to update the ttlClock every minute.
 */
 var TTL_CLOCK_INTERVAL = 6e4;
+var slowHydrationLogThrottle = new LogThrottle({ windowMs: 5 * 6e4 });
 var ViewSyncerService = class {
 	id;
 	createdAtMs = Date.now();
@@ -61,6 +68,7 @@ var ViewSyncerService = class {
 	#drainCoordinator;
 	#keepaliveMs;
 	#slowHydrateThreshold;
+	#hydrationCircuitBreaker;
 	#lastConnectTime = Date.now();
 	/**
 	* The TTL clock is used to determine the time at which queries are considered
@@ -88,6 +96,13 @@ var ViewSyncerService = class {
 	#lock = new Lock();
 	#cvrStore;
 	#stopped = resolver();
+	/**
+	* Set when {@link #cleanup} begins. Lock tasks that were in flight when the
+	* view-syncer was stopped may still complete after the timers have been
+	* cleared; this flag prevents them from scheduling new timers, which would
+	* otherwise outlive the service (and retain everything it references).
+	*/
+	#shuttingDown = false;
 	#initialized = resolver();
 	#cvr;
 	/**
@@ -102,7 +117,8 @@ var ViewSyncerService = class {
 	*   updates are processed via `#syncQueryPipelineSet`.
 	*
 	* Resets to `false` if `#advancePipelines` returns a `ResetPipelinesSignal`
-	* and pipelines must be reset and rehydrated.
+	* and pipelines must be reset and rehydrated, or if a reloaded CVR is ahead
+	* of the pipelines (see `#resetPipelinesIfBehindCVR`).
 	*/
 	#pipelinesHydrated = false;
 	#servedVersion = null;
@@ -110,6 +126,7 @@ var ViewSyncerService = class {
 	#expiredQueriesTimer = 0;
 	#authMaintenanceTimer = 0;
 	#setTimeout;
+	#now;
 	#customQueryTransformer;
 	#queryReplacements = /* @__PURE__ */ new Map();
 	#activeClients = getOrCreateUpDownCounter("sync", "active-clients", "Number of active sync clients");
@@ -127,6 +144,7 @@ var ViewSyncerService = class {
 		description: "Observations of sync.e2e_serving_lag that came out negative and were clamped to zero. Non-zero means the upstream database clock is running ahead of this pod by more than the entire pipeline latency, so sync.e2e_serving_lag is biased low and reads healthier than reality. See replication.upstream_clock_skew for the magnitude.",
 		unit: "{observation}"
 	});
+	#advancementMetrics = new AdvancementMetrics();
 	#transactionAdvanceTime = getOrCreateLatencyHistogram("sync", "advance-time", "Time to advance all queries for a given client group after applying a new transaction to the replica.");
 	#queryTransformations = getOrCreateCounter("sync", "query.transformations", "Number of query transformations performed");
 	#queryTransformationTime = getOrCreateLatencyHistogram("sync", "query.transformation-time", "Time to transform custom queries via API server.");
@@ -136,10 +154,64 @@ var ViewSyncerService = class {
 	#pipelineResets = getOrCreateCounter("sync", "pipeline-resets", "Number of pipeline resets");
 	#rowSetSignatureDrifts = getOrCreateCounter("sync", "query.row-set-signature-drifts", "Number of times re-hydration of an unchanged query produced a different row-set signature than what is stored in the CVR (forcing a configVersion bump and full re-execution). Expected to be near-zero in steady state; persistent non-zero values indicate non-deterministic query execution (e.g. Cap operator picking different N-row subsets).");
 	#sameHashRehydrationVersionBumps = getOrCreateCounter("sync", "query.same-hash-rehydrations-forced-bump", "Number of times query-set reconciliation forced a configVersion bump for already-gotten same-transformation-hash query rehydration because trackQueries would not otherwise bump. Expected to be near-zero; non-zero values indicate pipeline/CVR row-set drift reached query-set reconciliation.");
+	#hydrationBudgetExhaustions = getOrCreateCounter("sync", "hydration_budget_exhaustions", {
+		description: "Number of hydration passes that exhaust their budget.",
+		unit: "{pass}"
+	});
+	#hydrationBudgetEvictions = getOrCreateCounter("sync", "hydration_budget_evictions", {
+		description: "Number of inactive queries removed after hydration budget exhaustion.",
+		unit: "{query}"
+	});
+	#hydrationBudgetElapsed = getOrCreateValueHistogram("sync", "hydration_budget_elapsed", {
+		description: "Elapsed milliseconds when optional query hydration stopped.",
+		unit: "ms",
+		bucketBoundaries: [
+			1,
+			2,
+			5,
+			10,
+			20,
+			50,
+			100,
+			200,
+			500,
+			1e3,
+			5e3
+		]
+	});
+	#hydrationBudgetOvershoot = getOrCreateValueHistogram("sync", "hydration_budget_overshoot", {
+		description: "Milliseconds elapsed beyond the configured hydration budget.",
+		unit: "ms",
+		bucketBoundaries: [
+			0,
+			1,
+			2,
+			5,
+			10,
+			20,
+			50,
+			100,
+			200,
+			500,
+			1e3
+		]
+	});
+	#queryEvictions = getOrCreateCounter("sync", "query_evictions", {
+		description: "Number of queries evicted from the CVR ahead of their removal by the client, grouped by reason. Inactive queries are evicted by ttl and hydration-budget; hydration-timeout and hydration-circuit-breaker evict active queries as well.",
+		unit: "{query}"
+	});
+	#hydrationTimeouts = getOrCreateCounter("sync", "hydration_timeouts", {
+		description: "Number of query hydrations aborted for exceeding the query hydration timeout.",
+		unit: "{query}"
+	});
+	#hydrationCircuitBreakerRejections = getOrCreateCounter("sync", "hydration_circuit_breaker_rejections", {
+		description: "Number of queries rejected without hydration because their hydration circuit breaker was open.",
+		unit: "{query}"
+	});
 	#inspectorDelegate;
 	#config;
 	#runPriorityOp;
-	constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis)) {
+	constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis), now = performance.now.bind(performance)) {
 		this.#config = config;
 		this.id = clientGroupID;
 		this.connContextManager = connContextManager;
@@ -154,6 +226,8 @@ var ViewSyncerService = class {
 		this.#customQueryTransformer = customQueryTransformer;
 		this.#cvrStore = new CVRStore(lc, cvrDb, shard, taskID, clientGroupID, () => this.#stateChanges.cancel());
 		this.#setTimeout = setTimeoutFn;
+		this.#now = now;
+		this.#hydrationCircuitBreaker = new HydrationCircuitBreaker(config.viewSyncerQueryHydrationTimeoutMs ?? 0, void 0, now);
 		this.#runPriorityOp = runPriorityOp;
 		this.keepalive();
 	}
@@ -180,11 +254,13 @@ var ViewSyncerService = class {
 				this.#stateChanges.cancel();
 				return;
 			}
+			let reloaded = false;
 			if (!this.#cvr) {
 				this.#lc.debug?.("loading cvr");
 				this.#cvr = await this.#runPriorityOp(lc, "loading cvr", () => this.#cvrStore.load(lc, this.#lastConnectTime));
 				this.#ttlClock = this.#cvr.ttlClock;
 				this.#ttlClockBase = Date.now();
+				reloaded = true;
 			} else {
 				const now = Date.now();
 				this.#cvr = {
@@ -193,6 +269,10 @@ var ViewSyncerService = class {
 				};
 			}
 			try {
+				if (reloaded && this.#resetPipelinesIfBehindCVR(lc, this.#cvr)) {
+					const connCtx = this.connContextManager.getBackgroundConnectionContext();
+					if (connCtx) await this.#maybeHydratePipelines(lc, this.#cvr, connCtx);
+				}
 				await fn(lc, this.#cvr);
 			} catch (e) {
 				this.#cvr = void 0;
@@ -202,11 +282,47 @@ var ViewSyncerService = class {
 			}
 		});
 	}
+	/**
+	* The CVR is reloaded after an error clears the cached copy. By then another
+	* view-syncer may have taken over the client group and flushed the CVR at a
+	* version ahead of this instance's hydrated pipelines (e.g. its replica was
+	* further ahead). The pipelines can no longer be diffed against the CVR, so
+	* reset them and let `#maybeHydratePipelines` rehydrate once the replica has
+	* caught up to the CVR.
+	*
+	* A CVR behind the pipelines is expected (advancements that do not change
+	* the CVR are not flushed) and is handled by the normal update path.
+	*
+	* Returns whether the pipelines were reset.
+	*
+	* Must be called from within the #lock.
+	*/
+	#resetPipelinesIfBehindCVR(lc, cvr) {
+		if (!this.#pipelinesHydrated) return false;
+		const pipelineVersion = this.#pipelines.currentVersion();
+		if (pipelineVersion >= cvr.version.stateVersion) return false;
+		lc.info?.(`resetting pipelines: pipelines@${pipelineVersion} are behind reloaded cvr@${versionString(cvr.version)}`);
+		this.#pipelineResets.add(1, { reason: "behind-cvr" });
+		this.#pipelinesHydrated = false;
+		this.connContextManager.setSharedRetransformReady(false);
+		this.#pipelines.reset(must(cvr.clientSchema, "cvr.clientSchema missing after initialization"));
+		return true;
+	}
 	readyState() {
-		return Promise.race([this.#initialized.promise, this.#drainCoordinator.draining]);
+		return new Promise((resolve, reject) => {
+			const unsubscribe = this.#drainCoordinator.onDraining(() => resolve("draining"));
+			this.#initialized.promise.then((state) => {
+				unsubscribe();
+				resolve(state);
+			}, (err) => {
+				unsubscribe();
+				reject(err);
+			});
+		});
 	}
 	async run() {
 		try {
+			this.#scheduleShutdown(this.#keepaliveMs);
 			if (await this.readyState() === "draining") {
 				this.#lc.debug?.(`draining view-syncer ${this.id} before running`);
 				this.stop();
@@ -227,17 +343,28 @@ var ViewSyncerService = class {
 						lc.info?.(`resetting CVR: ${message}`);
 						throw new ClientNotFoundError(message);
 					}
+					let previousQueries;
 					if (this.#pipelinesHydrated) {
 						const result = await this.#advancePipelines(lc, cvr);
 						if (result === "success") return;
-						lc.info?.(`resetting pipelines: ${result.message}`);
+						lc.info?.(`resetting pipelines: ${result.message}${result.breakdown ? ` (${result.breakdown})` : ""}`);
 						this.#pipelineResets.add(1, { reason: result.reason });
+						switch (result.reason) {
+							case "advancement-timeout":
+							case "scalar-subquery":
+							case "truncation":
+							case "schema-change":
+								if (!Object.values(cvr.queries).some((q) => q.type === "client")) previousQueries = new Map(this.#pipelines.queries());
+								break;
+							case "permissions-change": break;
+							default: unreachable(result.reason);
+						}
 						this.#pipelines.reset(clientSchema);
 						this.#pipelinesHydrated = false;
 						this.connContextManager.setSharedRetransformReady(false);
 					}
 					const backgroundConnCtx = this.connContextManager.getBackgroundConnectionContext();
-					if (backgroundConnCtx) await this.#maybeHydratePipelines(lc, cvr, backgroundConnCtx);
+					if (backgroundConnCtx) await this.#maybeHydratePipelines(lc, cvr, backgroundConnCtx, previousQueries);
 					else lc.info?.("No validated background connection; deferring pipeline init");
 				});
 			}
@@ -275,17 +402,31 @@ var ViewSyncerService = class {
 	* - Syncs missing, errored, or drifted queries via `#syncQueryPipelineSet`.
 	* - Sets `#pipelinesHydrated = true` and enables shared retransformations.
 	*/
-	async #maybeHydratePipelines(lc, cvr, connCtx) {
+	async #maybeHydratePipelines(lc, cvr, connCtx, previousQueries) {
 		if (!this.#pipelines.initialized()) return;
-		const version = this.#pipelines.advanceWithoutDiff();
+		let version;
+		try {
+			version = this.#pipelines.advanceWithoutDiff();
+		} catch (e) {
+			if (!(e instanceof ResetPipelinesSignal)) throw e;
+			lc.info?.(`resetting pipelines: ${e.message}${e.breakdown ? ` (${e.breakdown})` : ""}`);
+			this.#pipelineResets.add(1, { reason: e.reason });
+			this.#pipelines.reset(must(cvr.clientSchema, "cvr.clientSchema missing after initialization"));
+			version = this.#pipelines.currentVersion();
+		}
 		const cvrVer = versionString(cvr.version);
 		if (version < cvr.version.stateVersion) {
 			lc.debug?.(`replica@${version} is behind cvr@${cvrVer}`);
 			return;
 		}
 		lc.info?.(`init pipelines@${version} (cvr@${cvrVer})`);
-		const driftedQueryIDs = await this.#hydrateUnchangedQueries(lc, cvr, connCtx);
-		await this.#syncQueryPipelineSet(lc, cvr, "missing", connCtx, driftedQueryIDs);
+		const hydrationBudget = new HydrationBudget(this.#config.viewSyncerHydrationBudgetMs ?? 0, this.#now);
+		const hydrationPassStats = {
+			activeHydratedQueries: 0,
+			inactiveHydratedQueries: 0
+		};
+		const driftedQueryIDs = await this.#hydrateUnchangedQueries(lc, cvr, connCtx, hydrationPassStats, hydrationBudget, previousQueries);
+		await this.#syncQueryPipelineSet(lc, cvr, "missing", connCtx, driftedQueryIDs, hydrationBudget, hydrationPassStats, previousQueries);
 		this.#pipelinesHydrated = true;
 		this.connContextManager.setSharedRetransformReady(true);
 	}
@@ -379,7 +520,14 @@ var ViewSyncerService = class {
 		return true;
 	}
 	#shutdownTimer = null;
+	#stopShutdownTimer() {
+		if (this.#shutdownTimer !== null) {
+			clearTimeout(this.#shutdownTimer);
+			this.#shutdownTimer = null;
+		}
+	}
 	#scheduleShutdown(delayMs = 0) {
+		if (this.#shuttingDown) return;
 		this.#shutdownTimer ??= this.#setTimeout(() => {
 			this.#shutdownTimer = null;
 			this.#runInLockWithCVR(() => {}).catch((e) => this.#stateChanges.fail(e));
@@ -426,6 +574,7 @@ var ViewSyncerService = class {
 	*/
 	#scheduleAuthMaintenance(lc) {
 		this.#stopAuthMaintenanceTimer();
+		if (this.#shuttingDown) return;
 		const plan = this.connContextManager.planMaintenance();
 		if (plan.earliestDeadlineAt === void 0) {
 			lc.debug?.("No auth maintenance wakeup scheduled");
@@ -531,17 +680,41 @@ var ViewSyncerService = class {
 		this.#ttlClockBase = now;
 		return ttlClock;
 	}
-	#flushUpdater(lc, updater) {
+	/**
+	* @param patchesPoked Whether patches computed against the updater's CVR
+	*     have already been poked to clients. If so, the CVR is checked to be
+	*     current even when the flush has nothing to write: another
+	*     view-syncer may have already committed identical rows at a newer
+	*     version, which would otherwise leave this one poking from a stale
+	*     CVR whose version it cannot advance.
+	*/
+	#flushUpdater(lc, updater, patchesPoked = false) {
 		return startAsyncSpan(tracer, "vs.#flushUpdater", () => this.#runPriorityOp(lc, "flushing cvr", async () => {
 			const now = Date.now();
 			const ttlClock = this.#getTTLClock(now);
-			const { cvr, flushed } = await updater.flush(lc, this.#lastConnectTime, now, ttlClock);
+			const { cvr, flushed } = await updater.flush(lc, this.#lastConnectTime, now, ttlClock, patchesPoked);
 			if (flushed) this.#startTTLClockInterval(lc);
 			return cvr;
 		}));
 	}
+	/**
+	* Flushes an updater whose changes have already been poked. If the flush
+	* fails (e.g. the CVR was concurrently modified), the poke is cancelled
+	* before the error propagates. The error may only fail the client that
+	* initiated the operation, and any other client left mid-poke would fail
+	* its next pokeStart.
+	*/
+	async #flushPoked(lc, updater, pokers) {
+		try {
+			return await this.#flushUpdater(lc, updater, pokers.patchesSent);
+		} catch (e) {
+			await pokers.cancel();
+			throw e;
+		}
+	}
 	#startTTLClockInterval(lc) {
 		this.#stopTTLClockInterval();
+		if (this.#shuttingDown) return;
 		this.#ttlClockInterval = this.#setTimeout(() => {
 			this.#updateTTLClockInCVRWithoutLock(lc);
 			this.#startTTLClockInterval(lc);
@@ -570,7 +743,7 @@ var ViewSyncerService = class {
 		if (cmpVersions(cvr.version, this.#cvr.version) < 0) {
 			const newCVR = this.#cvr;
 			await startAsyncSpan(tracer, "vs.#updateCVRConfig.pokeClients", async () => {
-				const pokers = startPoke(this.#getClients(cvr.version), newCVR.version);
+				const pokers = startPoke(lc, this.#getClients(cvr.version), newCVR.version);
 				for (const patch of patches) await pokers.addPatch(patch);
 				await pokers.end(newCVR.version);
 			});
@@ -672,6 +845,7 @@ var ViewSyncerService = class {
 	#scheduleExpireEviction(lc, cvr) {
 		const { ttlClock } = cvr;
 		this.#stopExpireTimer();
+		if (this.#shuttingDown) return;
 		const next = nextEvictionTime(cvr);
 		if (next === void 0) {
 			lc.debug?.("no inactive queries with ttl");
@@ -699,7 +873,7 @@ var ViewSyncerService = class {
 	*
 	* This must be called from within the #lock.
 	*/
-	async #hydrateUnchangedQueries(lc, cvr, connCtx) {
+	async #hydrateUnchangedQueries(lc, cvr, connCtx, hydrationPassStats, hydrationBudget, previousQueries) {
 		assert(this.#pipelines.initialized(), "pipelines must be initialized");
 		const dbVersion = this.#pipelines.currentVersion();
 		const cvrVersion = cvr.version;
@@ -708,25 +882,27 @@ var ViewSyncerService = class {
 			return /* @__PURE__ */ new Set();
 		}
 		const gotQueries = Object.entries(cvr.queries).filter(([_, state]) => state.transformationHash !== void 0);
+		const { required: requiredGotQueries } = classifyQueriesForHydration(gotQueries.map(([, query]) => query));
 		const customQueries = /* @__PURE__ */ new Map();
 		const otherQueries = [];
-		let inactivatedCount = 0;
-		for (const [, query] of gotQueries) {
-			if (query.type !== "internal" && Object.values(query.clientState).every(({ inactivatedAt }) => inactivatedAt !== void 0)) {
-				inactivatedCount++;
-				continue;
-			}
-			if (query.type === "custom") customQueries.set(query.id, query);
-			else otherQueries.push(query);
-		}
+		const inactivatedCount = gotQueries.length - requiredGotQueries.length;
 		const transformedQueries = [];
+		for (const query of requiredGotQueries) if (query.type === "custom") {
+			const previous = previousQueries?.get(query.id);
+			if (previous && previous.transformationHash === query.transformationHash) transformedQueries.push({
+				id: query.id,
+				transformationHash: previous.transformationHash,
+				transformedAst: previous.originalAst ?? previous.transformedAst
+			});
+			else customQueries.set(query.id, query);
+		} else otherQueries.push(query);
 		let customErrorCount = 0;
 		let customHashMismatchCount = 0;
 		let otherHashMismatchCount = 0;
 		if (customQueries.size > 0 && !this.#customQueryTransformer) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
 		const customQueryTransformer = this.#customQueryTransformer;
 		if (customQueryTransformer && customQueries.size > 0) {
-			const transformedCustomQueries = await this.#runPriorityOp(lc, "#hydrateUnchangedQueries transforming custom queries", () => customQueryTransformer.transform(connCtx, customQueries.values()));
+			const transformedCustomQueries = await hydrationBudget.excluding(() => this.#runPriorityOp(lc, "#hydrateUnchangedQueries transforming custom queries", () => customQueryTransformer.transform(connCtx, customQueries.values())));
 			if (transformedCustomQueries.kind === "success" && !transformedCustomQueries.cached) this.connContextManager.validateConnection(connCtx, connCtx.revision, transformedCustomQueries.validation);
 			if (Array.isArray(transformedCustomQueries.result)) for (const q of transformedCustomQueries.result) if ("error" in q) customErrorCount++;
 			else if (q.transformationHash !== customQueries.get(q.id)?.transformationHash) customHashMismatchCount++;
@@ -746,6 +922,10 @@ var ViewSyncerService = class {
 		for (const { id: queryID, transformationHash, transformedAst } of transformedQueries) {
 			const query = cvr.queries[queryID];
 			const queryName = query.type === "custom" ? query.name : void 0;
+			if (query.type !== "internal" && this.#hydrationCircuitBreaker.isOpen(transformationHash)) {
+				lc.info?.(`skipping hydration of ${queryID}: hydration circuit breaker open`);
+				continue;
+			}
 			const covered = queryCoveringIndex ? this.#findQueryCoverageShadowHit(queryCoveringIndex, queryID, transformationHash, transformedAst, queryName) : void 0;
 			totalHydratedQueries++;
 			if (covered) {
@@ -754,24 +934,46 @@ var ViewSyncerService = class {
 			}
 			const timer = new TimeSliceTimer(lc);
 			let count = 0;
+			let timedOut = false;
 			await startAsyncSpan(tracer, "vs.#hydrateUnchangedQueries.addQuery", async (span) => {
 				span.setAttribute("queryHash", queryID);
 				span.setAttribute("transformationHash", transformationHash);
 				span.setAttribute("table", transformedAst.table);
 				if (queryName !== void 0) span.setAttribute("queryName", queryName);
-				for (const change of this.#pipelines.addQuery(transformationHash, queryID, transformedAst, await timer.start(), queryName, "unchanged-query-rehydrate")) if (change === "yield") await timer.yieldProcess("yield in hydrateUnchangedQueries");
-				else count++;
+				for (const change of this.#pipelines.addQuery(transformationHash, queryID, transformedAst, await timer.start(), queryName, "unchanged-query-rehydrate")) if (change === "yield") {
+					if (query.type !== "internal" && this.#hydrationCircuitBreaker.exceeded(timer.totalElapsed())) {
+						timedOut = true;
+						await timer.yieldProcess("yield in hydrateUnchangedQueries");
+						break;
+					}
+					await timer.yieldProcess("yield in hydrateUnchangedQueries");
+				} else count++;
 			});
 			const elapsed = timer.totalElapsed();
+			if (timedOut) {
+				this.#recordHydrationTimeout(lc, {
+					id: queryID,
+					ast: transformedAst,
+					transformationHash,
+					name: queryName
+				}, elapsed, count);
+				continue;
+			}
+			hydrationPassStats.activeHydratedQueries++;
 			this.#hydrations.add(1);
 			this.#hydrationTime.recordMs(elapsed);
-			this.#addQueryMaterializationServerMetric(transformationHash, elapsed);
-			this.#inspectorDelegate.addQuery(transformationHash, transformedAst);
+			this.#addQueryMaterializationServerMetric(queryID, elapsed);
+			this.#inspectorDelegate.addQuery(queryID, transformedAst);
 			lc.debug?.(`hydrated ${count} rows for ${queryID} (${elapsed} ms)`);
+			if (elapsed > this.#slowHydrateThreshold) this.#logSlowHydration(lc, {
+				id: queryID,
+				ast: transformedAst,
+				transformationHash,
+				name: queryName
+			}, elapsed);
 			let drifted = false;
-			const storedSigHex = cvr.queries[queryID]?.rowSetSignature;
-			if (storedSigHex !== void 0 && storedSigHex !== null) {
-				const priorSig = parseSignature(storedSigHex);
+			const priorSig = this.#cvrStore.rowSetSignature(queryID);
+			if (priorSig !== void 0) {
 				const candidateSig = this.#pipelines.rowSetSignature(queryID) ?? 0n;
 				if (priorSig !== candidateSig) {
 					lc.warn?.(`rowSetSignature drift for query ${queryID}: prior=${priorSig.toString(16)} new=${candidateSig.toString(16)} (${count} rows). Removing from pipelines for full re-execution.`);
@@ -827,11 +1029,7 @@ var ViewSyncerService = class {
 			return;
 		}
 		const appErrorGroups = /* @__PURE__ */ new Map();
-		for (const err of errorOrErrors) for (const clientId of getAffectedClientIDs([err.id])) {
-			const group = appErrorGroups.get(clientId) ?? [];
-			group.push(err);
-			appErrorGroups.set(clientId, group);
-		}
+		for (const err of errorOrErrors) for (const clientId of getAffectedClientIDs([err.id])) getOrInsertComputed(appErrorGroups, clientId, newArray).push(err);
 		for (const [clientId, errors] of appErrorGroups) this.#clients.get(clientId)?.sendQueryTransformApplicationErrors(errors);
 	}
 	#addQueryMaterializationServerMetric(queryID, elapsed) {
@@ -867,7 +1065,10 @@ var ViewSyncerService = class {
 	*
 	* This must be called from within the #lock.
 	*/
-	#syncQueryPipelineSet(lc, cvr, customQueryTransformMode, connCtx, driftedQueryIDs = /* @__PURE__ */ new Set()) {
+	#syncQueryPipelineSet(lc, cvr, customQueryTransformMode, connCtx, driftedQueryIDs = /* @__PURE__ */ new Set(), hydrationBudget = new HydrationBudget(this.#config.viewSyncerHydrationBudgetMs ?? 0, this.#now), hydrationPassStats = {
+		activeHydratedQueries: 0,
+		inactiveHydratedQueries: 0
+	}, previousQueries) {
 		return startAsyncSpan(tracer, "vs.#syncQueryPipelineSet", async (span) => {
 			const start = performance.now();
 			span.setAttribute("clientGroupID", this.id);
@@ -876,34 +1077,42 @@ var ViewSyncerService = class {
 			const now = Date.now();
 			const ttlClock = this.#getTTLClock(now);
 			const cvrQueryEntires = Object.entries(cvr.queries);
-			const customQueries = /* @__PURE__ */ new Map();
-			const otherQueries = [];
 			const transformedQueries = [];
-			for (const [id, query] of cvrQueryEntires) if (query.type === "custom") {
-				assert(id === query.id, "custom query id mismatch");
-				customQueries.set(id, query);
-			} else otherQueries.push({
-				id,
-				query
-			});
-			for (const { id, query: origQuery } of otherQueries) {
-				assert(id === origQuery.id, "query id mismatch");
-				const transformed = transformAndHashQuery(lc, origQuery.id, origQuery.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, connCtx.auth?.type === "jwt" ? connCtx.auth : void 0, origQuery.type === "internal");
-				transformedQueries.push({
-					id,
-					origQuery,
-					transformed
-				});
-			}
-			if (customQueries.size > 0 && !this.#customQueryTransformer) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
-			let erroredQueryIDs;
-			const customQueriesToTransform = customQueryTransformMode === "all" ? [...customQueries.values()] : customQueryTransformMode && [...customQueries.values()].filter((q) => !this.#pipelines.queries().has(q.id));
+			const naturallyExpiredQueryIDs = new Set(cvrQueryEntires.map(([, query]) => query).filter((query) => expired(ttlClock, query)).map((query) => query.id));
+			if (naturallyExpiredQueryIDs.size > 0) this.#queryEvictions.add(naturallyExpiredQueryIDs.size, { reason: "ttl" });
+			const hydrationCandidates = cvrQueryEntires.map(([, query]) => query).filter((query) => !naturallyExpiredQueryIDs.has(query.id));
+			const { required, optional } = classifyQueriesForHydration(hydrationCandidates);
+			const requiredQueryIDs = new Set(required.map((query) => query.id));
+			const optionalToHydrate = hydrationBudget.limitMs === 0 ? optional : optional.filter((query) => query.patchVersion !== void 0);
+			if (!this.#customQueryTransformer && hydrationCandidates.some((query) => query.type === "custom")) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
 			const customQueryTransformer = this.#customQueryTransformer;
-			if (customQueryTransformer && customQueriesToTransform.length > 0) {
+			const erroredQueryIDs = [];
+			let transformedCustomQueryCount = 0;
+			const transformOtherQueries = (queries) => {
+				for (const origQuery of queries) {
+					if (origQuery.type === "custom") continue;
+					const transformed = transformAndHashQuery(lc, origQuery.id, origQuery.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, connCtx.auth?.type === "jwt" ? connCtx.auth : void 0, origQuery.type === "internal");
+					transformedQueries.push({
+						id: origQuery.id,
+						origQuery,
+						transformed
+					});
+				}
+			};
+			const shouldTransformCustomQuery = (query) => {
+				if (customQueryTransformMode === "all") return true;
+				if (this.#pipelines.queries().has(query.id)) return false;
+				const previous = previousQueries?.get(query.id);
+				if (previous && previous.transformationHash === query.transformationHash) return false;
+				return true;
+			};
+			const transformCustomQueries = async (queries) => {
+				if (!customQueryTransformer || queries.length === 0) return;
+				transformedCustomQueryCount += queries.length;
 				const transformStart = performance.now();
 				let transformedCustomQueries;
 				try {
-					transformedCustomQueries = await this.#runPriorityOp(lc, "#syncQueryPipelineSet transforming custom queries", () => customQueryTransformer.transform(connCtx, customQueriesToTransform));
+					transformedCustomQueries = await hydrationBudget.excluding(() => this.#runPriorityOp(lc, "#syncQueryPipelineSet transforming custom queries", () => customQueryTransformer.transform(connCtx, queries)));
 					if (transformedCustomQueries.kind === "failed") throw new ProtocolErrorWithLevel(transformedCustomQueries.result, "warn");
 					else {
 						if (!transformedCustomQueries.cached) this.connContextManager.validateConnection(connCtx, connCtx.revision, transformedCustomQueries.validation);
@@ -917,8 +1126,9 @@ var ViewSyncerService = class {
 					this.#queryTransformationTime.recordMs(transformDuration);
 				}
 				const successfullyTransformedCustomQueries = /* @__PURE__ */ new Map();
-				erroredQueryIDs = this.#processTransformedCustomQueries(lc, transformedCustomQueries.result, (q) => {
-					const origQuery = customQueries.get(q.id);
+				const customQueryMap = new Map(queries.map((query) => [query.id, query]));
+				erroredQueryIDs.push(...this.#processTransformedCustomQueries(lc, transformedCustomQueries.result, (q) => {
+					const origQuery = customQueryMap.get(q.id);
 					if (origQuery) {
 						successfullyTransformedCustomQueries.set(q.id, q);
 						transformedQueries.push({
@@ -927,7 +1137,7 @@ var ViewSyncerService = class {
 							transformed: q
 						});
 					}
-				}, customQueries);
+				}, customQueryMap));
 				for (const [queryID, newTransform] of successfullyTransformedCustomQueries) {
 					const existingTransformHash = cvr.queries[queryID]?.transformationHash;
 					if (existingTransformHash) {
@@ -940,21 +1150,55 @@ var ViewSyncerService = class {
 						} else this.#queryTransformationNoOps.add(1);
 					}
 				}
+			};
+			const queriesToTransform = [...required, ...optionalToHydrate];
+			transformOtherQueries(queriesToTransform);
+			if (customQueryTransformMode === "missing" && previousQueries) {
+				for (const query of queriesToTransform) if (query.type === "custom" && !this.#pipelines.queries().has(query.id)) {
+					const previous = previousQueries.get(query.id);
+					if (previous && previous.transformationHash === query.transformationHash) {
+						this.#queryTransformationNoOps.add(1);
+						transformedQueries.push({
+							id: query.id,
+							origQuery: query,
+							transformed: {
+								id: query.id,
+								transformationHash: previous.transformationHash,
+								transformedAst: previous.originalAst ?? previous.transformedAst
+							}
+						});
+					}
+				}
 			}
-			const removeQueriesQueryIds = new Set([...Object.values(cvr.queries).filter((q) => expired(ttlClock, q)).map((q) => q.id), ...erroredQueryIDs || []]);
+			await transformCustomQueries(queriesToTransform.filter((query) => query.type === "custom" && shouldTransformCustomQuery(query)));
+			const circuitBrokenQueries = transformedQueries.filter(({ id, origQuery, transformed }) => origQuery.type !== "internal" && this.#pipelines.queries().get(id)?.transformationHash !== transformed.transformationHash && this.#hydrationCircuitBreaker.isOpen(transformed.transformationHash)).map(({ id, origQuery, transformed }) => ({
+				id,
+				transformationHash: transformed.transformationHash,
+				name: origQuery.type === "custom" ? origQuery.name : void 0
+			}));
+			if (circuitBrokenQueries.length > 0) this.#rejectCircuitBrokenQueries(lc, cvr, circuitBrokenQueries);
+			const removeQueriesQueryIds = new Set([
+				...naturallyExpiredQueryIDs,
+				...erroredQueryIDs,
+				...circuitBrokenQueries.map(({ id }) => id)
+			]);
 			const addQueries = transformedQueries.map(({ id, origQuery, transformed }) => ({
 				id,
 				ast: transformed.transformedAst,
 				transformationHash: transformed.transformationHash,
 				name: origQuery.type === "custom" ? origQuery.name : void 0
 			})).filter((q) => !removeQueriesQueryIds.has(q.id) && this.#pipelines.queries().get(q.id)?.transformationHash !== q.transformationHash);
-			lc.info?.(`syncQueryPipelineSet: ${cvrQueryEntires.length} CVR queries, ${customQueriesToTransform.length} custom re-transformed, ${erroredQueryIDs?.length ?? 0} errored, ${removeQueriesQueryIds.size} to remove, ${addQueries.length} to add`);
+			const hydrationOrder = new Map([...required, ...optional].map((query, index) => [query.id, index]));
+			addQueries.sort((a, b) => must(hydrationOrder.get(a.id)) - must(hydrationOrder.get(b.id)));
+			const requiredAddQueries = addQueries.filter((query) => requiredQueryIDs.has(query.id));
+			const optionalAddQueries = addQueries.filter((query) => !requiredQueryIDs.has(query.id));
+			lc.info?.(`syncQueryPipelineSet: ${cvrQueryEntires.length} CVR queries, ${transformedCustomQueryCount} custom re-transformed, ${erroredQueryIDs.length} errored, ${circuitBrokenQueries.length} circuit-broken, ${removeQueriesQueryIds.size} to remove, ${addQueries.length} to add`);
 			for (const q of addQueries) {
 				const orig = cvr.queries[q.id];
 				lc.debug?.("ViewSyncer adding query", q.ast, "transformed from", orig.type === "custom" ? orig.name : orig.ast);
 			}
 			if (addQueries.length > 0 || removeQueriesQueryIds.size > 0) {
-				await this.#addAndRemoveQueries(lc, cvr, addQueries, Array.from(removeQueriesQueryIds, (id) => ({ id })), driftedQueryIDs);
+				await this.#addAndRemoveQueries(lc, cvr, requiredAddQueries, optionalAddQueries, Array.from(removeQueriesQueryIds, (id) => ({ id })), hydrationBudget, hydrationPassStats, driftedQueryIDs);
 				if (addQueries.length > 0) this.#viewSyncerHydration.recordMs(performance.now() - start);
 			} else await this.#catchupClients(lc, cvr);
 		});
@@ -987,14 +1231,134 @@ var ViewSyncerService = class {
 		record.count++;
 		if (record.count >= THRASH_THRESHOLD) this.#lc.warn?.(`Query thrashing detected for query ${queryID}. ${record.count} replacements in 60s. This may indicate clients with different auth contexts connecting to the same client group.`);
 	}
-	#addAndRemoveQueries(lc, cvr, addQueries, removeQueries, driftedQueryIDs = /* @__PURE__ */ new Set()) {
+	/**
+	* Records that hydrating `query` was aborted for exceeding the query
+	* hydration timeout, and opens its circuit breaker.
+	*/
+	#recordHydrationTimeout(lc, query, elapsedMs, rowCount) {
+		this.#hydrationCircuitBreaker.trip(query.transformationHash);
+		this.#hydrationTimeouts.add(1);
+		if (!lc.warn) return;
+		const shape = queryShape(query.ast);
+		lc.warn("Query hydration aborted for exceeding the hydration timeout", {
+			clientGroupID: this.id,
+			queryHash: query.id,
+			transformationHash: query.transformationHash,
+			...query.name !== void 0 && { queryName: query.name },
+			queryShape: shape.hash,
+			hydrationTimeoutMs: this.#hydrationCircuitBreaker.timeoutMs,
+			hydrationElapsedMs: elapsedMs,
+			...rowCount !== void 0 && { hydrationRowCount: rowCount },
+			circuitBreakerOpenMs: this.#hydrationCircuitBreaker.openMs,
+			zql: shape.zql
+		});
+	}
+	/**
+	* Logs a hydration that exceeded the slow hydration threshold.
+	*
+	* The query is logged as its {@link queryShape}, with literal values
+	* redacted, and each shape is logged at most once per
+	* {@link SLOW_HYDRATION_LOG_WINDOW_MS}. The next log of a shape reports how
+	* many slow hydrations of it were suppressed in the meantime.
+	*/
+	#logSlowHydration(lc, query, elapsedMs) {
+		if (!lc.warn) return;
+		const stats = this.#pipelines.hydrationStats(query.id);
+		const shape = stats?.shape ?? queryShape(query.ast);
+		const suppressed = slowHydrationLogThrottle.admit(`${query.name ?? ""}:${shape.hash}`);
+		if (suppressed === void 0) return;
+		lc.warn("Slow query materialization", {
+			zeroEvent: "query-slow-hydration",
+			clientGroupID: this.id,
+			queryHash: query.id,
+			transformationHash: query.transformationHash,
+			...query.name !== void 0 && { queryName: query.name },
+			queryShape: shape.hash,
+			hydrationTimeMs: elapsedMs,
+			...stats && {
+				hydrationRowCount: stats.rowCount,
+				hydrationRowsRead: stats.rowsRead
+			},
+			...stats !== void 0 && stats.planWarnings.length > 0 && { planWarnings: stats.planWarnings.map(planWarningMessage) },
+			...suppressed > 0 && { suppressedSinceLastLog: suppressed },
+			zql: shape.zql
+		});
+	}
+	/** Records that `query` was rejected without hydration by its open breaker. */
+	#recordCircuitBreakerRejection(lc, query) {
+		this.#hydrationCircuitBreakerRejections.add(1);
+		lc.warn?.("Query rejected by its open hydration circuit breaker", {
+			clientGroupID: this.id,
+			queryHash: query.id,
+			transformationHash: query.transformationHash,
+			...query.name !== void 0 && { queryName: query.name },
+			hydrationTimeoutMs: this.#hydrationCircuitBreaker.timeoutMs,
+			circuitBreakerOpenMs: this.#hydrationCircuitBreaker.openMs
+		});
+	}
+	/**
+	* Handles queries whose hydration circuit breaker is open: they are counted,
+	* logged, and errored to the affected clients. The caller removes them from
+	* the CVR.
+	*/
+	#rejectCircuitBrokenQueries(lc, cvr, queries) {
+		for (const query of queries) this.#recordCircuitBreakerRejection(lc, query);
+		this.#queryEvictions.add(queries.length, { reason: "hydration-circuit-breaker" });
+		this.#sendHydrationTimeoutErrors(cvr, queries);
+	}
+	/**
+	* Sends a per-query error to every client that desires one of `queries`.
+	* The error goes out as a `transformError` application error, which every
+	* client understands as "this query errored" without affecting the
+	* connection or the client's other queries.
+	*/
+	#sendHydrationTimeoutErrors(cvr, queries) {
+		const timeoutMs = this.#hydrationCircuitBreaker.timeoutMs;
+		const errorsByClient = /* @__PURE__ */ new Map();
+		for (const { id, name } of queries) {
+			const query = cvr.queries[id];
+			if (query === void 0 || query.type === "internal") continue;
+			const error = {
+				error: "app",
+				id,
+				name: name ?? (query.type === "custom" ? query.name : "legacy"),
+				message: `Query hydration exceeded the ${timeoutMs}ms limit (ZERO_VIEW_SYNCER_QUERY_HYDRATION_TIMEOUT_MS) and was aborted`,
+				details: {
+					kind: "HydrationTimeout",
+					timeoutMs
+				}
+			};
+			for (const clientID of Object.keys(query.clientState)) getOrInsertComputed(errorsByClient, clientID, newArray).push(error);
+		}
+		for (const [clientID, errors] of errorsByClient) this.#clients.get(clientID)?.sendQueryTransformApplicationErrors(errors);
+	}
+	#recordHydrationBudgetExhaustion(lc, hydrationBudget, activeHydratedQueries, inactiveHydratedQueries, inactiveEvictedQueryIDs) {
+		const elapsedMs = hydrationBudget.exhaustedAtMs;
+		if (elapsedMs === void 0) return;
+		this.#hydrationBudgetExhaustions.add(1);
+		this.#hydrationBudgetEvictions.add(inactiveEvictedQueryIDs.length);
+		this.#hydrationBudgetElapsed.record(elapsedMs);
+		this.#hydrationBudgetOvershoot.record(Math.max(0, elapsedMs - hydrationBudget.limitMs));
+		if (inactiveEvictedQueryIDs.length > 0) this.#queryEvictions.add(inactiveEvictedQueryIDs.length, { reason: "hydration-budget" });
+		lc.info?.("view-syncer hydration budget exhausted", {
+			clientGroupID: this.id,
+			hydrationBudgetMs: hydrationBudget.limitMs,
+			hydrationElapsedMs: elapsedMs,
+			activeHydratedQueries,
+			inactiveHydratedQueries,
+			inactiveEvictedQueries: inactiveEvictedQueryIDs.length,
+			firstEvictedQueryHash: inactiveEvictedQueryIDs[0] ?? null
+		});
+	}
+	#addAndRemoveQueries(lc, cvr, requiredQueries, optionalQueries, removeQueries, hydrationBudget, hydrationPassStats, driftedQueryIDs = /* @__PURE__ */ new Set()) {
 		return startAsyncSpan(tracer, "vs.#addAndRemoveQueries", async () => {
+			const addQueries = [...requiredQueries, ...optionalQueries];
 			assert(addQueries.length > 0 || removeQueries.length > 0, "Must have queries to add or remove");
 			const start = performance.now();
 			const stateVersion = this.#pipelines.currentVersion();
 			lc = lc.withContext("stateVersion", stateVersion);
 			lc.info?.(`hydrating ${addQueries.length} queries`);
-			const updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, stateVersion, this.#pipelines.replicaVersion, (queryID) => this.#pipelines.rowSetSignature(queryID));
+			const updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, stateVersion, this.#pipelines.replicaVersion, this.#pipelines);
 			const sameHashRehydratedQueryIDs = addQueries.filter((q) => cvr.queries[q.id]?.transformationHash === q.transformationHash).map((q) => q.id);
 			const trackQueriesWillBumpVersion = stateVersion > cvr.version.stateVersion || removeQueries.length > 0 || addQueries.some((q) => cvr.queries[q.id]?.transformationHash !== q.transformationHash);
 			if (sameHashRehydratedQueryIDs.length > 0 && !trackQueriesWillBumpVersion) {
@@ -1005,7 +1369,9 @@ var ViewSyncerService = class {
 				updater.ensureNewVersion();
 			}
 			const { queryPatches, newVersion } = updater.trackQueries(lc, addQueries, removeQueries);
-			const pokers = startPoke(this.#getClients(), newVersion);
+			if (hydrationBudget.limitMs !== 0 && optionalQueries.length > 0) assert(cmpVersions(cvr.version, newVersion) < 0, "Optional hydration requires a final poke version before row processing so a removal is actually poked");
+			const clients = this.#getClients();
+			const pokers = startPoke(lc, clients, newVersion);
 			for (const patch of queryPatches) await pokers.addPatch(patch);
 			for (const q of removeQueries) {
 				this.#pipelines.removeQuery(q.id);
@@ -1021,22 +1387,54 @@ var ViewSyncerService = class {
 			let totalHydratedQueries = 0;
 			let coveredHydratedQueries = 0;
 			let firstCoveredQuery;
+			const hydratedQueryIDs = [];
+			const budgetEvictedQueryIDs = [];
+			const timedOutQueries = [];
+			const rejectedQueries = [];
+			const circuitBreaker = this.#hydrationCircuitBreaker;
 			const self = this;
 			await yieldProcess(lc);
-			function* generateRowChanges(slowHydrateThreshold) {
-				for (const q of addQueries) {
+			function* hydrateQueries(queries, optional, slowHydrateThreshold) {
+				for (let i = 0; i < queries.length; i++) {
+					if (optional && hydrationBudget.exhausted()) {
+						budgetEvictedQueryIDs.push(...queries.slice(i).map((query) => query.id));
+						return;
+					}
+					const q = must(queries[i]);
 					let queryLC = lc.withContext("hash", q.id).withContext("queryHash", q.id).withContext("transformationHash", q.transformationHash);
 					if (q.name !== void 0) queryLC = queryLC.withContext("queryName", q.name);
 					queryLC.debug?.(`adding pipeline for query`, q.ast);
+					const breakable = cvr.queries[q.id]?.type !== "internal";
+					if (breakable && circuitBreaker.isOpen(q.transformationHash)) {
+						rejectedQueries.push(q);
+						self.#recordCircuitBreakerRejection(queryLC, q);
+						continue;
+					}
 					const covered = queryCoveringIndex ? self.#findQueryCoverageShadowHit(queryCoveringIndex, q.id, q.transformationHash, q.ast, q.name) : void 0;
 					totalHydratedQueries++;
 					if (covered) {
 						coveredHydratedQueries++;
 						firstCoveredQuery ??= covered;
 					}
-					yield* pipelines.addQuery(q.transformationHash, q.id, q.ast, timer.startWithoutYielding(), q.name, "query-set-sync");
+					let timedOut = false;
+					for (const change of pipelines.addQuery(q.transformationHash, q.id, q.ast, timer.startWithoutYielding(), q.name, "query-set-sync")) {
+						if (change === "yield" && breakable && circuitBreaker.exceeded(timer.totalElapsed())) {
+							timedOut = true;
+							yield change;
+							break;
+						}
+						yield change;
+					}
 					const elapsed = timer.stop();
 					totalProcessTime += elapsed;
+					if (timedOut) {
+						timedOutQueries.push(q);
+						self.#recordHydrationTimeout(queryLC, q, elapsed);
+						continue;
+					}
+					hydratedQueryIDs.push(q.id);
+					if (optional) hydrationPassStats.inactiveHydratedQueries++;
+					else hydrationPassStats.activeHydratedQueries++;
 					self.#addQueryMaterializationServerMetric(q.id, elapsed);
 					self.#inspectorDelegate.addQuery(q.id, q.ast);
 					queryCoveringIndex?.add(q.id, {
@@ -1044,24 +1442,49 @@ var ViewSyncerService = class {
 						transformationHash: q.transformationHash,
 						...q.name !== void 0 && { queryName: q.name }
 					});
-					if (elapsed > slowHydrateThreshold) queryLC.warn?.("Slow query materialization", elapsed, q.ast);
+					if (elapsed > slowHydrateThreshold) self.#logSlowHydration(queryLC, q, elapsed);
 					manualSpan(tracer, "vs.addAndConsumeQuery", elapsed, {
 						hash: q.id,
 						transformationHash: q.transformationHash,
 						...q.name !== void 0 && { name: q.name }
 					});
+					hydrations.add(1);
+					hydrationTime.recordMs(elapsed);
+				}
+			}
+			function* generateRowChanges(slowHydrateThreshold) {
+				yield* hydrateQueries(requiredQueries, false, slowHydrateThreshold);
+				yield* hydrateQueries(optionalQueries, true, slowHydrateThreshold);
+			}
+			await this.#processChanges(lc, timer, generateRowChanges(this.#slowHydrateThreshold), (rows) => updater.received(lc, rows), pokers);
+			const abortedQueries = [...timedOutQueries, ...rejectedQueries];
+			if (abortedQueries.length > 0) {
+				const abortedQueryIDs = abortedQueries.map(({ id }) => id);
+				for (const patch of await updater.abortExecutedQueries(lc, abortedQueryIDs)) await pokers.addPatch(patch);
+				for (const queryID of abortedQueryIDs) {
+					this.#pipelines.removeQuery(queryID);
+					this.#inspectorDelegate.removeQuery(queryID);
+					this.#queryReplacements.delete(queryID);
 				}
-				hydrations.add(1);
-				hydrationTime.recordMs(totalProcessTime);
+				if (timedOutQueries.length > 0) this.#queryEvictions.add(timedOutQueries.length, { reason: "hydration-timeout" });
+				if (rejectedQueries.length > 0) this.#queryEvictions.add(rejectedQueries.length, { reason: "hydration-circuit-breaker" });
+				this.#sendHydrationTimeoutErrors(cvr, abortedQueries);
 			}
-			await this.#processChanges(lc, timer, generateRowChanges(this.#slowHydrateThreshold), updater, pokers);
+			for (const patch of updater.removeTrackedQueries(budgetEvictedQueryIDs)) await pokers.addPatch(patch);
+			for (const queryID of budgetEvictedQueryIDs) {
+				this.#pipelines.removeQuery(queryID);
+				this.#inspectorDelegate.removeQuery(queryID);
+				this.#queryReplacements.delete(queryID);
+			}
+			this.#recordHydrationBudgetExhaustion(lc, hydrationBudget, hydrationPassStats.activeHydratedQueries, hydrationPassStats.inactiveHydratedQueries, budgetEvictedQueryIDs);
 			this.#logQueryCoverageShadowSummary(lc, "add", totalHydratedQueries, coveredHydratedQueries, firstCoveredQuery);
 			await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.deleteUnreferencedRows", async () => {
 				for (const patch of await updater.deleteUnreferencedRows(lc)) await pokers.addPatch(patch);
 			});
-			this.#cvr = await this.#flushUpdater(lc, updater);
+			this.#cvr = await this.#flushPoked(lc, updater, pokers);
+			if (budgetEvictedQueryIDs.length > 0) this.#scheduleExpireEviction(lc, this.#cvr);
 			const finalVersion = this.#cvr.version;
-			await this.#catchupClients(lc, cvr, finalVersion, addQueries.map((q) => q.id), pokers);
+			await this.#catchupClients(lc, cvr, finalVersion, hydratedQueryIDs, pokers);
 			await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.pokeEnd", () => pokers.end(finalVersion));
 			this.#markVersionServed(stateVersion);
 			const wallTime = performance.now() - start;
@@ -1090,7 +1513,7 @@ var ViewSyncerService = class {
 		return startAsyncSpan(tracer, "vs.#catchupClients", async (span) => {
 			current ??= cvr.version;
 			const clients = this.#getClients();
-			const pokers = usePokers ?? startPoke(clients, cvr.version);
+			const pokers = usePokers ?? startPoke(lc, clients, cvr.version);
 			span.setAttribute("numClients", clients.length);
 			const catchupFrom = clients.map((c) => c.version()).reduce((a, b) => cmpVersions(a, b) < 0 ? a : b, cvr.version);
 			const rowPatches = this.#cvrStore.catchupRowPatches(lc, catchupFrom, cvr, current, excludeQueryHashes);
@@ -1137,7 +1560,7 @@ var ViewSyncerService = class {
 			}
 		});
 	}
-	#processChanges(lc, timer, changes, updater, pokers) {
+	#processChanges(lc, timer, changes, onBatch, pokers) {
 		return startAsyncSpan(tracer, "vs.#processChanges", async () => {
 			const start = performance.now();
 			const rows = new CustomKeyMap(rowIDString);
@@ -1146,7 +1569,7 @@ var ViewSyncerService = class {
 				const wallElapsed = performance.now() - start;
 				total += rows.size;
 				lc.debug?.(`processing ${rows.size} (of ${total}) rows (${wallElapsed} ms)`);
-				const patches = await updater.received(lc, rows);
+				const patches = await onBatch(rows);
 				await startAsyncSpan(tracer, "processBatch.flushToClient", async (span) => {
 					span.setAttribute("patches", patches.length);
 					for (const patch of patches) await pokers.addPatch(patch);
@@ -1212,29 +1635,60 @@ var ViewSyncerService = class {
 			assert(this.#pipelines.initialized(), "pipelines must be initialized (advancePipelines");
 			const start = performance.now();
 			const timer = new TimeSliceTimer(lc);
-			const { version, numChanges, changes } = this.#pipelines.advance(timer);
-			lc = lc.withContext("newVersion", version);
-			const updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, version, this.#pipelines.replicaVersion, (queryID) => this.#pipelines.rowSetSignature(queryID));
-			const pokers = startPoke(this.#getClients(cvr.version), updater.updatedVersion());
-			lc.debug?.(`applying ${numChanges} to advance to ${version}`);
+			let outcome = { outcome: "error" };
+			let setupTimeMs;
 			try {
-				await this.#processChanges(lc, await timer.start(), changes, updater, pokers);
-			} catch (e) {
-				if (e instanceof ResetPipelinesSignal) {
-					await pokers.cancel();
-					return e;
+				let pokers;
+				let updater;
+				let version;
+				let numChanges = 0;
+				try {
+					const advancement = this.#pipelines.advance(timer);
+					version = advancement.version;
+					numChanges = advancement.numChanges;
+					lc = lc.withContext("newVersion", version);
+					const targetVersion = cmpVersions(cvr.version, { stateVersion: version }) < 0 ? { stateVersion: version } : cvr.version;
+					pokers = startPoke(lc, this.#getClients(cvr.version), targetVersion);
+					lc.debug?.(`applying ${numChanges} to advance to ${version}`);
+					setupTimeMs = performance.now() - start;
+					await this.#processChanges(lc, await timer.start(), advancement.changes, (rows) => {
+						updater ??= new CVRQueryDrivenUpdater(this.#cvrStore, cvr, version, this.#pipelines.replicaVersion, this.#pipelines);
+						return updater.received(lc, rows);
+					}, pokers);
+				} catch (e) {
+					setupTimeMs ??= performance.now() - start;
+					if (e instanceof ResetPipelinesSignal) {
+						await pokers?.cancel();
+						this.#cvrStore.discardPending();
+						outcome = {
+							outcome: "reset",
+							reason: e.reason
+						};
+						return e;
+					}
+					throw e;
 				}
-				throw e;
+				assert(pokers && version !== void 0, "advancement state missing");
+				if (updater) this.#cvr = await this.#flushPoked(lc, updater, pokers);
+				const finalVersion = (this.#cvr ?? cvr).version;
+				await startAsyncSpan(tracer, "vs.#advancePipelines.pokeEnd", () => pokers.end(finalVersion));
+				this.#markVersionServed(version);
+				const wallTime = performance.now() - start;
+				const totalProcessTime = timer.totalElapsed();
+				lc.debug?.(`finished processing advancement of ${numChanges} changes ((process: ${totalProcessTime} ms, wall: ${wallTime} ms))`);
+				this.#transactionAdvanceTime.recordMs(totalProcessTime);
+				outcome = { outcome: "success" };
+				return "success";
+			} finally {
+				const setupMs = must(setupTimeMs);
+				const processingTimeMs = setupMs + timer.totalElapsed();
+				const wallTimeMs = performance.now() - start;
+				this.#advancementMetrics.record({
+					processingTimeMs,
+					setupTimeMs: setupMs,
+					wallTimeMs
+				}, outcome);
 			}
-			this.#cvr = await this.#flushUpdater(lc, updater);
-			const finalVersion = this.#cvr.version;
-			await startAsyncSpan(tracer, "vs.#advancePipelines.pokeEnd", () => pokers.end(finalVersion));
-			this.#markVersionServed(version);
-			const wallTime = performance.now() - start;
-			const totalProcessTime = timer.totalElapsed();
-			lc.debug?.(`finished processing advancement of ${numChanges} changes ((process: ${totalProcessTime} ms, wall: ${wallTime} ms))`);
-			this.#transactionAdvanceTime.recordMs(totalProcessTime);
-			return "success";
 		});
 	}
 	async inspect(selector, msg) {
@@ -1246,7 +1700,7 @@ var ViewSyncerService = class {
 			clientID,
 			wsID: client.wsID
 		});
-		return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this.#cvrStore, this.#config, connCtx);
+		return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this, this.#cvrStore, this.#config, connCtx);
 	};
 	async #runBackgroundRetransform(lc) {
 		const attemptRetransform = async (connCtx) => {
@@ -1333,15 +1787,18 @@ var ViewSyncerService = class {
 		return this.#stopped.promise;
 	}
 	async #cleanup(err) {
+		this.#shuttingDown = true;
 		this.connContextManager.setSharedRetransformReady(false);
 		this.#stopTTLClockInterval();
 		this.#stopExpireTimer();
 		this.#stopAuthMaintenanceTimer();
+		this.#stopShutdownTimer();
 		this.#customQueryTransformer?.destroy();
 		for (const client of this.#clients.values()) if (err) client.fail(err);
 		else client.close(`closed clientGroupID=${this.id}`);
 		await this.#lock.withLock(() => {});
 		this.#pipelines.destroy();
+		this.#inspectorDelegate.clearAuthenticated(this.id, this);
 	}
 	/**
 	* Test helper: Manually mark initialization as complete.
@@ -1377,8 +1834,10 @@ function isTransformFailedError(error) {
 	return error.errorBody.kind === "TransformFailed" && !isAuthErrorBody(error.errorBody);
 }
 /**
-* A query must be expired for all clients in order to be considered
-* expired.
+* Whether a query's TTL has elapsed. A query must be expired for all clients
+* in order to be considered expired.
+*
+* Exported for testing.
 */
 function expired(ttlClock, q) {
 	if (q.type === "internal") return false;