@rocicorp/zero 1.11.0-canary.2 → 1.11.0-canary.21

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -1,4 +1,6 @@
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
+import { getOrInsertComputed } from "../../../../shared/src/map.js";
+import { newArray } from "../../../../shared/src/arrays.js";
 import { must } from "../../../../shared/src/must.js";
 import { h64 } from "../../../../shared/src/hash.js";
 import { Internal, InvalidConnectionRequest, InvalidConnectionRequestBaseCookie, Rehome } from "../../../../zero-protocol/src/error-kind-enum.js";
@@ -10,12 +12,14 @@ import { manualSpan, startAsyncSpan, startSpan } from "../../../../otel/src/span
 import { ZERO_VERSION_COLUMN_NAME } from "../replicator/schema/constants.js";
 import { transformAndHashQuery } from "../../auth/read-authorizer.js";
 import { randInt } from "../../../../shared/src/rand.js";
-import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateNativeHistogram, getOrCreateUpDownCounter } from "../../observability/metrics.js";
+import { LogThrottle } from "../../observability/log-throttle.js";
+import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateNativeHistogram, getOrCreateUpDownCounter, getOrCreateValueHistogram } from "../../observability/metrics.js";
 import { rowIDString } from "../../types/row-key.js";
 import "../replicator/schema/replication-state.js";
+import { planWarningMessage } from "./plan-warnings.js";
+import { queryShape } from "./query-shape.js";
 import { parseSignature } from "./row-set-signature.js";
 import { ResetPipelinesSignal } from "./snapshotter.js";
-import "./pipeline-driver.js";
 import { Subscription } from "../../types/subscription.js";
 import { CustomKeyMap } from "../../../../shared/src/custom-key-map.js";
 import { ttlClockAsNumber, ttlClockFromNumber } from "./ttl-clock.js";
@@ -25,8 +29,10 @@ import { isAuthErrorBody } from "../../auth/auth.js";
 import { ClientHandler, startPoke } from "./client-handler.js";
 import { tracer } from "./tracer.js";
 import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
-import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, nextEvictionTime } from "./cvr.js";
+import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, classifyQueriesForHydration, nextEvictionTime } from "./cvr.js";
 import { E2EServingLagTracker } from "./e2e-serving-lag.js";
+import { HydrationBudget } from "./hydration-budget.js";
+import { HydrationCircuitBreaker } from "./hydration-circuit-breaker.js";
 import { handleInspect } from "./inspect-handler.js";
 import { QueryCoveringIndex } from "./query-covering.js";
 import { resolver } from "@rocicorp/resolver";
@@ -50,6 +56,7 @@ function shutdownBeforeInitializationError() {
 * use a timer to update the ttlClock every minute.
 */
 var TTL_CLOCK_INTERVAL = 6e4;
+var slowHydrationLogThrottle = new LogThrottle({ windowMs: 5 * 6e4 });
 var ViewSyncerService = class {
 	id;
 	createdAtMs = Date.now();
@@ -61,6 +68,7 @@ var ViewSyncerService = class {
 	#drainCoordinator;
 	#keepaliveMs;
 	#slowHydrateThreshold;
+	#hydrationCircuitBreaker;
 	#lastConnectTime = Date.now();
 	/**
 	* The TTL clock is used to determine the time at which queries are considered
@@ -88,14 +96,37 @@ var ViewSyncerService = class {
 	#lock = new Lock();
 	#cvrStore;
 	#stopped = resolver();
+	/**
+	* Set when {@link #cleanup} begins. Lock tasks that were in flight when the
+	* view-syncer was stopped may still complete after the timers have been
+	* cleared; this flag prevents them from scheduling new timers, which would
+	* otherwise outlive the service (and retain everything it references).
+	*/
+	#shuttingDown = false;
 	#initialized = resolver();
 	#cvr;
-	#pipelinesSynced = false;
+	/**
+	* Indicates whether the query pipelines have completed initial catch-up and
+	* hydration with the CVR snapshot, and are ready for steady-state operation.
+	*
+	* - When `false`: The syncer is in its initial catch-up phase. The replica
+	*   advances without diffs until it reaches `cvr.version.stateVersion`,
+	*   after which `#maybeHydratePipelines` hydrates queries.
+	* - When `true`: Pipelines are fully populated and in steady-state; replica
+	*   changes advance incrementally via `#advancePipelines`, and client query
+	*   updates are processed via `#syncQueryPipelineSet`.
+	*
+	* Resets to `false` if `#advancePipelines` returns a `ResetPipelinesSignal`
+	* and pipelines must be reset and rehydrated, or if a reloaded CVR is ahead
+	* of the pipelines (see `#resetPipelinesIfBehindCVR`).
+	*/
+	#pipelinesHydrated = false;
 	#servedVersion = null;
 	#e2eServingLagTracker = new E2EServingLagTracker();
 	#expiredQueriesTimer = 0;
 	#authMaintenanceTimer = 0;
 	#setTimeout;
+	#now;
 	#customQueryTransformer;
 	#queryReplacements = /* @__PURE__ */ new Map();
 	#activeClients = getOrCreateUpDownCounter("sync", "active-clients", "Number of active sync clients");
@@ -122,10 +153,64 @@ var ViewSyncerService = class {
 	#pipelineResets = getOrCreateCounter("sync", "pipeline-resets", "Number of pipeline resets");
 	#rowSetSignatureDrifts = getOrCreateCounter("sync", "query.row-set-signature-drifts", "Number of times re-hydration of an unchanged query produced a different row-set signature than what is stored in the CVR (forcing a configVersion bump and full re-execution). Expected to be near-zero in steady state; persistent non-zero values indicate non-deterministic query execution (e.g. Cap operator picking different N-row subsets).");
 	#sameHashRehydrationVersionBumps = getOrCreateCounter("sync", "query.same-hash-rehydrations-forced-bump", "Number of times query-set reconciliation forced a configVersion bump for already-gotten same-transformation-hash query rehydration because trackQueries would not otherwise bump. Expected to be near-zero; non-zero values indicate pipeline/CVR row-set drift reached query-set reconciliation.");
+	#hydrationBudgetExhaustions = getOrCreateCounter("sync", "hydration_budget_exhaustions", {
+		description: "Number of hydration passes that exhaust their budget.",
+		unit: "{pass}"
+	});
+	#hydrationBudgetEvictions = getOrCreateCounter("sync", "hydration_budget_evictions", {
+		description: "Number of inactive queries removed after hydration budget exhaustion.",
+		unit: "{query}"
+	});
+	#hydrationBudgetElapsed = getOrCreateValueHistogram("sync", "hydration_budget_elapsed", {
+		description: "Elapsed milliseconds when optional query hydration stopped.",
+		unit: "ms",
+		bucketBoundaries: [
+			1,
+			2,
+			5,
+			10,
+			20,
+			50,
+			100,
+			200,
+			500,
+			1e3,
+			5e3
+		]
+	});
+	#hydrationBudgetOvershoot = getOrCreateValueHistogram("sync", "hydration_budget_overshoot", {
+		description: "Milliseconds elapsed beyond the configured hydration budget.",
+		unit: "ms",
+		bucketBoundaries: [
+			0,
+			1,
+			2,
+			5,
+			10,
+			20,
+			50,
+			100,
+			200,
+			500,
+			1e3
+		]
+	});
+	#queryEvictions = getOrCreateCounter("sync", "query_evictions", {
+		description: "Number of queries evicted from the CVR ahead of their removal by the client, grouped by reason. Inactive queries are evicted by ttl and hydration-budget; hydration-timeout and hydration-circuit-breaker evict active queries as well.",
+		unit: "{query}"
+	});
+	#hydrationTimeouts = getOrCreateCounter("sync", "hydration_timeouts", {
+		description: "Number of query hydrations aborted for exceeding the query hydration timeout.",
+		unit: "{query}"
+	});
+	#hydrationCircuitBreakerRejections = getOrCreateCounter("sync", "hydration_circuit_breaker_rejections", {
+		description: "Number of queries rejected without hydration because their hydration circuit breaker was open.",
+		unit: "{query}"
+	});
 	#inspectorDelegate;
 	#config;
 	#runPriorityOp;
-	constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis)) {
+	constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis), now = performance.now.bind(performance)) {
 		this.#config = config;
 		this.id = clientGroupID;
 		this.connContextManager = connContextManager;
@@ -140,6 +225,8 @@ var ViewSyncerService = class {
 		this.#customQueryTransformer = customQueryTransformer;
 		this.#cvrStore = new CVRStore(lc, cvrDb, shard, taskID, clientGroupID, () => this.#stateChanges.cancel());
 		this.#setTimeout = setTimeoutFn;
+		this.#now = now;
+		this.#hydrationCircuitBreaker = new HydrationCircuitBreaker(config.viewSyncerQueryHydrationTimeoutMs ?? 0, void 0, now);
 		this.#runPriorityOp = runPriorityOp;
 		this.keepalive();
 	}
@@ -166,11 +253,13 @@ var ViewSyncerService = class {
 				this.#stateChanges.cancel();
 				return;
 			}
+			let reloaded = false;
 			if (!this.#cvr) {
 				this.#lc.debug?.("loading cvr");
 				this.#cvr = await this.#runPriorityOp(lc, "loading cvr", () => this.#cvrStore.load(lc, this.#lastConnectTime));
 				this.#ttlClock = this.#cvr.ttlClock;
 				this.#ttlClockBase = Date.now();
+				reloaded = true;
 			} else {
 				const now = Date.now();
 				this.#cvr = {
@@ -179,6 +268,10 @@ var ViewSyncerService = class {
 				};
 			}
 			try {
+				if (reloaded && this.#resetPipelinesIfBehindCVR(lc, this.#cvr)) {
+					const connCtx = this.connContextManager.getBackgroundConnectionContext();
+					if (connCtx) await this.#maybeHydratePipelines(lc, this.#cvr, connCtx);
+				}
 				await fn(lc, this.#cvr);
 			} catch (e) {
 				this.#cvr = void 0;
@@ -188,11 +281,47 @@ var ViewSyncerService = class {
 			}
 		});
 	}
+	/**
+	* The CVR is reloaded after an error clears the cached copy. By then another
+	* view-syncer may have taken over the client group and flushed the CVR at a
+	* version ahead of this instance's hydrated pipelines (e.g. its replica was
+	* further ahead). The pipelines can no longer be diffed against the CVR, so
+	* reset them and let `#maybeHydratePipelines` rehydrate once the replica has
+	* caught up to the CVR.
+	*
+	* A CVR behind the pipelines is expected (advancements that do not change
+	* the CVR are not flushed) and is handled by the normal update path.
+	*
+	* Returns whether the pipelines were reset.
+	*
+	* Must be called from within the #lock.
+	*/
+	#resetPipelinesIfBehindCVR(lc, cvr) {
+		if (!this.#pipelinesHydrated) return false;
+		const pipelineVersion = this.#pipelines.currentVersion();
+		if (pipelineVersion >= cvr.version.stateVersion) return false;
+		lc.info?.(`resetting pipelines: pipelines@${pipelineVersion} are behind reloaded cvr@${versionString(cvr.version)}`);
+		this.#pipelineResets.add(1, { reason: "behind-cvr" });
+		this.#pipelinesHydrated = false;
+		this.connContextManager.setSharedRetransformReady(false);
+		this.#pipelines.reset(must(cvr.clientSchema, "cvr.clientSchema missing after initialization"));
+		return true;
+	}
 	readyState() {
-		return Promise.race([this.#initialized.promise, this.#drainCoordinator.draining]);
+		return new Promise((resolve, reject) => {
+			const unsubscribe = this.#drainCoordinator.onDraining(() => resolve("draining"));
+			this.#initialized.promise.then((state) => {
+				unsubscribe();
+				resolve(state);
+			}, (err) => {
+				unsubscribe();
+				reject(err);
+			});
+		});
 	}
 	async run() {
 		try {
+			this.#scheduleShutdown(this.#keepaliveMs);
 			if (await this.readyState() === "draining") {
 				this.#lc.debug?.(`draining view-syncer ${this.id} before running`);
 				this.stop();
@@ -213,26 +342,29 @@ var ViewSyncerService = class {
 						lc.info?.(`resetting CVR: ${message}`);
 						throw new ClientNotFoundError(message);
 					}
-					if (this.#pipelinesSynced) {
+					let previousQueries;
+					if (this.#pipelinesHydrated) {
 						const result = await this.#advancePipelines(lc, cvr);
 						if (result === "success") return;
 						lc.info?.(`resetting pipelines: ${result.message}`);
 						this.#pipelineResets.add(1, { reason: result.reason });
+						switch (result.reason) {
+							case "advancement-timeout":
+							case "scalar-subquery":
+							case "truncation":
+							case "schema-change":
+								if (!Object.values(cvr.queries).some((q) => q.type === "client")) previousQueries = new Map(this.#pipelines.queries());
+								break;
+							case "permissions-change": break;
+							default: unreachable(result.reason);
+						}
 						this.#pipelines.reset(clientSchema);
-						this.#pipelinesSynced = false;
+						this.#pipelinesHydrated = false;
 						this.connContextManager.setSharedRetransformReady(false);
 					}
-					const version = this.#pipelines.advanceWithoutDiff();
-					const cvrVer = versionString(cvr.version);
-					if (version < cvr.version.stateVersion) {
-						lc.debug?.(`replica@${version} is behind cvr@${cvrVer}`);
-						return;
-					}
-					lc.info?.(`init pipelines@${version} (cvr@${cvrVer})`);
-					const driftedQueryIDs = await this.#hydrateUnchangedQueries(lc, cvr);
-					await this.#syncQueryPipelineSet(lc, cvr, "missing", void 0, driftedQueryIDs);
-					this.#pipelinesSynced = true;
-					this.connContextManager.setSharedRetransformReady(true);
+					const backgroundConnCtx = this.connContextManager.getBackgroundConnectionContext();
+					if (backgroundConnCtx) await this.#maybeHydratePipelines(lc, cvr, backgroundConnCtx, previousQueries);
+					else lc.info?.("No validated background connection; deferring pipeline init");
 				});
 			}
 			if (this.#drainCoordinator.shouldDrain()) this.#drainCoordinator.drainNextIn(this.#totalHydrationTimeMs());
@@ -246,11 +378,66 @@ var ViewSyncerService = class {
 			this.#stopped.resolve();
 		}
 	}
+	/**
+	* Ensures that query pipelines are hydrated and ready for steady-state
+	* operation.
+	*
+	* Preconditions:
+	* 1. Must be called from within the `#lock`.
+	* 2. The caller must provide an active, validated `ConnectionContext` for
+	*    evaluating custom query transforms and permissions.
+	*
+	* Readiness prerequisites:
+	* - The underlying `QueryPipelineDriver` must be initialized with the schema
+	*   (i.e. `this.#pipelines.initialized()` is true).
+	* - The SQLite replica must have caught up to at least the CVR's
+	*   `stateVersion` (`version >= cvr.version.stateVersion`).
+	*
+	* If either prerequisite is not met, this method returns early as a no-op,
+	* waiting for the replica to catch up or the driver to initialize.
+	*
+	* Once ready:
+	* - Hydrates unchanged queries from the CVR snapshot without full row diffing.
+	* - Syncs missing, errored, or drifted queries via `#syncQueryPipelineSet`.
+	* - Sets `#pipelinesHydrated = true` and enables shared retransformations.
+	*/
+	async #maybeHydratePipelines(lc, cvr, connCtx, previousQueries) {
+		if (!this.#pipelines.initialized()) return;
+		let version;
+		try {
+			version = this.#pipelines.advanceWithoutDiff();
+		} catch (e) {
+			if (!(e instanceof ResetPipelinesSignal)) throw e;
+			lc.info?.(`resetting pipelines: ${e.message}`);
+			this.#pipelineResets.add(1, { reason: e.reason });
+			this.#pipelines.reset(must(cvr.clientSchema, "cvr.clientSchema missing after initialization"));
+			version = this.#pipelines.currentVersion();
+		}
+		const cvrVer = versionString(cvr.version);
+		if (version < cvr.version.stateVersion) {
+			lc.debug?.(`replica@${version} is behind cvr@${cvrVer}`);
+			return;
+		}
+		lc.info?.(`init pipelines@${version} (cvr@${cvrVer})`);
+		const hydrationBudget = new HydrationBudget(this.#config.viewSyncerHydrationBudgetMs ?? 0, this.#now);
+		const hydrationPassStats = {
+			activeHydratedQueries: 0,
+			inactiveHydratedQueries: 0
+		};
+		const driftedQueryIDs = await this.#hydrateUnchangedQueries(lc, cvr, connCtx, hydrationPassStats, hydrationBudget, previousQueries);
+		await this.#syncQueryPipelineSet(lc, cvr, "missing", connCtx, driftedQueryIDs, hydrationBudget, hydrationPassStats, previousQueries);
+		this.#pipelinesHydrated = true;
+		this.connContextManager.setSharedRetransformReady(true);
+	}
 	#removeExpiredQueries = async (lc, cvr) => {
 		if (hasExpiredQueries(cvr)) {
 			lc = lc.withContext("method", "#removeExpiredQueries");
 			lc.debug?.("Queries have expired");
-			if (this.#pipelinesSynced) await this.#syncQueryPipelineSet(lc, cvr, "missing", void 0);
+			if (this.#pipelinesHydrated) {
+				const connCtx = this.connContextManager.getBackgroundConnectionContext();
+				if (connCtx) await this.#syncQueryPipelineSet(lc, cvr, "missing", connCtx);
+				else lc.info?.("No validated background connection to remove expired queries; deferring");
+			}
 		}
 		this.#scheduleExpireEviction(lc, cvr);
 	};
@@ -332,7 +519,14 @@ var ViewSyncerService = class {
 		return true;
 	}
 	#shutdownTimer = null;
+	#stopShutdownTimer() {
+		if (this.#shutdownTimer !== null) {
+			clearTimeout(this.#shutdownTimer);
+			this.#shutdownTimer = null;
+		}
+	}
 	#scheduleShutdown(delayMs = 0) {
+		if (this.#shuttingDown) return;
 		this.#shutdownTimer ??= this.#setTimeout(() => {
 			this.#shutdownTimer = null;
 			this.#runInLockWithCVR(() => {}).catch((e) => this.#stateChanges.fail(e));
@@ -379,6 +573,7 @@ var ViewSyncerService = class {
 	*/
 	#scheduleAuthMaintenance(lc) {
 		this.#stopAuthMaintenanceTimer();
+		if (this.#shuttingDown) return;
 		const plan = this.connContextManager.planMaintenance();
 		if (plan.earliestDeadlineAt === void 0) {
 			lc.debug?.("No auth maintenance wakeup scheduled");
@@ -466,7 +661,7 @@ var ViewSyncerService = class {
 			}
 			lc.debug?.("Auth changed, re-validating and re-transforming queries");
 			const connCtx = this.connContextManager.mustGetConnectionContext(selector);
-			if (!this.#pipelinesSynced) {
+			if (!this.#pipelinesHydrated) {
 				if (!await this.#validateConnection(connCtx)) return;
 			}
 			return await this.#handleConfigUpdate(lc, clientID, {}, cvr, "all", void 0, connCtx);
@@ -484,17 +679,41 @@ var ViewSyncerService = class {
 		this.#ttlClockBase = now;
 		return ttlClock;
 	}
-	#flushUpdater(lc, updater) {
+	/**
+	* @param patchesPoked Whether patches computed against the updater's CVR
+	*     have already been poked to clients. If so, the CVR is checked to be
+	*     current even when the flush has nothing to write: another
+	*     view-syncer may have already committed identical rows at a newer
+	*     version, which would otherwise leave this one poking from a stale
+	*     CVR whose version it cannot advance.
+	*/
+	#flushUpdater(lc, updater, patchesPoked = false) {
 		return startAsyncSpan(tracer, "vs.#flushUpdater", () => this.#runPriorityOp(lc, "flushing cvr", async () => {
 			const now = Date.now();
 			const ttlClock = this.#getTTLClock(now);
-			const { cvr, flushed } = await updater.flush(lc, this.#lastConnectTime, now, ttlClock);
+			const { cvr, flushed } = await updater.flush(lc, this.#lastConnectTime, now, ttlClock, patchesPoked);
 			if (flushed) this.#startTTLClockInterval(lc);
 			return cvr;
 		}));
 	}
+	/**
+	* Flushes an updater whose changes have already been poked. If the flush
+	* fails (e.g. the CVR was concurrently modified), the poke is cancelled
+	* before the error propagates. The error may only fail the client that
+	* initiated the operation, and any other client left mid-poke would fail
+	* its next pokeStart.
+	*/
+	async #flushPoked(lc, updater, pokers) {
+		try {
+			return await this.#flushUpdater(lc, updater, pokers.patchesSent);
+		} catch (e) {
+			await pokers.cancel();
+			throw e;
+		}
+	}
 	#startTTLClockInterval(lc) {
 		this.#stopTTLClockInterval();
+		if (this.#shuttingDown) return;
 		this.#ttlClockInterval = this.#setTimeout(() => {
 			this.#updateTTLClockInCVRWithoutLock(lc);
 			this.#startTTLClockInterval(lc);
@@ -523,12 +742,13 @@ var ViewSyncerService = class {
 		if (cmpVersions(cvr.version, this.#cvr.version) < 0) {
 			const newCVR = this.#cvr;
 			await startAsyncSpan(tracer, "vs.#updateCVRConfig.pokeClients", async () => {
-				const pokers = startPoke(this.#getClients(cvr.version), newCVR.version);
+				const pokers = startPoke(lc, this.#getClients(cvr.version), newCVR.version);
 				for (const patch of patches) await pokers.addPatch(patch);
 				await pokers.end(newCVR.version);
 			});
 		}
-		if (this.#pipelinesSynced) await this.#syncQueryPipelineSet(lc, this.#cvr, customQueryTransformMode, connCtx);
+		if (!this.#pipelinesHydrated) await this.#maybeHydratePipelines(lc, this.#cvr, connCtx);
+		else await this.#syncQueryPipelineSet(lc, this.#cvr, customQueryTransformMode, connCtx);
 		return this.#cvr;
 	}
 	/**
@@ -624,6 +844,7 @@ var ViewSyncerService = class {
 	#scheduleExpireEviction(lc, cvr) {
 		const { ttlClock } = cvr;
 		this.#stopExpireTimer();
+		if (this.#shuttingDown) return;
 		const next = nextEvictionTime(cvr);
 		if (next === void 0) {
 			lc.debug?.("no inactive queries with ttl");
@@ -651,7 +872,7 @@ var ViewSyncerService = class {
 	*
 	* This must be called from within the #lock.
 	*/
-	async #hydrateUnchangedQueries(lc, cvr) {
+	async #hydrateUnchangedQueries(lc, cvr, connCtx, hydrationPassStats, hydrationBudget, previousQueries) {
 		assert(this.#pipelines.initialized(), "pipelines must be initialized");
 		const dbVersion = this.#pipelines.currentVersion();
 		const cvrVersion = cvr.version;
@@ -660,33 +881,34 @@ var ViewSyncerService = class {
 			return /* @__PURE__ */ new Set();
 		}
 		const gotQueries = Object.entries(cvr.queries).filter(([_, state]) => state.transformationHash !== void 0);
+		const { required: requiredGotQueries } = classifyQueriesForHydration(gotQueries.map(([, query]) => query));
 		const customQueries = /* @__PURE__ */ new Map();
 		const otherQueries = [];
-		let inactivatedCount = 0;
-		for (const [, query] of gotQueries) {
-			if (query.type !== "internal" && Object.values(query.clientState).every(({ inactivatedAt }) => inactivatedAt !== void 0)) {
-				inactivatedCount++;
-				continue;
-			}
-			if (query.type === "custom") customQueries.set(query.id, query);
-			else otherQueries.push(query);
-		}
+		const inactivatedCount = gotQueries.length - requiredGotQueries.length;
 		const transformedQueries = [];
+		for (const query of requiredGotQueries) if (query.type === "custom") {
+			const previous = previousQueries?.get(query.id);
+			if (previous && previous.transformationHash === query.transformationHash) transformedQueries.push({
+				id: query.id,
+				transformationHash: previous.transformationHash,
+				transformedAst: previous.originalAst ?? previous.transformedAst
+			});
+			else customQueries.set(query.id, query);
+		} else otherQueries.push(query);
 		let customErrorCount = 0;
 		let customHashMismatchCount = 0;
 		let otherHashMismatchCount = 0;
 		if (customQueries.size > 0 && !this.#customQueryTransformer) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
-		const backgroundContext = this.connContextManager.mustGetBackgroundConnectionContext();
 		const customQueryTransformer = this.#customQueryTransformer;
 		if (customQueryTransformer && customQueries.size > 0) {
-			const transformedCustomQueries = await this.#runPriorityOp(lc, "#hydrateUnchangedQueries transforming custom queries", () => customQueryTransformer.transform(backgroundContext, customQueries.values()));
-			if (transformedCustomQueries.kind === "success" && !transformedCustomQueries.cached) this.connContextManager.validateConnection(backgroundContext, backgroundContext.revision, transformedCustomQueries.validation);
+			const transformedCustomQueries = await hydrationBudget.excluding(() => this.#runPriorityOp(lc, "#hydrateUnchangedQueries transforming custom queries", () => customQueryTransformer.transform(connCtx, customQueries.values())));
+			if (transformedCustomQueries.kind === "success" && !transformedCustomQueries.cached) this.connContextManager.validateConnection(connCtx, connCtx.revision, transformedCustomQueries.validation);
 			if (Array.isArray(transformedCustomQueries.result)) for (const q of transformedCustomQueries.result) if ("error" in q) customErrorCount++;
 			else if (q.transformationHash !== customQueries.get(q.id)?.transformationHash) customHashMismatchCount++;
 			else transformedQueries.push(q);
 		}
 		for (const q of otherQueries) {
-			const transformed = transformAndHashQuery(lc, q.id, q.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, backgroundContext.auth?.type === "jwt" ? backgroundContext.auth : void 0, q.type === "internal");
+			const transformed = transformAndHashQuery(lc, q.id, q.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, connCtx.auth?.type === "jwt" ? connCtx.auth : void 0, q.type === "internal");
 			if (transformed.transformationHash === q.transformationHash) transformedQueries.push(transformed);
 			else otherHashMismatchCount++;
 		}
@@ -699,6 +921,10 @@ var ViewSyncerService = class {
 		for (const { id: queryID, transformationHash, transformedAst } of transformedQueries) {
 			const query = cvr.queries[queryID];
 			const queryName = query.type === "custom" ? query.name : void 0;
+			if (query.type !== "internal" && this.#hydrationCircuitBreaker.isOpen(transformationHash)) {
+				lc.info?.(`skipping hydration of ${queryID}: hydration circuit breaker open`);
+				continue;
+			}
 			const covered = queryCoveringIndex ? this.#findQueryCoverageShadowHit(queryCoveringIndex, queryID, transformationHash, transformedAst, queryName) : void 0;
 			totalHydratedQueries++;
 			if (covered) {
@@ -707,20 +933,43 @@ var ViewSyncerService = class {
 			}
 			const timer = new TimeSliceTimer(lc);
 			let count = 0;
+			let timedOut = false;
 			await startAsyncSpan(tracer, "vs.#hydrateUnchangedQueries.addQuery", async (span) => {
 				span.setAttribute("queryHash", queryID);
 				span.setAttribute("transformationHash", transformationHash);
 				span.setAttribute("table", transformedAst.table);
 				if (queryName !== void 0) span.setAttribute("queryName", queryName);
-				for (const change of this.#pipelines.addQuery(transformationHash, queryID, transformedAst, await timer.start(), queryName, "unchanged-query-rehydrate")) if (change === "yield") await timer.yieldProcess("yield in hydrateUnchangedQueries");
-				else count++;
+				for (const change of this.#pipelines.addQuery(transformationHash, queryID, transformedAst, await timer.start(), queryName, "unchanged-query-rehydrate")) if (change === "yield") {
+					if (query.type !== "internal" && this.#hydrationCircuitBreaker.exceeded(timer.totalElapsed())) {
+						timedOut = true;
+						await timer.yieldProcess("yield in hydrateUnchangedQueries");
+						break;
+					}
+					await timer.yieldProcess("yield in hydrateUnchangedQueries");
+				} else count++;
 			});
 			const elapsed = timer.totalElapsed();
+			if (timedOut) {
+				this.#recordHydrationTimeout(lc, {
+					id: queryID,
+					ast: transformedAst,
+					transformationHash,
+					name: queryName
+				}, elapsed, count);
+				continue;
+			}
+			hydrationPassStats.activeHydratedQueries++;
 			this.#hydrations.add(1);
 			this.#hydrationTime.recordMs(elapsed);
-			this.#addQueryMaterializationServerMetric(transformationHash, elapsed);
-			this.#inspectorDelegate.addQuery(transformationHash, transformedAst);
+			this.#addQueryMaterializationServerMetric(queryID, elapsed);
+			this.#inspectorDelegate.addQuery(queryID, transformedAst);
 			lc.debug?.(`hydrated ${count} rows for ${queryID} (${elapsed} ms)`);
+			if (elapsed > this.#slowHydrateThreshold) this.#logSlowHydration(lc, {
+				id: queryID,
+				ast: transformedAst,
+				transformationHash,
+				name: queryName
+			}, elapsed);
 			let drifted = false;
 			const storedSigHex = cvr.queries[queryID]?.rowSetSignature;
 			if (storedSigHex !== void 0 && storedSigHex !== null) {
@@ -780,11 +1029,7 @@ var ViewSyncerService = class {
 			return;
 		}
 		const appErrorGroups = /* @__PURE__ */ new Map();
-		for (const err of errorOrErrors) for (const clientId of getAffectedClientIDs([err.id])) {
-			const group = appErrorGroups.get(clientId) ?? [];
-			group.push(err);
-			appErrorGroups.set(clientId, group);
-		}
+		for (const err of errorOrErrors) for (const clientId of getAffectedClientIDs([err.id])) getOrInsertComputed(appErrorGroups, clientId, newArray).push(err);
 		for (const [clientId, errors] of appErrorGroups) this.#clients.get(clientId)?.sendQueryTransformApplicationErrors(errors);
 	}
 	#addQueryMaterializationServerMetric(queryID, elapsed) {
@@ -820,7 +1065,10 @@ var ViewSyncerService = class {
 	*
 	* This must be called from within the #lock.
 	*/
-	#syncQueryPipelineSet(lc, cvr, customQueryTransformMode, connCtx, driftedQueryIDs = /* @__PURE__ */ new Set()) {
+	#syncQueryPipelineSet(lc, cvr, customQueryTransformMode, connCtx, driftedQueryIDs = /* @__PURE__ */ new Set(), hydrationBudget = new HydrationBudget(this.#config.viewSyncerHydrationBudgetMs ?? 0, this.#now), hydrationPassStats = {
+		activeHydratedQueries: 0,
+		inactiveHydratedQueries: 0
+	}, previousQueries) {
 		return startAsyncSpan(tracer, "vs.#syncQueryPipelineSet", async (span) => {
 			const start = performance.now();
 			span.setAttribute("clientGroupID", this.id);
@@ -829,38 +1077,45 @@ var ViewSyncerService = class {
 			const now = Date.now();
 			const ttlClock = this.#getTTLClock(now);
 			const cvrQueryEntires = Object.entries(cvr.queries);
-			const customQueries = /* @__PURE__ */ new Map();
-			const otherQueries = [];
 			const transformedQueries = [];
-			const resolvedConnCtx = connCtx ?? this.connContextManager.mustGetBackgroundConnectionContext();
-			for (const [id, query] of cvrQueryEntires) if (query.type === "custom") {
-				assert(id === query.id, "custom query id mismatch");
-				customQueries.set(id, query);
-			} else otherQueries.push({
-				id,
-				query
-			});
-			for (const { id, query: origQuery } of otherQueries) {
-				assert(id === origQuery.id, "query id mismatch");
-				const transformed = transformAndHashQuery(lc, origQuery.id, origQuery.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, resolvedConnCtx.auth?.type === "jwt" ? resolvedConnCtx.auth : void 0, origQuery.type === "internal");
-				transformedQueries.push({
-					id,
-					origQuery,
-					transformed
-				});
-			}
-			if (customQueries.size > 0 && !this.#customQueryTransformer) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
-			let erroredQueryIDs;
-			const customQueriesToTransform = customQueryTransformMode === "all" ? [...customQueries.values()] : customQueryTransformMode && [...customQueries.values()].filter((q) => !this.#pipelines.queries().has(q.id));
+			const naturallyExpiredQueryIDs = new Set(cvrQueryEntires.map(([, query]) => query).filter((query) => expired(ttlClock, query)).map((query) => query.id));
+			if (naturallyExpiredQueryIDs.size > 0) this.#queryEvictions.add(naturallyExpiredQueryIDs.size, { reason: "ttl" });
+			const hydrationCandidates = cvrQueryEntires.map(([, query]) => query).filter((query) => !naturallyExpiredQueryIDs.has(query.id));
+			const { required, optional } = classifyQueriesForHydration(hydrationCandidates);
+			const requiredQueryIDs = new Set(required.map((query) => query.id));
+			const optionalToHydrate = hydrationBudget.limitMs === 0 ? optional : optional.filter((query) => query.patchVersion !== void 0);
+			if (!this.#customQueryTransformer && hydrationCandidates.some((query) => query.type === "custom")) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
 			const customQueryTransformer = this.#customQueryTransformer;
-			if (customQueryTransformer && customQueriesToTransform.length > 0) {
+			const erroredQueryIDs = [];
+			let transformedCustomQueryCount = 0;
+			const transformOtherQueries = (queries) => {
+				for (const origQuery of queries) {
+					if (origQuery.type === "custom") continue;
+					const transformed = transformAndHashQuery(lc, origQuery.id, origQuery.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, connCtx.auth?.type === "jwt" ? connCtx.auth : void 0, origQuery.type === "internal");
+					transformedQueries.push({
+						id: origQuery.id,
+						origQuery,
+						transformed
+					});
+				}
+			};
+			const shouldTransformCustomQuery = (query) => {
+				if (customQueryTransformMode === "all") return true;
+				if (this.#pipelines.queries().has(query.id)) return false;
+				const previous = previousQueries?.get(query.id);
+				if (previous && previous.transformationHash === query.transformationHash) return false;
+				return true;
+			};
+			const transformCustomQueries = async (queries) => {
+				if (!customQueryTransformer || queries.length === 0) return;
+				transformedCustomQueryCount += queries.length;
 				const transformStart = performance.now();
 				let transformedCustomQueries;
 				try {
-					transformedCustomQueries = await this.#runPriorityOp(lc, "#syncQueryPipelineSet transforming custom queries", () => customQueryTransformer.transform(resolvedConnCtx, customQueriesToTransform));
+					transformedCustomQueries = await hydrationBudget.excluding(() => this.#runPriorityOp(lc, "#syncQueryPipelineSet transforming custom queries", () => customQueryTransformer.transform(connCtx, queries)));
 					if (transformedCustomQueries.kind === "failed") throw new ProtocolErrorWithLevel(transformedCustomQueries.result, "warn");
 					else {
-						if (!transformedCustomQueries.cached) this.connContextManager.validateConnection(resolvedConnCtx, resolvedConnCtx.revision, transformedCustomQueries.validation);
+						if (!transformedCustomQueries.cached) this.connContextManager.validateConnection(connCtx, connCtx.revision, transformedCustomQueries.validation);
 						this.#queryTransformations.add(1, { result: "success" });
 					}
 				} catch (e) {
@@ -871,8 +1126,9 @@ var ViewSyncerService = class {
 					this.#queryTransformationTime.recordMs(transformDuration);
 				}
 				const successfullyTransformedCustomQueries = /* @__PURE__ */ new Map();
-				erroredQueryIDs = this.#processTransformedCustomQueries(lc, transformedCustomQueries.result, (q) => {
-					const origQuery = customQueries.get(q.id);
+				const customQueryMap = new Map(queries.map((query) => [query.id, query]));
+				erroredQueryIDs.push(...this.#processTransformedCustomQueries(lc, transformedCustomQueries.result, (q) => {
+					const origQuery = customQueryMap.get(q.id);
 					if (origQuery) {
 						successfullyTransformedCustomQueries.set(q.id, q);
 						transformedQueries.push({
@@ -881,7 +1137,7 @@ var ViewSyncerService = class {
 							transformed: q
 						});
 					}
-				}, customQueries);
+				}, customQueryMap));
 				for (const [queryID, newTransform] of successfullyTransformedCustomQueries) {
 					const existingTransformHash = cvr.queries[queryID]?.transformationHash;
 					if (existingTransformHash) {
@@ -894,21 +1150,55 @@ var ViewSyncerService = class {
 						} else this.#queryTransformationNoOps.add(1);
 					}
 				}
+			};
+			const queriesToTransform = [...required, ...optionalToHydrate];
+			transformOtherQueries(queriesToTransform);
+			if (customQueryTransformMode === "missing" && previousQueries) {
+				for (const query of queriesToTransform) if (query.type === "custom" && !this.#pipelines.queries().has(query.id)) {
+					const previous = previousQueries.get(query.id);
+					if (previous && previous.transformationHash === query.transformationHash) {
+						this.#queryTransformationNoOps.add(1);
+						transformedQueries.push({
+							id: query.id,
+							origQuery: query,
+							transformed: {
+								id: query.id,
+								transformationHash: previous.transformationHash,
+								transformedAst: previous.originalAst ?? previous.transformedAst
+							}
+						});
+					}
+				}
 			}
-			const removeQueriesQueryIds = new Set([...Object.values(cvr.queries).filter((q) => expired(ttlClock, q)).map((q) => q.id), ...erroredQueryIDs || []]);
+			await transformCustomQueries(queriesToTransform.filter((query) => query.type === "custom" && shouldTransformCustomQuery(query)));
+			const circuitBrokenQueries = transformedQueries.filter(({ id, origQuery, transformed }) => origQuery.type !== "internal" && this.#pipelines.queries().get(id)?.transformationHash !== transformed.transformationHash && this.#hydrationCircuitBreaker.isOpen(transformed.transformationHash)).map(({ id, origQuery, transformed }) => ({
+				id,
+				transformationHash: transformed.transformationHash,
+				name: origQuery.type === "custom" ? origQuery.name : void 0
+			}));
+			if (circuitBrokenQueries.length > 0) this.#rejectCircuitBrokenQueries(lc, cvr, circuitBrokenQueries);
+			const removeQueriesQueryIds = new Set([
+				...naturallyExpiredQueryIDs,
+				...erroredQueryIDs,
+				...circuitBrokenQueries.map(({ id }) => id)
+			]);
 			const addQueries = transformedQueries.map(({ id, origQuery, transformed }) => ({
 				id,
 				ast: transformed.transformedAst,
 				transformationHash: transformed.transformationHash,
 				name: origQuery.type === "custom" ? origQuery.name : void 0
 			})).filter((q) => !removeQueriesQueryIds.has(q.id) && this.#pipelines.queries().get(q.id)?.transformationHash !== q.transformationHash);
-			lc.info?.(`syncQueryPipelineSet: ${cvrQueryEntires.length} CVR queries, ${customQueriesToTransform.length} custom re-transformed, ${erroredQueryIDs?.length ?? 0} errored, ${removeQueriesQueryIds.size} to remove, ${addQueries.length} to add`);
+			const hydrationOrder = new Map([...required, ...optional].map((query, index) => [query.id, index]));
+			addQueries.sort((a, b) => must(hydrationOrder.get(a.id)) - must(hydrationOrder.get(b.id)));
+			const requiredAddQueries = addQueries.filter((query) => requiredQueryIDs.has(query.id));
+			const optionalAddQueries = addQueries.filter((query) => !requiredQueryIDs.has(query.id));
+			lc.info?.(`syncQueryPipelineSet: ${cvrQueryEntires.length} CVR queries, ${transformedCustomQueryCount} custom re-transformed, ${erroredQueryIDs.length} errored, ${circuitBrokenQueries.length} circuit-broken, ${removeQueriesQueryIds.size} to remove, ${addQueries.length} to add`);
 			for (const q of addQueries) {
 				const orig = cvr.queries[q.id];
 				lc.debug?.("ViewSyncer adding query", q.ast, "transformed from", orig.type === "custom" ? orig.name : orig.ast);
 			}
 			if (addQueries.length > 0 || removeQueriesQueryIds.size > 0) {
-				await this.#addAndRemoveQueries(lc, cvr, addQueries, Array.from(removeQueriesQueryIds, (id) => ({ id })), driftedQueryIDs);
+				await this.#addAndRemoveQueries(lc, cvr, requiredAddQueries, optionalAddQueries, Array.from(removeQueriesQueryIds, (id) => ({ id })), hydrationBudget, hydrationPassStats, driftedQueryIDs);
 				if (addQueries.length > 0) this.#viewSyncerHydration.recordMs(performance.now() - start);
 			} else await this.#catchupClients(lc, cvr);
 		});
@@ -941,8 +1231,128 @@ var ViewSyncerService = class {
 		record.count++;
 		if (record.count >= THRASH_THRESHOLD) this.#lc.warn?.(`Query thrashing detected for query ${queryID}. ${record.count} replacements in 60s. This may indicate clients with different auth contexts connecting to the same client group.`);
 	}
-	#addAndRemoveQueries(lc, cvr, addQueries, removeQueries, driftedQueryIDs = /* @__PURE__ */ new Set()) {
+	/**
+	* Records that hydrating `query` was aborted for exceeding the query
+	* hydration timeout, and opens its circuit breaker.
+	*/
+	#recordHydrationTimeout(lc, query, elapsedMs, rowCount) {
+		this.#hydrationCircuitBreaker.trip(query.transformationHash);
+		this.#hydrationTimeouts.add(1);
+		if (!lc.warn) return;
+		const shape = queryShape(query.ast);
+		lc.warn("Query hydration aborted for exceeding the hydration timeout", {
+			clientGroupID: this.id,
+			queryHash: query.id,
+			transformationHash: query.transformationHash,
+			...query.name !== void 0 && { queryName: query.name },
+			queryShape: shape.hash,
+			hydrationTimeoutMs: this.#hydrationCircuitBreaker.timeoutMs,
+			hydrationElapsedMs: elapsedMs,
+			...rowCount !== void 0 && { hydrationRowCount: rowCount },
+			circuitBreakerOpenMs: this.#hydrationCircuitBreaker.openMs,
+			zql: shape.zql
+		});
+	}
+	/**
+	* Logs a hydration that exceeded the slow hydration threshold.
+	*
+	* The query is logged as its {@link queryShape}, with literal values
+	* redacted, and each shape is logged at most once per
+	* {@link SLOW_HYDRATION_LOG_WINDOW_MS}. The next log of a shape reports how
+	* many slow hydrations of it were suppressed in the meantime.
+	*/
+	#logSlowHydration(lc, query, elapsedMs) {
+		if (!lc.warn) return;
+		const stats = this.#pipelines.hydrationStats(query.id);
+		const shape = stats?.shape ?? queryShape(query.ast);
+		const suppressed = slowHydrationLogThrottle.admit(`${query.name ?? ""}:${shape.hash}`);
+		if (suppressed === void 0) return;
+		lc.warn("Slow query materialization", {
+			zeroEvent: "query-slow-hydration",
+			clientGroupID: this.id,
+			queryHash: query.id,
+			transformationHash: query.transformationHash,
+			...query.name !== void 0 && { queryName: query.name },
+			queryShape: shape.hash,
+			hydrationTimeMs: elapsedMs,
+			...stats && {
+				hydrationRowCount: stats.rowCount,
+				hydrationRowsRead: stats.rowsRead
+			},
+			...stats !== void 0 && stats.planWarnings.length > 0 && { planWarnings: stats.planWarnings.map(planWarningMessage) },
+			...suppressed > 0 && { suppressedSinceLastLog: suppressed },
+			zql: shape.zql
+		});
+	}
+	/** Records that `query` was rejected without hydration by its open breaker. */
+	#recordCircuitBreakerRejection(lc, query) {
+		this.#hydrationCircuitBreakerRejections.add(1);
+		lc.warn?.("Query rejected by its open hydration circuit breaker", {
+			clientGroupID: this.id,
+			queryHash: query.id,
+			transformationHash: query.transformationHash,
+			...query.name !== void 0 && { queryName: query.name },
+			hydrationTimeoutMs: this.#hydrationCircuitBreaker.timeoutMs,
+			circuitBreakerOpenMs: this.#hydrationCircuitBreaker.openMs
+		});
+	}
+	/**
+	* Handles queries whose hydration circuit breaker is open: they are counted,
+	* logged, and errored to the affected clients. The caller removes them from
+	* the CVR.
+	*/
+	#rejectCircuitBrokenQueries(lc, cvr, queries) {
+		for (const query of queries) this.#recordCircuitBreakerRejection(lc, query);
+		this.#queryEvictions.add(queries.length, { reason: "hydration-circuit-breaker" });
+		this.#sendHydrationTimeoutErrors(cvr, queries);
+	}
+	/**
+	* Sends a per-query error to every client that desires one of `queries`.
+	* The error goes out as a `transformError` application error, which every
+	* client understands as "this query errored" without affecting the
+	* connection or the client's other queries.
+	*/
+	#sendHydrationTimeoutErrors(cvr, queries) {
+		const timeoutMs = this.#hydrationCircuitBreaker.timeoutMs;
+		const errorsByClient = /* @__PURE__ */ new Map();
+		for (const { id, name } of queries) {
+			const query = cvr.queries[id];
+			if (query === void 0 || query.type === "internal") continue;
+			const error = {
+				error: "app",
+				id,
+				name: name ?? (query.type === "custom" ? query.name : "legacy"),
+				message: `Query hydration exceeded the ${timeoutMs}ms limit (ZERO_VIEW_SYNCER_QUERY_HYDRATION_TIMEOUT_MS) and was aborted`,
+				details: {
+					kind: "HydrationTimeout",
+					timeoutMs
+				}
+			};
+			for (const clientID of Object.keys(query.clientState)) getOrInsertComputed(errorsByClient, clientID, newArray).push(error);
+		}
+		for (const [clientID, errors] of errorsByClient) this.#clients.get(clientID)?.sendQueryTransformApplicationErrors(errors);
+	}
+	#recordHydrationBudgetExhaustion(lc, hydrationBudget, activeHydratedQueries, inactiveHydratedQueries, inactiveEvictedQueryIDs) {
+		const elapsedMs = hydrationBudget.exhaustedAtMs;
+		if (elapsedMs === void 0) return;
+		this.#hydrationBudgetExhaustions.add(1);
+		this.#hydrationBudgetEvictions.add(inactiveEvictedQueryIDs.length);
+		this.#hydrationBudgetElapsed.record(elapsedMs);
+		this.#hydrationBudgetOvershoot.record(Math.max(0, elapsedMs - hydrationBudget.limitMs));
+		if (inactiveEvictedQueryIDs.length > 0) this.#queryEvictions.add(inactiveEvictedQueryIDs.length, { reason: "hydration-budget" });
+		lc.info?.("view-syncer hydration budget exhausted", {
+			clientGroupID: this.id,
+			hydrationBudgetMs: hydrationBudget.limitMs,
+			hydrationElapsedMs: elapsedMs,
+			activeHydratedQueries,
+			inactiveHydratedQueries,
+			inactiveEvictedQueries: inactiveEvictedQueryIDs.length,
+			firstEvictedQueryHash: inactiveEvictedQueryIDs[0] ?? null
+		});
+	}
+	#addAndRemoveQueries(lc, cvr, requiredQueries, optionalQueries, removeQueries, hydrationBudget, hydrationPassStats, driftedQueryIDs = /* @__PURE__ */ new Set()) {
 		return startAsyncSpan(tracer, "vs.#addAndRemoveQueries", async () => {
+			const addQueries = [...requiredQueries, ...optionalQueries];
 			assert(addQueries.length > 0 || removeQueries.length > 0, "Must have queries to add or remove");
 			const start = performance.now();
 			const stateVersion = this.#pipelines.currentVersion();
@@ -959,7 +1369,9 @@ var ViewSyncerService = class {
 				updater.ensureNewVersion();
 			}
 			const { queryPatches, newVersion } = updater.trackQueries(lc, addQueries, removeQueries);
-			const pokers = startPoke(this.#getClients(), newVersion);
+			if (hydrationBudget.limitMs !== 0 && optionalQueries.length > 0) assert(cmpVersions(cvr.version, newVersion) < 0, "Optional hydration requires a final poke version before row processing so a removal is actually poked");
+			const clients = this.#getClients();
+			const pokers = startPoke(lc, clients, newVersion);
 			for (const patch of queryPatches) await pokers.addPatch(patch);
 			for (const q of removeQueries) {
 				this.#pipelines.removeQuery(q.id);
@@ -975,22 +1387,54 @@ var ViewSyncerService = class {
 			let totalHydratedQueries = 0;
 			let coveredHydratedQueries = 0;
 			let firstCoveredQuery;
+			const hydratedQueryIDs = [];
+			const budgetEvictedQueryIDs = [];
+			const timedOutQueries = [];
+			const rejectedQueries = [];
+			const circuitBreaker = this.#hydrationCircuitBreaker;
 			const self = this;
 			await yieldProcess(lc);
-			function* generateRowChanges(slowHydrateThreshold) {
-				for (const q of addQueries) {
+			function* hydrateQueries(queries, optional, slowHydrateThreshold) {
+				for (let i = 0; i < queries.length; i++) {
+					if (optional && hydrationBudget.exhausted()) {
+						budgetEvictedQueryIDs.push(...queries.slice(i).map((query) => query.id));
+						return;
+					}
+					const q = must(queries[i]);
 					let queryLC = lc.withContext("hash", q.id).withContext("queryHash", q.id).withContext("transformationHash", q.transformationHash);
 					if (q.name !== void 0) queryLC = queryLC.withContext("queryName", q.name);
 					queryLC.debug?.(`adding pipeline for query`, q.ast);
+					const breakable = cvr.queries[q.id]?.type !== "internal";
+					if (breakable && circuitBreaker.isOpen(q.transformationHash)) {
+						rejectedQueries.push(q);
+						self.#recordCircuitBreakerRejection(queryLC, q);
+						continue;
+					}
 					const covered = queryCoveringIndex ? self.#findQueryCoverageShadowHit(queryCoveringIndex, q.id, q.transformationHash, q.ast, q.name) : void 0;
 					totalHydratedQueries++;
 					if (covered) {
 						coveredHydratedQueries++;
 						firstCoveredQuery ??= covered;
 					}
-					yield* pipelines.addQuery(q.transformationHash, q.id, q.ast, timer.startWithoutYielding(), q.name, "query-set-sync");
+					let timedOut = false;
+					for (const change of pipelines.addQuery(q.transformationHash, q.id, q.ast, timer.startWithoutYielding(), q.name, "query-set-sync")) {
+						if (change === "yield" && breakable && circuitBreaker.exceeded(timer.totalElapsed())) {
+							timedOut = true;
+							yield change;
+							break;
+						}
+						yield change;
+					}
 					const elapsed = timer.stop();
 					totalProcessTime += elapsed;
+					if (timedOut) {
+						timedOutQueries.push(q);
+						self.#recordHydrationTimeout(queryLC, q, elapsed);
+						continue;
+					}
+					hydratedQueryIDs.push(q.id);
+					if (optional) hydrationPassStats.inactiveHydratedQueries++;
+					else hydrationPassStats.activeHydratedQueries++;
 					self.#addQueryMaterializationServerMetric(q.id, elapsed);
 					self.#inspectorDelegate.addQuery(q.id, q.ast);
 					queryCoveringIndex?.add(q.id, {
@@ -998,24 +1442,49 @@ var ViewSyncerService = class {
 						transformationHash: q.transformationHash,
 						...q.name !== void 0 && { queryName: q.name }
 					});
-					if (elapsed > slowHydrateThreshold) queryLC.warn?.("Slow query materialization", elapsed, q.ast);
+					if (elapsed > slowHydrateThreshold) self.#logSlowHydration(queryLC, q, elapsed);
 					manualSpan(tracer, "vs.addAndConsumeQuery", elapsed, {
 						hash: q.id,
 						transformationHash: q.transformationHash,
 						...q.name !== void 0 && { name: q.name }
 					});
+					hydrations.add(1);
+					hydrationTime.recordMs(elapsed);
 				}
-				hydrations.add(1);
-				hydrationTime.recordMs(totalProcessTime);
+			}
+			function* generateRowChanges(slowHydrateThreshold) {
+				yield* hydrateQueries(requiredQueries, false, slowHydrateThreshold);
+				yield* hydrateQueries(optionalQueries, true, slowHydrateThreshold);
 			}
 			await this.#processChanges(lc, timer, generateRowChanges(this.#slowHydrateThreshold), updater, pokers);
+			const abortedQueries = [...timedOutQueries, ...rejectedQueries];
+			if (abortedQueries.length > 0) {
+				const abortedQueryIDs = abortedQueries.map(({ id }) => id);
+				for (const patch of await updater.abortExecutedQueries(lc, abortedQueryIDs)) await pokers.addPatch(patch);
+				for (const queryID of abortedQueryIDs) {
+					this.#pipelines.removeQuery(queryID);
+					this.#inspectorDelegate.removeQuery(queryID);
+					this.#queryReplacements.delete(queryID);
+				}
+				if (timedOutQueries.length > 0) this.#queryEvictions.add(timedOutQueries.length, { reason: "hydration-timeout" });
+				if (rejectedQueries.length > 0) this.#queryEvictions.add(rejectedQueries.length, { reason: "hydration-circuit-breaker" });
+				this.#sendHydrationTimeoutErrors(cvr, abortedQueries);
+			}
+			for (const patch of updater.removeTrackedQueries(budgetEvictedQueryIDs)) await pokers.addPatch(patch);
+			for (const queryID of budgetEvictedQueryIDs) {
+				this.#pipelines.removeQuery(queryID);
+				this.#inspectorDelegate.removeQuery(queryID);
+				this.#queryReplacements.delete(queryID);
+			}
+			this.#recordHydrationBudgetExhaustion(lc, hydrationBudget, hydrationPassStats.activeHydratedQueries, hydrationPassStats.inactiveHydratedQueries, budgetEvictedQueryIDs);
 			this.#logQueryCoverageShadowSummary(lc, "add", totalHydratedQueries, coveredHydratedQueries, firstCoveredQuery);
 			await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.deleteUnreferencedRows", async () => {
 				for (const patch of await updater.deleteUnreferencedRows(lc)) await pokers.addPatch(patch);
 			});
-			this.#cvr = await this.#flushUpdater(lc, updater);
+			this.#cvr = await this.#flushPoked(lc, updater, pokers);
+			if (budgetEvictedQueryIDs.length > 0) this.#scheduleExpireEviction(lc, this.#cvr);
 			const finalVersion = this.#cvr.version;
-			await this.#catchupClients(lc, cvr, finalVersion, addQueries.map((q) => q.id), pokers);
+			await this.#catchupClients(lc, cvr, finalVersion, hydratedQueryIDs, pokers);
 			await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.pokeEnd", () => pokers.end(finalVersion));
 			this.#markVersionServed(stateVersion);
 			const wallTime = performance.now() - start;
@@ -1044,7 +1513,7 @@ var ViewSyncerService = class {
 		return startAsyncSpan(tracer, "vs.#catchupClients", async (span) => {
 			current ??= cvr.version;
 			const clients = this.#getClients();
-			const pokers = usePokers ?? startPoke(clients, cvr.version);
+			const pokers = usePokers ?? startPoke(lc, clients, cvr.version);
 			span.setAttribute("numClients", clients.length);
 			const catchupFrom = clients.map((c) => c.version()).reduce((a, b) => cmpVersions(a, b) < 0 ? a : b, cvr.version);
 			const rowPatches = this.#cvrStore.catchupRowPatches(lc, catchupFrom, cvr, current, excludeQueryHashes);
@@ -1156,7 +1625,9 @@ var ViewSyncerService = class {
 	*
 	* Must be called from within the #lock.
 	*
-	* Returns false if the advancement failed due to a schema change.
+	* Returns 'success' if changes were successfully processed and poked,
+	* or a `ResetPipelinesSignal` if advancement aborted (e.g. schema change,
+	* timeout) and pipelines need to be reset and rehydrated.
 	*/
 	#advancePipelines(lc, cvr) {
 		return startAsyncSpan(tracer, "vs.#advancePipelines", async (span) => {
@@ -1164,21 +1635,29 @@ var ViewSyncerService = class {
 			assert(this.#pipelines.initialized(), "pipelines must be initialized (advancePipelines");
 			const start = performance.now();
 			const timer = new TimeSliceTimer(lc);
-			const { version, numChanges, changes } = this.#pipelines.advance(timer);
-			lc = lc.withContext("newVersion", version);
-			const updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, version, this.#pipelines.replicaVersion, (queryID) => this.#pipelines.rowSetSignature(queryID));
-			const pokers = startPoke(this.#getClients(cvr.version), updater.updatedVersion());
-			lc.debug?.(`applying ${numChanges} to advance to ${version}`);
+			let pokers;
+			let updater;
+			let version;
+			let numChanges = 0;
 			try {
-				await this.#processChanges(lc, await timer.start(), changes, updater, pokers);
+				const advancement = this.#pipelines.advance(timer);
+				version = advancement.version;
+				numChanges = advancement.numChanges;
+				lc = lc.withContext("newVersion", version);
+				updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, version, this.#pipelines.replicaVersion, (queryID) => this.#pipelines.rowSetSignature(queryID));
+				pokers = startPoke(lc, this.#getClients(cvr.version), updater.updatedVersion());
+				lc.debug?.(`applying ${numChanges} to advance to ${version}`);
+				await this.#processChanges(lc, await timer.start(), advancement.changes, updater, pokers);
 			} catch (e) {
 				if (e instanceof ResetPipelinesSignal) {
-					await pokers.cancel();
+					await pokers?.cancel();
+					this.#cvrStore.discardPending();
 					return e;
 				}
 				throw e;
 			}
-			this.#cvr = await this.#flushUpdater(lc, updater);
+			assert(updater && pokers && version !== void 0, "advancement state missing");
+			this.#cvr = await this.#flushPoked(lc, updater, pokers);
 			const finalVersion = this.#cvr.version;
 			await startAsyncSpan(tracer, "vs.#advancePipelines.pokeEnd", () => pokers.end(finalVersion));
 			this.#markVersionServed(version);
@@ -1198,7 +1677,7 @@ var ViewSyncerService = class {
 			clientID,
 			wsID: client.wsID
 		});
-		return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this.#cvrStore, this.#config, connCtx);
+		return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this, this.#cvrStore, this.#config, connCtx);
 	};
 	async #runBackgroundRetransform(lc) {
 		const attemptRetransform = async (connCtx) => {
@@ -1285,15 +1764,18 @@ var ViewSyncerService = class {
 		return this.#stopped.promise;
 	}
 	async #cleanup(err) {
+		this.#shuttingDown = true;
 		this.connContextManager.setSharedRetransformReady(false);
 		this.#stopTTLClockInterval();
 		this.#stopExpireTimer();
 		this.#stopAuthMaintenanceTimer();
+		this.#stopShutdownTimer();
 		this.#customQueryTransformer?.destroy();
 		for (const client of this.#clients.values()) if (err) client.fail(err);
 		else client.close(`closed clientGroupID=${this.id}`);
 		await this.#lock.withLock(() => {});
 		this.#pipelines.destroy();
+		this.#inspectorDelegate.clearAuthenticated(this.id, this);
 	}
 	/**
 	* Test helper: Manually mark initialization as complete.
@@ -1329,8 +1811,10 @@ function isTransformFailedError(error) {
 	return error.errorBody.kind === "TransformFailed" && !isAuthErrorBody(error.errorBody);
 }
 /**
-* A query must be expired for all clients in order to be considered
-* expired.
+* Whether a query's TTL has elapsed. A query must be expired for all clients
+* in order to be considered expired.
+*
+* Exported for testing.
 */
 function expired(ttlClock, q) {
 	if (q.type === "internal") return false;