@rocicorp/zero 1.11.0-canary.2 → 1.11.0-canary.4

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -10,7 +10,7 @@ import { manualSpan, startAsyncSpan, startSpan } from "../../../../otel/src/span
 import { ZERO_VERSION_COLUMN_NAME } from "../replicator/schema/constants.js";
 import { transformAndHashQuery } from "../../auth/read-authorizer.js";
 import { randInt } from "../../../../shared/src/rand.js";
-import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateNativeHistogram, getOrCreateUpDownCounter } from "../../observability/metrics.js";
+import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateNativeHistogram, getOrCreateUpDownCounter, getOrCreateValueHistogram } from "../../observability/metrics.js";
 import { rowIDString } from "../../types/row-key.js";
 import "../replicator/schema/replication-state.js";
 import { parseSignature } from "./row-set-signature.js";
@@ -25,8 +25,9 @@ import { isAuthErrorBody } from "../../auth/auth.js";
 import { ClientHandler, startPoke } from "./client-handler.js";
 import { tracer } from "./tracer.js";
 import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
-import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, nextEvictionTime } from "./cvr.js";
+import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, classifyQueriesForHydration, nextEvictionTime } from "./cvr.js";
 import { E2EServingLagTracker } from "./e2e-serving-lag.js";
+import { HydrationBudget } from "./hydration-budget.js";
 import { handleInspect } from "./inspect-handler.js";
 import { QueryCoveringIndex } from "./query-covering.js";
 import { resolver } from "@rocicorp/resolver";
@@ -90,12 +91,27 @@ var ViewSyncerService = class {
 	#stopped = resolver();
 	#initialized = resolver();
 	#cvr;
-	#pipelinesSynced = false;
+	/**
+	* Indicates whether the query pipelines have completed initial catch-up and
+	* hydration with the CVR snapshot, and are ready for steady-state operation.
+	*
+	* - When `false`: The syncer is in its initial catch-up phase. The replica
+	*   advances without diffs until it reaches `cvr.version.stateVersion`,
+	*   after which `#maybeHydratePipelines` hydrates queries.
+	* - When `true`: Pipelines are fully populated and in steady-state; replica
+	*   changes advance incrementally via `#advancePipelines`, and client query
+	*   updates are processed via `#syncQueryPipelineSet`.
+	*
+	* Resets to `false` if `#advancePipelines` returns a `ResetPipelinesSignal`
+	* and pipelines must be reset and rehydrated.
+	*/
+	#pipelinesHydrated = false;
 	#servedVersion = null;
 	#e2eServingLagTracker = new E2EServingLagTracker();
 	#expiredQueriesTimer = 0;
 	#authMaintenanceTimer = 0;
 	#setTimeout;
+	#now;
 	#customQueryTransformer;
 	#queryReplacements = /* @__PURE__ */ new Map();
 	#activeClients = getOrCreateUpDownCounter("sync", "active-clients", "Number of active sync clients");
@@ -122,10 +138,56 @@ var ViewSyncerService = class {
 	#pipelineResets = getOrCreateCounter("sync", "pipeline-resets", "Number of pipeline resets");
 	#rowSetSignatureDrifts = getOrCreateCounter("sync", "query.row-set-signature-drifts", "Number of times re-hydration of an unchanged query produced a different row-set signature than what is stored in the CVR (forcing a configVersion bump and full re-execution). Expected to be near-zero in steady state; persistent non-zero values indicate non-deterministic query execution (e.g. Cap operator picking different N-row subsets).");
 	#sameHashRehydrationVersionBumps = getOrCreateCounter("sync", "query.same-hash-rehydrations-forced-bump", "Number of times query-set reconciliation forced a configVersion bump for already-gotten same-transformation-hash query rehydration because trackQueries would not otherwise bump. Expected to be near-zero; non-zero values indicate pipeline/CVR row-set drift reached query-set reconciliation.");
+	#hydrationBudgetExhaustions = getOrCreateCounter("sync", "hydration_budget_exhaustions", {
+		description: "Number of hydration passes that exhaust their budget.",
+		unit: "{pass}"
+	});
+	#hydrationBudgetEvictions = getOrCreateCounter("sync", "hydration_budget_evictions", {
+		description: "Number of inactive queries removed after hydration budget exhaustion.",
+		unit: "{query}"
+	});
+	#hydrationBudgetElapsed = getOrCreateValueHistogram("sync", "hydration_budget_elapsed", {
+		description: "Elapsed milliseconds when optional query hydration stopped.",
+		unit: "ms",
+		bucketBoundaries: [
+			1,
+			2,
+			5,
+			10,
+			20,
+			50,
+			100,
+			200,
+			500,
+			1e3,
+			5e3
+		]
+	});
+	#hydrationBudgetOvershoot = getOrCreateValueHistogram("sync", "hydration_budget_overshoot", {
+		description: "Milliseconds elapsed beyond the configured hydration budget.",
+		unit: "ms",
+		bucketBoundaries: [
+			0,
+			1,
+			2,
+			5,
+			10,
+			20,
+			50,
+			100,
+			200,
+			500,
+			1e3
+		]
+	});
+	#queryEvictions = getOrCreateCounter("sync", "query_evictions", {
+		description: "Number of inactive queries evicted, grouped by reason.",
+		unit: "{query}"
+	});
 	#inspectorDelegate;
 	#config;
 	#runPriorityOp;
-	constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis)) {
+	constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis), now = performance.now.bind(performance)) {
 		this.#config = config;
 		this.id = clientGroupID;
 		this.connContextManager = connContextManager;
@@ -140,6 +202,7 @@ var ViewSyncerService = class {
 		this.#customQueryTransformer = customQueryTransformer;
 		this.#cvrStore = new CVRStore(lc, cvrDb, shard, taskID, clientGroupID, () => this.#stateChanges.cancel());
 		this.#setTimeout = setTimeoutFn;
+		this.#now = now;
 		this.#runPriorityOp = runPriorityOp;
 		this.keepalive();
 	}
@@ -213,26 +276,18 @@ var ViewSyncerService = class {
 						lc.info?.(`resetting CVR: ${message}`);
 						throw new ClientNotFoundError(message);
 					}
-					if (this.#pipelinesSynced) {
+					if (this.#pipelinesHydrated) {
 						const result = await this.#advancePipelines(lc, cvr);
 						if (result === "success") return;
 						lc.info?.(`resetting pipelines: ${result.message}`);
 						this.#pipelineResets.add(1, { reason: result.reason });
 						this.#pipelines.reset(clientSchema);
-						this.#pipelinesSynced = false;
+						this.#pipelinesHydrated = false;
 						this.connContextManager.setSharedRetransformReady(false);
 					}
-					const version = this.#pipelines.advanceWithoutDiff();
-					const cvrVer = versionString(cvr.version);
-					if (version < cvr.version.stateVersion) {
-						lc.debug?.(`replica@${version} is behind cvr@${cvrVer}`);
-						return;
-					}
-					lc.info?.(`init pipelines@${version} (cvr@${cvrVer})`);
-					const driftedQueryIDs = await this.#hydrateUnchangedQueries(lc, cvr);
-					await this.#syncQueryPipelineSet(lc, cvr, "missing", void 0, driftedQueryIDs);
-					this.#pipelinesSynced = true;
-					this.connContextManager.setSharedRetransformReady(true);
+					const backgroundConnCtx = this.connContextManager.getBackgroundConnectionContext();
+					if (backgroundConnCtx) await this.#maybeHydratePipelines(lc, cvr, backgroundConnCtx);
+					else lc.info?.("No validated background connection; deferring pipeline init");
 				});
 			}
 			if (this.#drainCoordinator.shouldDrain()) this.#drainCoordinator.drainNextIn(this.#totalHydrationTimeMs());
@@ -246,11 +301,57 @@ var ViewSyncerService = class {
 			this.#stopped.resolve();
 		}
 	}
+	/**
+	* Ensures that query pipelines are hydrated and ready for steady-state
+	* operation.
+	*
+	* Preconditions:
+	* 1. Must be called from within the `#lock`.
+	* 2. The caller must provide an active, validated `ConnectionContext` for
+	*    evaluating custom query transforms and permissions.
+	*
+	* Readiness prerequisites:
+	* - The underlying `QueryPipelineDriver` must be initialized with the schema
+	*   (i.e. `this.#pipelines.initialized()` is true).
+	* - The SQLite replica must have caught up to at least the CVR's
+	*   `stateVersion` (`version >= cvr.version.stateVersion`).
+	*
+	* If either prerequisite is not met, this method returns early as a no-op,
+	* waiting for the replica to catch up or the driver to initialize.
+	*
+	* Once ready:
+	* - Hydrates unchanged queries from the CVR snapshot without full row diffing.
+	* - Syncs missing, errored, or drifted queries via `#syncQueryPipelineSet`.
+	* - Sets `#pipelinesHydrated = true` and enables shared retransformations.
+	*/
+	async #maybeHydratePipelines(lc, cvr, connCtx) {
+		if (!this.#pipelines.initialized()) return;
+		const version = this.#pipelines.advanceWithoutDiff();
+		const cvrVer = versionString(cvr.version);
+		if (version < cvr.version.stateVersion) {
+			lc.debug?.(`replica@${version} is behind cvr@${cvrVer}`);
+			return;
+		}
+		lc.info?.(`init pipelines@${version} (cvr@${cvrVer})`);
+		const hydrationBudget = new HydrationBudget(this.#config.viewSyncerHydrationBudgetMs ?? 0, this.#now);
+		const hydrationPassStats = {
+			activeHydratedQueries: 0,
+			inactiveHydratedQueries: 0
+		};
+		const driftedQueryIDs = await this.#hydrateUnchangedQueries(lc, cvr, connCtx, hydrationPassStats, hydrationBudget);
+		await this.#syncQueryPipelineSet(lc, cvr, "missing", connCtx, driftedQueryIDs, hydrationBudget, hydrationPassStats);
+		this.#pipelinesHydrated = true;
+		this.connContextManager.setSharedRetransformReady(true);
+	}
 	#removeExpiredQueries = async (lc, cvr) => {
 		if (hasExpiredQueries(cvr)) {
 			lc = lc.withContext("method", "#removeExpiredQueries");
 			lc.debug?.("Queries have expired");
-			if (this.#pipelinesSynced) await this.#syncQueryPipelineSet(lc, cvr, "missing", void 0);
+			if (this.#pipelinesHydrated) {
+				const connCtx = this.connContextManager.getBackgroundConnectionContext();
+				if (connCtx) await this.#syncQueryPipelineSet(lc, cvr, "missing", connCtx);
+				else lc.info?.("No validated background connection to remove expired queries; deferring");
+			}
 		}
 		this.#scheduleExpireEviction(lc, cvr);
 	};
@@ -466,7 +567,7 @@ var ViewSyncerService = class {
 			}
 			lc.debug?.("Auth changed, re-validating and re-transforming queries");
 			const connCtx = this.connContextManager.mustGetConnectionContext(selector);
-			if (!this.#pipelinesSynced) {
+			if (!this.#pipelinesHydrated) {
 				if (!await this.#validateConnection(connCtx)) return;
 			}
 			return await this.#handleConfigUpdate(lc, clientID, {}, cvr, "all", void 0, connCtx);
@@ -528,7 +629,8 @@ var ViewSyncerService = class {
 				await pokers.end(newCVR.version);
 			});
 		}
-		if (this.#pipelinesSynced) await this.#syncQueryPipelineSet(lc, this.#cvr, customQueryTransformMode, connCtx);
+		if (!this.#pipelinesHydrated) await this.#maybeHydratePipelines(lc, this.#cvr, connCtx);
+		else await this.#syncQueryPipelineSet(lc, this.#cvr, customQueryTransformMode, connCtx);
 		return this.#cvr;
 	}
 	/**
@@ -651,7 +753,7 @@ var ViewSyncerService = class {
 	*
 	* This must be called from within the #lock.
 	*/
-	async #hydrateUnchangedQueries(lc, cvr) {
+	async #hydrateUnchangedQueries(lc, cvr, connCtx, hydrationPassStats, hydrationBudget) {
 		assert(this.#pipelines.initialized(), "pipelines must be initialized");
 		const dbVersion = this.#pipelines.currentVersion();
 		const cvrVersion = cvr.version;
@@ -660,33 +762,27 @@ var ViewSyncerService = class {
 			return /* @__PURE__ */ new Set();
 		}
 		const gotQueries = Object.entries(cvr.queries).filter(([_, state]) => state.transformationHash !== void 0);
+		const { required: requiredGotQueries } = classifyQueriesForHydration(gotQueries.map(([, query]) => query));
 		const customQueries = /* @__PURE__ */ new Map();
 		const otherQueries = [];
-		let inactivatedCount = 0;
-		for (const [, query] of gotQueries) {
-			if (query.type !== "internal" && Object.values(query.clientState).every(({ inactivatedAt }) => inactivatedAt !== void 0)) {
-				inactivatedCount++;
-				continue;
-			}
-			if (query.type === "custom") customQueries.set(query.id, query);
-			else otherQueries.push(query);
-		}
+		const inactivatedCount = gotQueries.length - requiredGotQueries.length;
+		for (const query of requiredGotQueries) if (query.type === "custom") customQueries.set(query.id, query);
+		else otherQueries.push(query);
 		const transformedQueries = [];
 		let customErrorCount = 0;
 		let customHashMismatchCount = 0;
 		let otherHashMismatchCount = 0;
 		if (customQueries.size > 0 && !this.#customQueryTransformer) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
-		const backgroundContext = this.connContextManager.mustGetBackgroundConnectionContext();
 		const customQueryTransformer = this.#customQueryTransformer;
 		if (customQueryTransformer && customQueries.size > 0) {
-			const transformedCustomQueries = await this.#runPriorityOp(lc, "#hydrateUnchangedQueries transforming custom queries", () => customQueryTransformer.transform(backgroundContext, customQueries.values()));
-			if (transformedCustomQueries.kind === "success" && !transformedCustomQueries.cached) this.connContextManager.validateConnection(backgroundContext, backgroundContext.revision, transformedCustomQueries.validation);
+			const transformedCustomQueries = await hydrationBudget.excluding(() => this.#runPriorityOp(lc, "#hydrateUnchangedQueries transforming custom queries", () => customQueryTransformer.transform(connCtx, customQueries.values())));
+			if (transformedCustomQueries.kind === "success" && !transformedCustomQueries.cached) this.connContextManager.validateConnection(connCtx, connCtx.revision, transformedCustomQueries.validation);
 			if (Array.isArray(transformedCustomQueries.result)) for (const q of transformedCustomQueries.result) if ("error" in q) customErrorCount++;
 			else if (q.transformationHash !== customQueries.get(q.id)?.transformationHash) customHashMismatchCount++;
 			else transformedQueries.push(q);
 		}
 		for (const q of otherQueries) {
-			const transformed = transformAndHashQuery(lc, q.id, q.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, backgroundContext.auth?.type === "jwt" ? backgroundContext.auth : void 0, q.type === "internal");
+			const transformed = transformAndHashQuery(lc, q.id, q.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, connCtx.auth?.type === "jwt" ? connCtx.auth : void 0, q.type === "internal");
 			if (transformed.transformationHash === q.transformationHash) transformedQueries.push(transformed);
 			else otherHashMismatchCount++;
 		}
@@ -716,6 +812,7 @@ var ViewSyncerService = class {
 				else count++;
 			});
 			const elapsed = timer.totalElapsed();
+			hydrationPassStats.activeHydratedQueries++;
 			this.#hydrations.add(1);
 			this.#hydrationTime.recordMs(elapsed);
 			this.#addQueryMaterializationServerMetric(transformationHash, elapsed);
@@ -820,7 +917,10 @@ var ViewSyncerService = class {
 	*
 	* This must be called from within the #lock.
 	*/
-	#syncQueryPipelineSet(lc, cvr, customQueryTransformMode, connCtx, driftedQueryIDs = /* @__PURE__ */ new Set()) {
+	#syncQueryPipelineSet(lc, cvr, customQueryTransformMode, connCtx, driftedQueryIDs = /* @__PURE__ */ new Set(), hydrationBudget = new HydrationBudget(this.#config.viewSyncerHydrationBudgetMs ?? 0, this.#now), hydrationPassStats = {
+		activeHydratedQueries: 0,
+		inactiveHydratedQueries: 0
+	}) {
 		return startAsyncSpan(tracer, "vs.#syncQueryPipelineSet", async (span) => {
 			const start = performance.now();
 			span.setAttribute("clientGroupID", this.id);
@@ -829,38 +929,39 @@ var ViewSyncerService = class {
 			const now = Date.now();
 			const ttlClock = this.#getTTLClock(now);
 			const cvrQueryEntires = Object.entries(cvr.queries);
-			const customQueries = /* @__PURE__ */ new Map();
-			const otherQueries = [];
 			const transformedQueries = [];
-			const resolvedConnCtx = connCtx ?? this.connContextManager.mustGetBackgroundConnectionContext();
-			for (const [id, query] of cvrQueryEntires) if (query.type === "custom") {
-				assert(id === query.id, "custom query id mismatch");
-				customQueries.set(id, query);
-			} else otherQueries.push({
-				id,
-				query
-			});
-			for (const { id, query: origQuery } of otherQueries) {
-				assert(id === origQuery.id, "query id mismatch");
-				const transformed = transformAndHashQuery(lc, origQuery.id, origQuery.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, resolvedConnCtx.auth?.type === "jwt" ? resolvedConnCtx.auth : void 0, origQuery.type === "internal");
-				transformedQueries.push({
-					id,
-					origQuery,
-					transformed
-				});
-			}
-			if (customQueries.size > 0 && !this.#customQueryTransformer) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
-			let erroredQueryIDs;
-			const customQueriesToTransform = customQueryTransformMode === "all" ? [...customQueries.values()] : customQueryTransformMode && [...customQueries.values()].filter((q) => !this.#pipelines.queries().has(q.id));
+			const naturallyExpiredQueryIDs = new Set(cvrQueryEntires.map(([, query]) => query).filter((query) => expired(ttlClock, query)).map((query) => query.id));
+			if (naturallyExpiredQueryIDs.size > 0) this.#queryEvictions.add(naturallyExpiredQueryIDs.size, { reason: "ttl" });
+			const hydrationCandidates = cvrQueryEntires.map(([, query]) => query).filter((query) => !naturallyExpiredQueryIDs.has(query.id));
+			const { required, optional } = classifyQueriesForHydration(hydrationCandidates);
+			const requiredQueryIDs = new Set(required.map((query) => query.id));
+			const optionalToHydrate = hydrationBudget.limitMs === 0 ? optional : optional.filter((query) => query.patchVersion !== void 0);
+			if (!this.#customQueryTransformer && hydrationCandidates.some((query) => query.type === "custom")) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
 			const customQueryTransformer = this.#customQueryTransformer;
-			if (customQueryTransformer && customQueriesToTransform.length > 0) {
+			const erroredQueryIDs = [];
+			let transformedCustomQueryCount = 0;
+			const transformOtherQueries = (queries) => {
+				for (const origQuery of queries) {
+					if (origQuery.type === "custom") continue;
+					const transformed = transformAndHashQuery(lc, origQuery.id, origQuery.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, connCtx.auth?.type === "jwt" ? connCtx.auth : void 0, origQuery.type === "internal");
+					transformedQueries.push({
+						id: origQuery.id,
+						origQuery,
+						transformed
+					});
+				}
+			};
+			const shouldTransformCustomQuery = (query) => customQueryTransformMode === "all" || customQueryTransformMode && !this.#pipelines.queries().has(query.id);
+			const transformCustomQueries = async (queries) => {
+				if (!customQueryTransformer || queries.length === 0) return;
+				transformedCustomQueryCount += queries.length;
 				const transformStart = performance.now();
 				let transformedCustomQueries;
 				try {
-					transformedCustomQueries = await this.#runPriorityOp(lc, "#syncQueryPipelineSet transforming custom queries", () => customQueryTransformer.transform(resolvedConnCtx, customQueriesToTransform));
+					transformedCustomQueries = await hydrationBudget.excluding(() => this.#runPriorityOp(lc, "#syncQueryPipelineSet transforming custom queries", () => customQueryTransformer.transform(connCtx, queries)));
 					if (transformedCustomQueries.kind === "failed") throw new ProtocolErrorWithLevel(transformedCustomQueries.result, "warn");
 					else {
-						if (!transformedCustomQueries.cached) this.connContextManager.validateConnection(resolvedConnCtx, resolvedConnCtx.revision, transformedCustomQueries.validation);
+						if (!transformedCustomQueries.cached) this.connContextManager.validateConnection(connCtx, connCtx.revision, transformedCustomQueries.validation);
 						this.#queryTransformations.add(1, { result: "success" });
 					}
 				} catch (e) {
@@ -871,8 +972,9 @@ var ViewSyncerService = class {
 					this.#queryTransformationTime.recordMs(transformDuration);
 				}
 				const successfullyTransformedCustomQueries = /* @__PURE__ */ new Map();
-				erroredQueryIDs = this.#processTransformedCustomQueries(lc, transformedCustomQueries.result, (q) => {
-					const origQuery = customQueries.get(q.id);
+				const customQueryMap = new Map(queries.map((query) => [query.id, query]));
+				erroredQueryIDs.push(...this.#processTransformedCustomQueries(lc, transformedCustomQueries.result, (q) => {
+					const origQuery = customQueryMap.get(q.id);
 					if (origQuery) {
 						successfullyTransformedCustomQueries.set(q.id, q);
 						transformedQueries.push({
@@ -881,7 +983,7 @@ var ViewSyncerService = class {
 							transformed: q
 						});
 					}
-				}, customQueries);
+				}, customQueryMap));
 				for (const [queryID, newTransform] of successfullyTransformedCustomQueries) {
 					const existingTransformHash = cvr.queries[queryID]?.transformationHash;
 					if (existingTransformHash) {
@@ -894,21 +996,28 @@ var ViewSyncerService = class {
 						} else this.#queryTransformationNoOps.add(1);
 					}
 				}
-			}
-			const removeQueriesQueryIds = new Set([...Object.values(cvr.queries).filter((q) => expired(ttlClock, q)).map((q) => q.id), ...erroredQueryIDs || []]);
+			};
+			const queriesToTransform = [...required, ...optionalToHydrate];
+			transformOtherQueries(queriesToTransform);
+			await transformCustomQueries(queriesToTransform.filter((query) => query.type === "custom" && shouldTransformCustomQuery(query)));
+			const removeQueriesQueryIds = new Set([...naturallyExpiredQueryIDs, ...erroredQueryIDs]);
 			const addQueries = transformedQueries.map(({ id, origQuery, transformed }) => ({
 				id,
 				ast: transformed.transformedAst,
 				transformationHash: transformed.transformationHash,
 				name: origQuery.type === "custom" ? origQuery.name : void 0
 			})).filter((q) => !removeQueriesQueryIds.has(q.id) && this.#pipelines.queries().get(q.id)?.transformationHash !== q.transformationHash);
-			lc.info?.(`syncQueryPipelineSet: ${cvrQueryEntires.length} CVR queries, ${customQueriesToTransform.length} custom re-transformed, ${erroredQueryIDs?.length ?? 0} errored, ${removeQueriesQueryIds.size} to remove, ${addQueries.length} to add`);
+			const hydrationOrder = new Map([...required, ...optional].map((query, index) => [query.id, index]));
+			addQueries.sort((a, b) => must(hydrationOrder.get(a.id)) - must(hydrationOrder.get(b.id)));
+			const requiredAddQueries = addQueries.filter((query) => requiredQueryIDs.has(query.id));
+			const optionalAddQueries = addQueries.filter((query) => !requiredQueryIDs.has(query.id));
+			lc.info?.(`syncQueryPipelineSet: ${cvrQueryEntires.length} CVR queries, ${transformedCustomQueryCount} custom re-transformed, ${erroredQueryIDs.length} errored, ${removeQueriesQueryIds.size} to remove, ${addQueries.length} to add`);
 			for (const q of addQueries) {
 				const orig = cvr.queries[q.id];
 				lc.debug?.("ViewSyncer adding query", q.ast, "transformed from", orig.type === "custom" ? orig.name : orig.ast);
 			}
 			if (addQueries.length > 0 || removeQueriesQueryIds.size > 0) {
-				await this.#addAndRemoveQueries(lc, cvr, addQueries, Array.from(removeQueriesQueryIds, (id) => ({ id })), driftedQueryIDs);
+				await this.#addAndRemoveQueries(lc, cvr, requiredAddQueries, optionalAddQueries, Array.from(removeQueriesQueryIds, (id) => ({ id })), hydrationBudget, hydrationPassStats, driftedQueryIDs);
 				if (addQueries.length > 0) this.#viewSyncerHydration.recordMs(performance.now() - start);
 			} else await this.#catchupClients(lc, cvr);
 		});
@@ -941,8 +1050,27 @@ var ViewSyncerService = class {
 		record.count++;
 		if (record.count >= THRASH_THRESHOLD) this.#lc.warn?.(`Query thrashing detected for query ${queryID}. ${record.count} replacements in 60s. This may indicate clients with different auth contexts connecting to the same client group.`);
 	}
-	#addAndRemoveQueries(lc, cvr, addQueries, removeQueries, driftedQueryIDs = /* @__PURE__ */ new Set()) {
+	#recordHydrationBudgetExhaustion(lc, hydrationBudget, activeHydratedQueries, inactiveHydratedQueries, inactiveEvictedQueryIDs) {
+		const elapsedMs = hydrationBudget.exhaustedAtMs;
+		if (elapsedMs === void 0) return;
+		this.#hydrationBudgetExhaustions.add(1);
+		this.#hydrationBudgetEvictions.add(inactiveEvictedQueryIDs.length);
+		this.#hydrationBudgetElapsed.record(elapsedMs);
+		this.#hydrationBudgetOvershoot.record(Math.max(0, elapsedMs - hydrationBudget.limitMs));
+		if (inactiveEvictedQueryIDs.length > 0) this.#queryEvictions.add(inactiveEvictedQueryIDs.length, { reason: "hydration-budget" });
+		lc.info?.("view-syncer hydration budget exhausted", {
+			clientGroupID: this.id,
+			hydrationBudgetMs: hydrationBudget.limitMs,
+			hydrationElapsedMs: elapsedMs,
+			activeHydratedQueries,
+			inactiveHydratedQueries,
+			inactiveEvictedQueries: inactiveEvictedQueryIDs.length,
+			firstEvictedQueryHash: inactiveEvictedQueryIDs[0] ?? null
+		});
+	}
+	#addAndRemoveQueries(lc, cvr, requiredQueries, optionalQueries, removeQueries, hydrationBudget, hydrationPassStats, driftedQueryIDs = /* @__PURE__ */ new Set()) {
 		return startAsyncSpan(tracer, "vs.#addAndRemoveQueries", async () => {
+			const addQueries = [...requiredQueries, ...optionalQueries];
 			assert(addQueries.length > 0 || removeQueries.length > 0, "Must have queries to add or remove");
 			const start = performance.now();
 			const stateVersion = this.#pipelines.currentVersion();
@@ -959,6 +1087,7 @@ var ViewSyncerService = class {
 				updater.ensureNewVersion();
 			}
 			const { queryPatches, newVersion } = updater.trackQueries(lc, addQueries, removeQueries);
+			if (hydrationBudget.limitMs !== 0 && optionalQueries.length > 0) assert(cmpVersions(cvr.version, newVersion) < 0, "Optional hydration requires a final poke version before row processing so a removal is actually poked");
 			const pokers = startPoke(this.#getClients(), newVersion);
 			for (const patch of queryPatches) await pokers.addPatch(patch);
 			for (const q of removeQueries) {
@@ -975,10 +1104,17 @@ var ViewSyncerService = class {
 			let totalHydratedQueries = 0;
 			let coveredHydratedQueries = 0;
 			let firstCoveredQuery;
+			const hydratedQueryIDs = [];
+			const budgetEvictedQueryIDs = [];
 			const self = this;
 			await yieldProcess(lc);
-			function* generateRowChanges(slowHydrateThreshold) {
-				for (const q of addQueries) {
+			function* hydrateQueries(queries, optional, slowHydrateThreshold) {
+				for (let i = 0; i < queries.length; i++) {
+					if (optional && hydrationBudget.exhausted()) {
+						budgetEvictedQueryIDs.push(...queries.slice(i).map((query) => query.id));
+						return;
+					}
+					const q = must(queries[i]);
 					let queryLC = lc.withContext("hash", q.id).withContext("queryHash", q.id).withContext("transformationHash", q.transformationHash);
 					if (q.name !== void 0) queryLC = queryLC.withContext("queryName", q.name);
 					queryLC.debug?.(`adding pipeline for query`, q.ast);
@@ -991,6 +1127,9 @@ var ViewSyncerService = class {
 					yield* pipelines.addQuery(q.transformationHash, q.id, q.ast, timer.startWithoutYielding(), q.name, "query-set-sync");
 					const elapsed = timer.stop();
 					totalProcessTime += elapsed;
+					hydratedQueryIDs.push(q.id);
+					if (optional) hydrationPassStats.inactiveHydratedQueries++;
+					else hydrationPassStats.activeHydratedQueries++;
 					self.#addQueryMaterializationServerMetric(q.id, elapsed);
 					self.#inspectorDelegate.addQuery(q.id, q.ast);
 					queryCoveringIndex?.add(q.id, {
@@ -1004,18 +1143,30 @@ var ViewSyncerService = class {
 						transformationHash: q.transformationHash,
 						...q.name !== void 0 && { name: q.name }
 					});
+					hydrations.add(1);
+					hydrationTime.recordMs(elapsed);
 				}
-				hydrations.add(1);
-				hydrationTime.recordMs(totalProcessTime);
+			}
+			function* generateRowChanges(slowHydrateThreshold) {
+				yield* hydrateQueries(requiredQueries, false, slowHydrateThreshold);
+				yield* hydrateQueries(optionalQueries, true, slowHydrateThreshold);
 			}
 			await this.#processChanges(lc, timer, generateRowChanges(this.#slowHydrateThreshold), updater, pokers);
+			for (const patch of updater.removeTrackedQueries(budgetEvictedQueryIDs)) await pokers.addPatch(patch);
+			for (const queryID of budgetEvictedQueryIDs) {
+				this.#pipelines.removeQuery(queryID);
+				this.#inspectorDelegate.removeQuery(queryID);
+				this.#queryReplacements.delete(queryID);
+			}
+			this.#recordHydrationBudgetExhaustion(lc, hydrationBudget, hydrationPassStats.activeHydratedQueries, hydrationPassStats.inactiveHydratedQueries, budgetEvictedQueryIDs);
 			this.#logQueryCoverageShadowSummary(lc, "add", totalHydratedQueries, coveredHydratedQueries, firstCoveredQuery);
 			await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.deleteUnreferencedRows", async () => {
 				for (const patch of await updater.deleteUnreferencedRows(lc)) await pokers.addPatch(patch);
 			});
 			this.#cvr = await this.#flushUpdater(lc, updater);
+			if (budgetEvictedQueryIDs.length > 0) this.#scheduleExpireEviction(lc, this.#cvr);
 			const finalVersion = this.#cvr.version;
-			await this.#catchupClients(lc, cvr, finalVersion, addQueries.map((q) => q.id), pokers);
+			await this.#catchupClients(lc, cvr, finalVersion, hydratedQueryIDs, pokers);
 			await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.pokeEnd", () => pokers.end(finalVersion));
 			this.#markVersionServed(stateVersion);
 			const wallTime = performance.now() - start;
@@ -1156,7 +1307,9 @@ var ViewSyncerService = class {
 	*
 	* Must be called from within the #lock.
 	*
-	* Returns false if the advancement failed due to a schema change.
+	* Returns 'success' if changes were successfully processed and poked,
+	* or a `ResetPipelinesSignal` if advancement aborted (e.g. schema change,
+	* timeout) and pipelines need to be reset and rehydrated.
 	*/
 	#advancePipelines(lc, cvr) {
 		return startAsyncSpan(tracer, "vs.#advancePipelines", async (span) => {
@@ -1329,8 +1482,10 @@ function isTransformFailedError(error) {
 	return error.errorBody.kind === "TransformFailed" && !isAuthErrorBody(error.errorBody);
 }
 /**
-* A query must be expired for all clients in order to be considered
-* expired.
+* Whether a query's TTL has elapsed. A query must be expired for all clients
+* in order to be considered expired.
+*
+* Exported for testing.
 */
 function expired(ttlClock, q) {
 	if (q.type === "internal") return false;