@rocicorp/zero 1.4.0 → 1.5.0-canary.1

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -11,6 +11,7 @@ import { transformAndHashQuery } from "../../auth/read-authorizer.js";
 import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateUpDownCounter } from "../../observability/metrics.js";
 import { rowIDString } from "../../types/row-key.js";
 import "../replicator/schema/replication-state.js";
+import { parseSignature } from "./row-set-signature.js";
 import { ResetPipelinesSignal } from "./snapshotter.js";
 import "./pipeline-driver.js";
 import { randInt } from "../../../../shared/src/rand.js";
@@ -41,7 +42,7 @@ function randomID() {
 var TTL_CLOCK_INTERVAL = 6e4;
 var ViewSyncerService = class {
 	id;
-	contextManager;
+	connContextManager;
 	#shard;
 	#lc;
 	#pipelines;
@@ -94,13 +95,14 @@ var ViewSyncerService = class {
 	#queryTransformationNoOps = getOrCreateCounter("sync", "query.transformation-no-ops", "Number of times query transformation resulted in no-op (hash unchanged)");
 	#lockWaitTime = getOrCreateLatencyHistogram("sync", "lock-wait-time", "Time spent waiting to acquire the ViewSyncer lock.");
 	#pipelineResets = getOrCreateCounter("sync", "pipeline-resets", "Number of pipeline resets");
+	#rowSetSignatureDrifts = getOrCreateCounter("sync", "query.row-set-signature-drifts", "Number of times re-hydration of an unchanged query produced a different row-set signature than what is stored in the CVR (forcing a configVersion bump and full re-execution). Expected to be near-zero in steady state; persistent non-zero values indicate non-deterministic query execution (e.g. Cap operator picking different N-row subsets).");
 	#inspectorDelegate;
 	#config;
 	#runPriorityOp;
-	constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, contextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis)) {
+	constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis)) {
 		this.#config = config;
 		this.id = clientGroupID;
-		this.contextManager = contextManager;
+		this.connContextManager = connContextManager;
 		this.#shard = shard;
 		this.#lc = lc;
 		this.#pipelines = pipelineDriver;
@@ -189,6 +191,7 @@ var ViewSyncerService = class {
 						this.#pipelineResets.add(1, { reason: result.reason });
 						this.#pipelines.reset(clientSchema);
 						this.#pipelinesSynced = false;
+						this.connContextManager.setSharedRetransformReady(false);
 					}
 					const version = this.#pipelines.advanceWithoutDiff();
 					const cvrVer = versionString(cvr.version);
@@ -197,9 +200,10 @@ var ViewSyncerService = class {
 						return;
 					}
 					lc.info?.(`init pipelines@${version} (cvr@${cvrVer})`);
-					await this.#hydrateUnchangedQueries(lc, cvr);
-					await this.#syncQueryPipelineSet(lc, cvr, "missing", void 0);
+					const driftedQueryIDs = await this.#hydrateUnchangedQueries(lc, cvr);
+					await this.#syncQueryPipelineSet(lc, cvr, "missing", void 0, driftedQueryIDs);
 					this.#pipelinesSynced = true;
+					this.connContextManager.setSharedRetransformReady(true);
 				});
 			}
 			if (this.#drainCoordinator.shouldDrain()) this.#drainCoordinator.drainNextIn(this.#totalHydrationTimeMs());
@@ -262,7 +266,7 @@ var ViewSyncerService = class {
 		return this.#clients.size === 0;
 	}
 	#deleteClientDueToDisconnect(clientID, client) {
-		this.contextManager.closeConnection({
+		this.connContextManager.closeConnection({
 			clientID,
 			wsID: client.wsID
 		});
@@ -290,13 +294,10 @@ var ViewSyncerService = class {
 	* deadlines. The timer plumbing is intentionally separate from the actual
 	* revalidation/retransform work so future policy changes only need to update
 	* the maintenance workers, not the wakeup logic.
-	*
-	* This is intentionally cheap & idempotent, so it can be called frequently
-	* when upstream state might have changed.
 	*/
 	#scheduleAuthMaintenance(lc) {
 		this.#stopAuthMaintenanceTimer();
-		const plan = this.contextManager.planMaintenance();
+		const plan = this.connContextManager.planMaintenance();
 		if (plan.earliestDeadlineAt === void 0) {
 			lc.debug?.("No auth maintenance wakeup scheduled");
 			return;
@@ -306,13 +307,17 @@ var ViewSyncerService = class {
 			delay,
 			earliestDeadlineAt: plan.earliestDeadlineAt
 		});
-		this.#authMaintenanceTimer = this.#setTimeout(() => {
-			this.#authMaintenanceTimer = 0;
-			this.#runInLockWithCVR((lc, cvr) => this.#runAuthMaintenance(lc, cvr)).catch((e) => this.#stateChanges.fail(e));
+		this.#authMaintenanceTimer = this.#setTimeout(async () => {
+			try {
+				this.#authMaintenanceTimer = 0;
+				await this.#runInLockWithCVR((lc, cvr) => this.#runAuthMaintenance(lc, cvr));
+			} catch (e) {
+				this.#stateChanges.fail(e instanceof Error ? e : new Error(String(e)));
+			}
 		}, delay);
 	}
 	async #runAuthMaintenance(lc, _cvr) {
-		const plan = this.contextManager.planMaintenance();
+		const plan = this.connContextManager.planMaintenance();
 		if (plan.dueRevalidations.length === 0 && !plan.dueRetransform) {
 			lc.debug?.("Auth maintenance woke up with no due work");
 			return;
@@ -321,52 +326,52 @@ var ViewSyncerService = class {
 			dueRevalidations: plan.dueRevalidations.length,
 			dueRetransform: plan.dueRetransform
 		});
-		for (const connection of plan.dueRevalidations) try {
-			await this.#validateConnection(connection);
+		for (const connCtx of plan.dueRevalidations) try {
+			await this.#validateConnection(connCtx);
 		} catch (e) {
 			if (isProtocolError(e) && isTransformFailedError(e)) {
 				lc.warn?.("Scheduled auth revalidation failed; deferring auth maintenance", {
-					clientID: connection.clientID,
-					wsID: connection.wsID,
+					clientID: connCtx.clientID,
+					wsID: connCtx.wsID,
 					message: e.message
 				});
-				this.contextManager.deferMaintenance("revalidate");
+				this.connContextManager.deferMaintenance("revalidate");
 				return;
 			}
 			throw e;
 		}
-		if (this.contextManager.planMaintenance().dueRetransform) await this.#runBackgroundRetransform(lc);
+		if (this.connContextManager.planMaintenance().dueRetransform) await this.#runBackgroundRetransform(lc);
 	}
 	initConnection(selector, initConnectionMessage) {
 		this.#lc.debug?.("viewSyncer.initConnection");
 		return startSpan(tracer, "vs.initConnection", () => {
-			const ctx = this.contextManager.mustGetConnectionContext(selector);
-			const lc = this.#lc.withContext("clientID", ctx.clientID).withContext("wsID", ctx.wsID);
+			const connCtx = this.connContextManager.mustGetConnectionContext(selector);
+			const lc = this.#lc.withContext("clientID", connCtx.clientID).withContext("wsID", connCtx.wsID);
 			const downstream = Subscription.create({ cleanup: (_, err) => {
 				err ? lc[getLogLevel(err)]?.(`client closed with error`, err) : lc.info?.("client closed");
-				this.#deleteClientDueToDisconnect(ctx.clientID, newClient);
-				this.#activeClients.add(-1, { [PROTOCOL_VERSION_ATTR]: ctx.protocolVersion });
+				this.#deleteClientDueToDisconnect(connCtx.clientID, newClient);
+				this.#activeClients.add(-1, { [PROTOCOL_VERSION_ATTR]: connCtx.protocolVersion });
 			} });
-			this.#activeClients.add(1, { [PROTOCOL_VERSION_ATTR]: ctx.protocolVersion });
+			this.#activeClients.add(1, { [PROTOCOL_VERSION_ATTR]: connCtx.protocolVersion });
 			if (this.#clients.size === 0) this.#ttlClockBase = Date.now();
-			const newClient = new ClientHandler(lc, this.id, ctx.clientID, ctx.wsID, this.#shard, ctx.baseCookie, downstream);
-			this.#clients.get(ctx.clientID)?.close(`replaced by wsID: ${ctx.wsID}`);
-			this.#clients.set(ctx.clientID, newClient);
-			startAsyncSpan(tracer, "vs.initConnection.async", () => this.#runInLockForClient(ctx, initConnectionMessage, async (lc, clientID, msg, cvr) => {
+			const newClient = new ClientHandler(lc, this.id, connCtx.clientID, connCtx.wsID, this.#shard, connCtx.baseCookie, downstream);
+			this.#clients.get(connCtx.clientID)?.close(`replaced by wsID: ${connCtx.wsID}`);
+			this.#clients.set(connCtx.clientID, newClient);
+			startAsyncSpan(tracer, "vs.initConnection.async", () => this.#runInLockForClient(connCtx, initConnectionMessage, async (lc, clientID, msg, cvr) => {
 				if (cvr.clientSchema === null && !msg.clientSchema) throw new ProtocolErrorWithLevel({
 					kind: InvalidConnectionRequest,
 					message: "The initConnection message for a new client group must include client schema.",
 					origin: ZeroCache
 				}, "warn");
-				if (!await this.#validateConnection(ctx)) return;
-				await this.#handleConfigUpdate(lc, clientID, msg, cvr, "all", ctx.profileID ?? `cg${this.id}`, ctx);
+				if (!await this.#validateConnection(connCtx)) return;
+				await this.#handleConfigUpdate(lc, clientID, msg, cvr, "all", connCtx.profileID ?? `cg${this.id}`, connCtx);
 				this.#initialized.resolve("initialized");
 			}, newClient)).catch((e) => newClient.fail(e));
 			return downstream;
 		});
 	}
 	async changeDesiredQueries(selector, msg) {
-		await this.#runInLockForClient(selector, msg, (lc, clientID, msg, cvr) => this.#handleConfigUpdate(lc, clientID, msg, cvr, "missing", void 0, this.contextManager.mustGetConnectionContext(selector)));
+		await this.#runInLockForClient(selector, msg, (lc, clientID, msg, cvr) => this.#handleConfigUpdate(lc, clientID, msg, cvr, "missing", void 0, this.connContextManager.mustGetConnectionContext(selector)));
 	}
 	async updateAuth(selector, msg, authRevisionChanged) {
 		await this.#runInLockForClient(selector, msg, async (lc, clientID, _, cvr) => {
@@ -375,15 +380,15 @@ var ViewSyncerService = class {
 				return;
 			}
 			lc.debug?.("Auth changed, re-validating and re-transforming queries");
-			const connection = this.contextManager.mustGetConnectionContext(selector);
+			const connCtx = this.connContextManager.mustGetConnectionContext(selector);
 			if (!this.#pipelinesSynced) {
-				if (!await this.#validateConnection(connection)) return;
+				if (!await this.#validateConnection(connCtx)) return;
 			}
-			return await this.#handleConfigUpdate(lc, clientID, {}, cvr, "all", void 0, connection);
+			return await this.#handleConfigUpdate(lc, clientID, {}, cvr, "all", void 0, connCtx);
 		});
 	}
 	async deleteClients(selector, msg) {
-		return await this.#runInLockForClient(selector, [msg[0], { deleted: msg[1] }], (lc, clientID, msg, cvr) => this.#handleConfigUpdate(lc, clientID, msg, cvr, "missing", void 0, this.contextManager.mustGetConnectionContext(selector))) ?? [];
+		return await this.#runInLockForClient(selector, [msg[0], { deleted: msg[1] }], (lc, clientID, msg, cvr) => this.#handleConfigUpdate(lc, clientID, msg, cvr, "missing", void 0, this.connContextManager.mustGetConnectionContext(selector))) ?? [];
 	}
 	#getTTLClock(now) {
 		const delta = now - this.#ttlClockBase;
@@ -425,7 +430,7 @@ var ViewSyncerService = class {
 			lc.warn?.("failed to update TTL clock", rid, `after ${Date.now() - start} ms`, e);
 		});
 	}
-	async #updateCVRConfig(lc, cvr, clientID, customQueryTransformMode, ctx, fn) {
+	async #updateCVRConfig(lc, cvr, clientID, customQueryTransformMode, connCtx, fn) {
 		const updater = new CVRConfigDrivenUpdater(this.#cvrStore, cvr, this.#shard);
 		updater.ensureClient(clientID);
 		const patches = fn(updater);
@@ -438,7 +443,7 @@ var ViewSyncerService = class {
 				await pokers.end(newCVR.version);
 			});
 		}
-		if (this.#pipelinesSynced) await this.#syncQueryPipelineSet(lc, this.#cvr, customQueryTransformMode, ctx);
+		if (this.#pipelinesSynced) await this.#syncQueryPipelineSet(lc, this.#cvr, customQueryTransformMode, connCtx);
 		return this.#cvr;
 	}
 	/**
@@ -455,7 +460,7 @@ var ViewSyncerService = class {
 			span.setAttribute("clientID", clientID);
 			let client;
 			let result;
-			let ctx;
+			let connCtx;
 			try {
 				await this.#runInLockWithCVR(async (lc, cvr) => {
 					lc = lc.withContext("clientID", clientID).withContext("wsID", wsID).withContext("cmd", cmd);
@@ -465,7 +470,7 @@ var ViewSyncerService = class {
 						lc.debug?.("mismatched wsID", client?.wsID, wsID);
 						return;
 					}
-					ctx = this.contextManager.getConnectionContext(selector);
+					connCtx = this.connContextManager.getConnectionContext(selector);
 					if (newClient) {
 						assert(newClient === client, "newClient must match existing client");
 						checkClientAndCVRVersions(client.version(), cvr.version);
@@ -475,7 +480,7 @@ var ViewSyncerService = class {
 				});
 			} catch (e) {
 				this.#lc.withContext("clientID", clientID).withContext("wsID", wsID).withContext("cmd", cmd)[getLogLevel(e)]?.(`closing connection with error`, e);
-				if (ctx) this.contextManager.failConnection(selector, ctx.revision);
+				if (connCtx) this.connContextManager.failConnection(selector, connCtx.revision);
 				if (client) client.fail(e);
 				else throw e;
 			}
@@ -486,10 +491,10 @@ var ViewSyncerService = class {
 		const clients = [...this.#clients.values()];
 		return atVersion ? clients.filter((c) => cmpVersions(c.version() ?? EMPTY_CVR_VERSION, atVersion) === 0) : clients;
 	}
-	#handleConfigUpdate = (lc, clientID, { clientSchema, deleted, desiredQueriesPatch, activeClients }, cvr, customQueryTransformMode, profileID, ctx) => startAsyncSpan(tracer, "vs.#handleConfigUpdate", async () => {
+	#handleConfigUpdate = (lc, clientID, { clientSchema, deleted, desiredQueriesPatch, activeClients }, cvr, customQueryTransformMode, profileID, connCtx) => startAsyncSpan(tracer, "vs.#handleConfigUpdate", async () => {
 		const deletedClientIDs = [];
 		const deletedClientGroupIDs = [];
-		cvr = await this.#updateCVRConfig(lc, cvr, clientID, customQueryTransformMode, ctx, (updater) => {
+		cvr = await this.#updateCVRConfig(lc, cvr, clientID, customQueryTransformMode, connCtx, (updater) => {
 			const { ttlClock } = cvr;
 			const patches = [];
 			if (clientSchema) updater.setClientSchema(lc, clientSchema);
@@ -567,7 +572,7 @@ var ViewSyncerService = class {
 		const cvrVersion = cvr.version;
 		if (cvrVersion.stateVersion !== dbVersion) {
 			lc.info?.(`CVR (${versionToCookie(cvrVersion)}) is behind db ${dbVersion}`);
-			return;
+			return /* @__PURE__ */ new Set();
 		}
 		const gotQueries = Object.entries(cvr.queries).filter(([_, state]) => state.transformationHash !== void 0);
 		const customQueries = /* @__PURE__ */ new Map();
@@ -586,11 +591,11 @@ var ViewSyncerService = class {
 		let customHashMismatchCount = 0;
 		let otherHashMismatchCount = 0;
 		if (customQueries.size > 0 && !this.#customQueryTransformer) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
-		const backgroundContext = this.contextManager.mustGetBackgroundConnectionContext();
+		const backgroundContext = this.connContextManager.mustGetBackgroundConnectionContext();
 		const customQueryTransformer = this.#customQueryTransformer;
 		if (customQueryTransformer && customQueries.size > 0) {
 			const transformedCustomQueries = await this.#runPriorityOp(lc, "#hydrateUnchangedQueries transforming custom queries", () => customQueryTransformer.transform(backgroundContext, customQueries.values()));
-			if (!transformedCustomQueries.cached) this.contextManager.validateConnection(backgroundContext, backgroundContext.revision);
+			if (transformedCustomQueries.kind === "success" && !transformedCustomQueries.cached) this.connContextManager.validateConnection(backgroundContext, backgroundContext.revision, transformedCustomQueries.validation);
 			if (Array.isArray(transformedCustomQueries.result)) for (const q of transformedCustomQueries.result) if ("error" in q) customErrorCount++;
 			else if (q.transformationHash !== customQueries.get(q.id)?.transformationHash) customHashMismatchCount++;
 			else transformedQueries.push(q);
@@ -601,6 +606,7 @@ var ViewSyncerService = class {
 			else otherHashMismatchCount++;
 		}
 		lc.info?.(`hydrateUnchangedQueries: ${gotQueries.length} got queries, ${inactivatedCount} inactivated, ${customErrorCount} custom transform errors, ${customHashMismatchCount} custom hash mismatches, ${otherHashMismatchCount} other hash mismatches, ${transformedQueries.length} hydrated`);
+		const driftedQueryIDs = /* @__PURE__ */ new Set();
 		for (const { id: queryID, transformationHash, transformedAst } of transformedQueries) {
 			const timer = new TimeSliceTimer(lc);
 			let count = 0;
@@ -616,7 +622,19 @@ var ViewSyncerService = class {
 			this.#hydrationTime.recordMs(elapsed);
 			this.#addQueryMaterializationServerMetric(transformationHash, elapsed);
 			lc.debug?.(`hydrated ${count} rows for ${queryID} (${elapsed} ms)`);
+			const storedSigHex = cvr.queries[queryID]?.rowSetSignature;
+			if (storedSigHex !== void 0 && storedSigHex !== null) {
+				const priorSig = parseSignature(storedSigHex);
+				const candidateSig = this.#pipelines.rowSetSignature(queryID) ?? 0n;
+				if (priorSig !== candidateSig) {
+					lc.warn?.(`rowSetSignature drift for query ${queryID}: prior=${priorSig.toString(16)} new=${candidateSig.toString(16)} (${count} rows). Removing from pipelines for full re-execution.`);
+					this.#rowSetSignatureDrifts.add(1);
+					this.#pipelines.removeQuery(queryID);
+					driftedQueryIDs.add(queryID);
+				}
+			}
 		}
+		return driftedQueryIDs;
 	}
 	#processTransformedCustomQueries(lc, transformedCustomQueries, cb, customQueryMap) {
 		if ("kind" in transformedCustomQueries) {
@@ -669,7 +687,7 @@ var ViewSyncerService = class {
 	*
 	* This must be called from within the #lock.
 	*/
-	#syncQueryPipelineSet(lc, cvr, customQueryTransformMode, ctx) {
+	#syncQueryPipelineSet(lc, cvr, customQueryTransformMode, connCtx, driftedQueryIDs = /* @__PURE__ */ new Set()) {
 		return startAsyncSpan(tracer, "vs.#syncQueryPipelineSet", async (span) => {
 			span.setAttribute("clientGroupID", this.id);
 			assert(this.#pipelines.initialized(), "pipelines must be initialized (syncQueryPipelineSet)");
@@ -680,7 +698,7 @@ var ViewSyncerService = class {
 			const customQueries = /* @__PURE__ */ new Map();
 			const otherQueries = [];
 			const transformedQueries = [];
-			const resolvedContext = ctx ?? this.contextManager.mustGetBackgroundConnectionContext();
+			const resolvedConnCtx = connCtx ?? this.connContextManager.mustGetBackgroundConnectionContext();
 			for (const [id, query] of cvrQueryEntires) if (query.type === "custom") {
 				assert(id === query.id, "custom query id mismatch");
 				customQueries.set(id, query);
@@ -690,7 +708,7 @@ var ViewSyncerService = class {
 			});
 			for (const { id, query: origQuery } of otherQueries) {
 				assert(id === origQuery.id, "query id mismatch");
-				const transformed = transformAndHashQuery(lc, origQuery.id, origQuery.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, resolvedContext.auth?.type === "jwt" ? resolvedContext.auth : void 0, origQuery.type === "internal");
+				const transformed = transformAndHashQuery(lc, origQuery.id, origQuery.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, resolvedConnCtx.auth?.type === "jwt" ? resolvedConnCtx.auth : void 0, origQuery.type === "internal");
 				transformedQueries.push({
 					id,
 					origQuery,
@@ -705,10 +723,10 @@ var ViewSyncerService = class {
 				const transformStart = performance.now();
 				let transformedCustomQueries;
 				try {
-					transformedCustomQueries = await this.#runPriorityOp(lc, "#syncQueryPipelineSet transforming custom queries", () => customQueryTransformer.transform(resolvedContext, customQueriesToTransform));
-					if ("kind" in transformedCustomQueries.result) throw new ProtocolErrorWithLevel(transformedCustomQueries.result, "warn");
+					transformedCustomQueries = await this.#runPriorityOp(lc, "#syncQueryPipelineSet transforming custom queries", () => customQueryTransformer.transform(resolvedConnCtx, customQueriesToTransform));
+					if (transformedCustomQueries.kind === "failed") throw new ProtocolErrorWithLevel(transformedCustomQueries.result, "warn");
 					else {
-						if (!transformedCustomQueries.cached) this.contextManager.validateConnection(resolvedContext, resolvedContext.revision);
+						if (!transformedCustomQueries.cached) this.connContextManager.validateConnection(resolvedConnCtx, resolvedConnCtx.revision, transformedCustomQueries.validation);
 						this.#queryTransformations.add(1, { result: "success" });
 					}
 				} catch (e) {
@@ -755,7 +773,7 @@ var ViewSyncerService = class {
 				const orig = cvr.queries[q.id];
 				lc.debug?.("ViewSyncer adding query", q.ast, "transformed from", orig.type === "custom" ? orig.name : orig.ast);
 			}
-			if (addQueries.length > 0 || removeQueriesQueryIds.size > 0) await this.#addAndRemoveQueries(lc, cvr, addQueries, Array.from(removeQueriesQueryIds, (id) => ({ id })));
+			if (addQueries.length > 0 || removeQueriesQueryIds.size > 0) await this.#addAndRemoveQueries(lc, cvr, addQueries, Array.from(removeQueriesQueryIds, (id) => ({ id })), driftedQueryIDs);
 			else await this.#catchupClients(lc, cvr);
 		});
 	}
@@ -787,7 +805,7 @@ var ViewSyncerService = class {
 		record.count++;
 		if (record.count >= THRASH_THRESHOLD) this.#lc.warn?.(`Query thrashing detected for query ${queryID}. ${record.count} replacements in 60s. This may indicate clients with different auth contexts connecting to the same client group.`);
 	}
-	#addAndRemoveQueries(lc, cvr, addQueries, removeQueries) {
+	#addAndRemoveQueries(lc, cvr, addQueries, removeQueries, driftedQueryIDs = /* @__PURE__ */ new Set()) {
 		return startAsyncSpan(tracer, "vs.#addAndRemoveQueries", async () => {
 			assert(addQueries.length > 0 || removeQueries.length > 0, "Must have queries to add or remove");
 			const start = performance.now();
@@ -795,9 +813,14 @@ var ViewSyncerService = class {
 			lc = lc.withContext("stateVersion", stateVersion);
 			lc.info?.(`hydrating ${addQueries.length} queries`);
 			const updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, stateVersion, this.#pipelines.replicaVersion, (queryID) => this.#pipelines.rowSetSignature(queryID));
-			const { newVersion, queryPatches } = updater.trackQueries(lc, addQueries, removeQueries);
+			const { queryPatches } = updater.trackQueries(lc, addQueries, removeQueries);
+			if (addQueries.some((q) => driftedQueryIDs.has(q.id))) updater.ensureNewVersion();
+			const newVersion = updater.updatedVersion();
 			const pokers = startPoke(this.#getClients(), newVersion);
-			for (const patch of queryPatches) await pokers.addPatch(patch);
+			for (const patch of queryPatches) await pokers.addPatch({
+				...patch,
+				toVersion: newVersion
+			});
 			for (const q of removeQueries) {
 				this.#pipelines.removeQuery(q.id);
 				this.#inspectorDelegate.removeQuery(q.id);
@@ -1008,88 +1031,98 @@ var ViewSyncerService = class {
 	}
 	#handleInspect = async (lc, clientID, body, cvr) => {
 		const client = must(this.#clients.get(clientID));
-		const ctx = this.contextManager.mustGetConnectionContext({
+		const connCtx = this.connContextManager.mustGetConnectionContext({
 			clientID,
 			wsID: client.wsID
 		});
-		return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this.#cvrStore, this.#config, ctx);
+		return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this.#cvrStore, this.#config, connCtx);
 	};
 	async #runBackgroundRetransform(lc) {
-		const attemptRetransform = async (connection) => {
-			await this.#syncQueryPipelineSet(lc, must(this.#cvr, "cvr missing during auth maintenance retransform"), "all", connection);
-			this.contextManager.markBackgroundRetransformSuccess({
-				clientID: connection.clientID,
-				wsID: connection.wsID
-			}, connection.revision);
+		const attemptRetransform = async (connCtx) => {
+			await this.#syncQueryPipelineSet(lc, must(this.#cvr, "cvr missing during auth maintenance retransform"), "all", connCtx);
+			this.connContextManager.markBackgroundRetransformSuccess({
+				clientID: connCtx.clientID,
+				wsID: connCtx.wsID
+			}, connCtx.revision);
 		};
-		let backgroundConnection = this.contextManager.getBackgroundConnectionContext();
-		if (!backgroundConnection) {
+		let backgroundConnCtx = this.connContextManager.getBackgroundConnectionContext();
+		if (!backgroundConnCtx) {
 			lc.debug?.("Skipping background retransform with no selected connection");
 			return;
 		}
 		for (;;) {
 			try {
-				await attemptRetransform(backgroundConnection);
+				await attemptRetransform(backgroundConnCtx);
 				return;
 			} catch (e) {
 				if (isProtocolError(e)) {
 					if (isAuthErrorBody(e.errorBody)) {
 						lc.warn?.("Background retransform auth failed; failing connection and searching for replacement", {
-							clientID: backgroundConnection.clientID,
+							clientID: backgroundConnCtx.clientID,
 							message: e.message
 						});
-						this.#failMaintenanceConnection(backgroundConnection, e);
+						this.#failMaintenanceConnection(backgroundConnCtx, e);
 					} else if (isTransformFailedError(e)) {
 						lc.warn?.("Background retransform failed; deferring auth maintenance", {
-							clientID: backgroundConnection.clientID,
+							clientID: backgroundConnCtx.clientID,
 							message: e.message
 						});
-						this.contextManager.deferMaintenance("retransform");
+						this.connContextManager.deferMaintenance("retransform");
 						return;
 					}
 				} else throw e;
 			}
-			const replacement = this.contextManager.getBackgroundConnectionContext();
-			if (!replacement) {
+			const replacementConnCtx = this.connContextManager.getBackgroundConnectionContext();
+			if (!replacementConnCtx) {
 				lc.debug?.("No replacement connection available for background retransform");
 				return;
 			}
 			lc.debug?.("Retrying background retransform with replacement connection", {
-				clientID: replacement.clientID,
-				wsID: replacement.wsID
+				clientID: replacementConnCtx.clientID,
+				wsID: replacementConnCtx.wsID
 			});
-			backgroundConnection = replacement;
+			backgroundConnCtx = replacementConnCtx;
 		}
 	}
-	async #validateConnection(ctx) {
+	async #validateConnection(connCtx) {
 		try {
+			let validation = void 0;
 			if (this.#customQueryTransformer) {
-				const validation = await this.#customQueryTransformer.validate(ctx);
-				if (validation !== void 0) throw new ProtocolErrorWithLevel(validation, "warn");
-			}
-			this.contextManager.validateConnection(ctx, ctx.revision);
+				const response = await this.#customQueryTransformer.validate(connCtx);
+				if (response.kind === "TransformFailed") throw new ProtocolErrorWithLevel(response, "warn");
+				validation = response.validation;
+			} else validation = { kind: "client-fallback" };
+			this.connContextManager.validateConnection(connCtx, connCtx.revision, validation);
 			return true;
 		} catch (e) {
 			if (isProtocolError(e) && isAuthErrorBody(e.errorBody)) {
-				this.#failMaintenanceConnection(ctx, e);
+				this.#lc.warn?.("Connection auth validation failed; invalidating connection", {
+					clientID: connCtx.clientID,
+					wsID: connCtx.wsID,
+					revision: connCtx.revision,
+					message: e.message
+				});
+				this.#failMaintenanceConnection(connCtx, e);
 				return false;
 			}
 			throw e;
 		}
 	}
-	#failMaintenanceConnection(ctx, error) {
-		if (!this.contextManager.failConnection(ctx, ctx.revision)) return;
+	#failMaintenanceConnection(connCtx, error) {
+		if (!this.connContextManager.failConnection(connCtx, connCtx.revision)) return;
 		const wrapped = wrapWithProtocolError(error);
-		const client = this.#clients.get(ctx.clientID);
-		if (client?.wsID === ctx.wsID) client.fail(wrapped);
+		const client = this.#clients.get(connCtx.clientID);
+		if (client?.wsID === connCtx.wsID) client.fail(wrapped);
 	}
 	stop() {
 		this.#lc.info?.("stopping view syncer");
+		this.connContextManager.setSharedRetransformReady(false);
 		this.#initialized.reject("shut down before initialization completed");
 		this.#stateChanges.cancel();
 		return this.#stopped.promise;
 	}
 	async #cleanup(err) {
+		this.connContextManager.setSharedRetransformReady(false);
 		this.#stopTTLClockInterval();
 		this.#stopExpireTimer();
 		this.#stopAuthMaintenanceTimer();