@rocicorp/zero 1.2.0 → 1.3.0-canary.1

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -2,7 +2,6 @@ import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { must } from "../../../../shared/src/must.js";
 import { InvalidConnectionRequest, InvalidConnectionRequestBaseCookie, Rehome } from "../../../../zero-protocol/src/error-kind-enum.js";
 import { ZeroCache } from "../../../../zero-protocol/src/error-origin-enum.js";
-import "../../../../zero-protocol/src/error-reason-enum.js";
 import { ProtocolError, isProtocolError } from "../../../../zero-protocol/src/error.js";
 import { MAX_TTL_MS, clampTTL } from "../../../../zql/src/query/ttl.js";
 import { stringify } from "../../../../shared/src/bigint-json.js";
@@ -19,10 +18,11 @@ import { Subscription } from "../../types/subscription.js";
 import { CustomKeyMap } from "../../../../shared/src/custom-key-map.js";
 import { ttlClockAsNumber, ttlClockFromNumber } from "./ttl-clock.js";
 import { EMPTY_CVR_VERSION, cmpVersions, versionFromString, versionString, versionToCookie } from "./schema/types.js";
-import { ProtocolErrorWithLevel, getLogLevel } from "../../types/error-with-level.js";
+import { ProtocolErrorWithLevel, getLogLevel, wrapWithProtocolError } from "../../types/error-with-level.js";
+import { isAuthErrorBody } from "../../auth/auth.js";
 import { ClientHandler, startPoke } from "./client-handler.js";
-import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
 import { tracer } from "./tracer.js";
+import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
 import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, nextEvictionTime } from "./cvr.js";
 import { handleInspect } from "./inspect-handler.js";
 import { resolver } from "@rocicorp/resolver";
@@ -41,6 +41,7 @@ function randomID() {
 var TTL_CLOCK_INTERVAL = 6e4;
 var ViewSyncerService = class {
 	id;
+	contextManager;
 	#shard;
 	#lc;
 	#pipelines;
@@ -48,10 +49,6 @@ var ViewSyncerService = class {
 	#drainCoordinator;
 	#keepaliveMs;
 	#slowHydrateThreshold;
-	#queryConfig;
-	#authSession;
-	userQueryURL;
-	userQueryHeaders;
 	#lastConnectTime = Date.now();
 	/**
 	* The TTL clock is used to determine the time at which queries are considered
@@ -82,10 +79,8 @@ var ViewSyncerService = class {
 	#initialized = resolver();
 	#cvr;
 	#pipelinesSynced = false;
-	#httpCookie;
-	#origin;
-	#lastAuthRevision = 0;
 	#expiredQueriesTimer = 0;
+	#authMaintenanceTimer = 0;
 	#setTimeout;
 	#customQueryTransformer;
 	#queryReplacements = /* @__PURE__ */ new Map();
@@ -106,15 +101,19 @@ var ViewSyncerService = class {
 	});
 	#queryTransformationHashChanges = getOrCreateCounter("sync", "query.transformation-hash-changes", "Number of times query transformation hash changed");
 	#queryTransformationNoOps = getOrCreateCounter("sync", "query.transformation-no-ops", "Number of times query transformation resulted in no-op (hash unchanged)");
+	#lockWaitTime = getOrCreateHistogram("sync", "lock-wait-time", {
+		description: "Time spent waiting to acquire the ViewSyncer lock.",
+		unit: "s"
+	});
+	#pipelineResets = getOrCreateCounter("sync", "pipeline-resets", "Number of pipeline resets");
 	#inspectorDelegate;
 	#config;
 	#runPriorityOp;
-	constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, customQueryTransformer, runPriorityOp, authSession, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis)) {
-		const queryConfig = config.query?.url ? config.query : config.getQueries;
+	constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, contextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis)) {
 		this.#config = config;
 		this.id = clientGroupID;
+		this.contextManager = contextManager;
 		this.#shard = shard;
-		this.#queryConfig = queryConfig;
 		this.#lc = lc;
 		this.#pipelines = pipelineDriver;
 		this.#stateChanges = versionChanges;
@@ -123,36 +122,17 @@ var ViewSyncerService = class {
 		this.#slowHydrateThreshold = slowHydrateThreshold;
 		this.#inspectorDelegate = inspectorDelegate;
 		this.#customQueryTransformer = customQueryTransformer;
-		this.#authSession = authSession;
 		this.#cvrStore = new CVRStore(lc, cvrDb, shard, taskID, clientGroupID, () => this.#stateChanges.cancel());
 		this.#setTimeout = setTimeoutFn;
 		this.#runPriorityOp = runPriorityOp;
 		this.keepalive();
 	}
-	get auth() {
-		return this.#authSession.auth;
-	}
-	clearAuth() {
-		this.#authSession.clear();
-		this.#lastAuthRevision = 0;
-	}
-	initAuthSession(userID, wireAuth) {
-		return this.#authSession.update(userID, wireAuth);
-	}
-	#getHeaderOptions(forwardCookie) {
-		return {
-			apiKey: this.#queryConfig.apiKey,
-			customHeaders: this.userQueryHeaders,
-			allowedClientHeaders: this.#queryConfig.allowedClientHeaders,
-			token: this.#authSession.auth?.raw,
-			cookie: forwardCookie ? this.#httpCookie : void 0,
-			origin: this.#origin
-		};
-	}
 	#runInLockWithCVR(fn) {
 		const rid = randomID();
 		this.#lc.debug?.("about to acquire lock for cvr ", rid);
+		const lockWaitStart = performance.now();
 		return this.#lock.withLock(async () => {
+			this.#lockWaitTime.record((performance.now() - lockWaitStart) / 1e3);
 			this.#lc.debug?.("acquired lock in #runInLockWithCVR ", rid);
 			const lc = this.#lc.withContext("lock", rid);
 			if (!this.#stateChanges.active) {
@@ -186,6 +166,8 @@ var ViewSyncerService = class {
 			} catch (e) {
 				this.#cvr = void 0;
 				throw e;
+			} finally {
+				this.#scheduleAuthMaintenance(lc);
 			}
 		});
 	}
@@ -216,6 +198,7 @@ var ViewSyncerService = class {
 						const result = await this.#advancePipelines(lc, cvr);
 						if (result === "success") return;
 						lc.info?.(`resetting pipelines: ${result.message}`);
+						this.#pipelineResets.add(1, { reason: result.reason });
 						this.#pipelines.reset(clientSchema);
 						this.#pipelinesSynced = false;
 					}
@@ -227,7 +210,7 @@ var ViewSyncerService = class {
 					}
 					lc.info?.(`init pipelines@${version} (cvr@${cvrVer})`);
 					await this.#hydrateUnchangedQueries(lc, cvr);
-					await this.#syncQueryPipelineSet(lc, cvr, "missing");
+					await this.#syncQueryPipelineSet(lc, cvr, "missing", void 0);
 					this.#pipelinesSynced = true;
 				});
 			}
@@ -246,13 +229,19 @@ var ViewSyncerService = class {
 		if (hasExpiredQueries(cvr)) {
 			lc = lc.withContext("method", "#removeExpiredQueries");
 			lc.debug?.("Queries have expired");
-			if (this.#pipelinesSynced) await this.#syncQueryPipelineSet(lc, cvr, "missing");
+			if (this.#pipelinesSynced) await this.#syncQueryPipelineSet(lc, cvr, "missing", void 0);
 		}
 		this.#scheduleExpireEviction(lc, cvr);
 	};
 	#totalHydrationTimeMs() {
 		return this.#pipelines.totalHydrationTimeMs();
 	}
+	get queryCount() {
+		return this.#pipelines.initialized() ? this.#pipelines.queries().size : 0;
+	}
+	get rowCount() {
+		return this.#cvrStore.rowCount;
+	}
 	#keepAliveUntil = 0;
 	/**
 	* Guarantees that the ViewSyncer will remain running for at least
@@ -285,6 +274,10 @@ var ViewSyncerService = class {
 		return this.#clients.size === 0;
 	}
 	#deleteClientDueToDisconnect(clientID, client) {
+		this.contextManager.closeConnection({
+			clientID,
+			wsID: client.wsID
+		});
 		if (this.#clients.get(clientID) === client) {
 			this.#clients.delete(clientID);
 			if (this.#clients.size === 0) {
@@ -299,73 +292,110 @@ var ViewSyncerService = class {
 		clearTimeout(this.#expiredQueriesTimer);
 		this.#expiredQueriesTimer = 0;
 	}
-	initConnection(ctx, initConnectionMessage) {
+	#stopAuthMaintenanceTimer() {
+		if (this.#authMaintenanceTimer !== 0) this.#lc.debug?.("Stopping auth maintenance timer");
+		clearTimeout(this.#authMaintenanceTimer);
+		this.#authMaintenanceTimer = 0;
+	}
+	/**
+	* Schedules the auth maintenance wakeup from coordinator-derived
+	* deadlines. The timer plumbing is intentionally separate from the actual
+	* revalidation/retransform work so future policy changes only need to update
+	* the maintenance workers, not the wakeup logic.
+	*
+	* This is intentionally cheap & idempotent, so it can be called frequently
+	* when upstream state might have changed.
+	*/
+	#scheduleAuthMaintenance(lc) {
+		this.#stopAuthMaintenanceTimer();
+		const plan = this.contextManager.planMaintenance();
+		if (plan.earliestDeadlineAt === void 0) {
+			lc.debug?.("No auth maintenance wakeup scheduled");
+			return;
+		}
+		const delay = Math.max(0, plan.earliestDeadlineAt - Date.now());
+		lc.debug?.(`Scheduling auth maintenance timer at ${new Date(plan.earliestDeadlineAt).toISOString()}`, {
+			delay,
+			earliestDeadlineAt: plan.earliestDeadlineAt
+		});
+		this.#authMaintenanceTimer = this.#setTimeout(() => {
+			this.#authMaintenanceTimer = 0;
+			this.#runInLockWithCVR((lc, cvr) => this.#runAuthMaintenance(lc, cvr)).catch((e) => this.#stateChanges.fail(e));
+		}, delay);
+	}
+	async #runAuthMaintenance(lc, _cvr) {
+		const plan = this.contextManager.planMaintenance();
+		if (plan.dueRevalidations.length === 0 && !plan.dueRetransform) {
+			lc.debug?.("Auth maintenance woke up with no due work");
+			return;
+		}
+		lc.debug?.("Auth maintenance woke up with pending work", {
+			dueRevalidations: plan.dueRevalidations.length,
+			dueRetransform: plan.dueRetransform
+		});
+		for (const connection of plan.dueRevalidations) try {
+			await this.#validateConnection(connection);
+		} catch (e) {
+			if (isProtocolError(e) && isTransformFailedError(e)) {
+				lc.warn?.("Scheduled auth revalidation failed; deferring auth maintenance", {
+					clientID: connection.clientID,
+					wsID: connection.wsID,
+					message: e.message
+				});
+				this.contextManager.deferMaintenance("revalidate");
+				return;
+			}
+			throw e;
+		}
+		if (this.contextManager.planMaintenance().dueRetransform) await this.#runBackgroundRetransform(lc);
+	}
+	initConnection(selector, initConnectionMessage) {
 		this.#lc.debug?.("viewSyncer.initConnection");
 		return startSpan(tracer, "vs.initConnection", () => {
-			const { clientID, profileID, wsID, baseCookie, httpCookie, origin, protocolVersion } = ctx;
-			this.#httpCookie = httpCookie;
-			this.#origin = origin;
-			const [, { userQueryURL, userQueryHeaders }] = initConnectionMessage;
-			if (this.userQueryURL === void 0) {
-				this.userQueryURL = userQueryURL;
-				this.userQueryHeaders = userQueryHeaders;
-			} else if (this.userQueryURL !== userQueryURL) this.#lc.warn?.("Client provided different query parameters than client group", {
-				clientID,
-				clientURL: userQueryURL,
-				clientGroupURL: this.userQueryURL
-			});
-			const lc = this.#lc.withContext("clientID", clientID).withContext("wsID", wsID);
+			const ctx = this.contextManager.mustGetConnectionContext(selector);
+			const lc = this.#lc.withContext("clientID", ctx.clientID).withContext("wsID", ctx.wsID);
 			const downstream = Subscription.create({ cleanup: (_, err) => {
 				err ? lc[getLogLevel(err)]?.(`client closed with error`, err) : lc.info?.("client closed");
-				this.#deleteClientDueToDisconnect(clientID, newClient);
-				this.#activeClients.add(-1, { [PROTOCOL_VERSION_ATTR]: protocolVersion });
+				this.#deleteClientDueToDisconnect(ctx.clientID, newClient);
+				this.#activeClients.add(-1, { [PROTOCOL_VERSION_ATTR]: ctx.protocolVersion });
 			} });
-			this.#activeClients.add(1, { [PROTOCOL_VERSION_ATTR]: protocolVersion });
+			this.#activeClients.add(1, { [PROTOCOL_VERSION_ATTR]: ctx.protocolVersion });
 			if (this.#clients.size === 0) this.#ttlClockBase = Date.now();
-			const newClient = new ClientHandler(lc, this.id, clientID, wsID, this.#shard, baseCookie, downstream);
-			this.#clients.get(clientID)?.close(`replaced by wsID: ${wsID}`);
-			this.#clients.set(clientID, newClient);
-			this.#runInLockForClient(ctx, initConnectionMessage, async (lc, clientID, msg, cvr) => {
+			const newClient = new ClientHandler(lc, this.id, ctx.clientID, ctx.wsID, this.#shard, ctx.baseCookie, downstream);
+			this.#clients.get(ctx.clientID)?.close(`replaced by wsID: ${ctx.wsID}`);
+			this.#clients.set(ctx.clientID, newClient);
+			startAsyncSpan(tracer, "vs.initConnection.async", () => this.#runInLockForClient(ctx, initConnectionMessage, async (lc, clientID, msg, cvr) => {
 				if (cvr.clientSchema === null && !msg.clientSchema) throw new ProtocolErrorWithLevel({
 					kind: InvalidConnectionRequest,
 					message: "The initConnection message for a new client group must include client schema.",
 					origin: ZeroCache
 				}, "warn");
-				await this.#handleConfigUpdate(lc, clientID, msg, cvr, "all", profileID ?? `cg${this.id}`);
+				if (!await this.#validateConnection(ctx)) return;
+				await this.#handleConfigUpdate(lc, clientID, msg, cvr, "all", ctx.profileID ?? `cg${this.id}`, ctx);
 				this.#initialized.resolve("initialized");
-			}, newClient).catch((e) => newClient.fail(e));
+			}, newClient)).catch((e) => newClient.fail(e));
 			return downstream;
 		});
 	}
-	async changeDesiredQueries(ctx, msg) {
-		await this.#runInLockForClient(ctx, msg, async (lc, clientID, msg, cvr) => {
-			let customQueryTransformMode = "missing";
-			const currentAuthRevision = this.#authSession.revision;
-			if (this.#lastAuthRevision < currentAuthRevision) {
-				customQueryTransformMode = "all";
-				lc.debug?.("Auth revision changed, setting customQueryTransformMode to all");
-			}
-			const result = await this.#handleConfigUpdate(lc, clientID, msg, cvr, customQueryTransformMode);
-			if (customQueryTransformMode === "all") this.#lastAuthRevision = currentAuthRevision;
-			return result;
-		});
+	async changeDesiredQueries(selector, msg) {
+		await this.#runInLockForClient(selector, msg, (lc, clientID, msg, cvr) => this.#handleConfigUpdate(lc, clientID, msg, cvr, "missing", void 0, this.contextManager.mustGetConnectionContext(selector)));
 	}
-	async updateAuth(ctx, msg) {
-		await this.#runInLockForClient(ctx, msg, async (lc, clientID, _, cvr) => {
-			const authResult = await this.#authSession.update(ctx.userID, msg[1].auth);
-			if (!authResult.ok) throw new ProtocolErrorWithLevel(authResult.error, "warn");
-			const currentAuthRevision = this.#authSession.revision;
-			if (this.#lastAuthRevision >= currentAuthRevision) {
-				lc.debug?.("Auth revision unchanged, skipping query re-transformation");
+	async updateAuth(selector, msg, authRevisionChanged) {
+		await this.#runInLockForClient(selector, msg, async (lc, clientID, _, cvr) => {
+			if (!authRevisionChanged) {
+				lc.debug?.("Auth unchanged, skipping query re-transformation");
 				return;
-			} else lc.debug?.("Auth revision changed, re-transforming queries");
-			const result = await this.#handleConfigUpdate(lc, clientID, {}, cvr, "all");
-			this.#lastAuthRevision = currentAuthRevision;
-			return result;
+			}
+			lc.debug?.("Auth changed, re-validating and re-transforming queries");
+			const connection = this.contextManager.mustGetConnectionContext(selector);
+			if (!this.#pipelinesSynced) {
+				if (!await this.#validateConnection(connection)) return;
+			}
+			return await this.#handleConfigUpdate(lc, clientID, {}, cvr, "all", void 0, connection);
 		});
 	}
-	async deleteClients(ctx, msg) {
-		return await this.#runInLockForClient(ctx, [msg[0], { deleted: msg[1] }], (lc, clientID, msg, cvr) => this.#handleConfigUpdate(lc, clientID, msg, cvr, "missing")) ?? [];
+	async deleteClients(selector, msg) {
+		return await this.#runInLockForClient(selector, [msg[0], { deleted: msg[1] }], (lc, clientID, msg, cvr) => this.#handleConfigUpdate(lc, clientID, msg, cvr, "missing", void 0, this.contextManager.mustGetConnectionContext(selector))) ?? [];
 	}
 	#getTTLClock(now) {
 		const delta = now - this.#ttlClockBase;
@@ -407,7 +437,7 @@ var ViewSyncerService = class {
 			lc.warn?.("failed to update TTL clock", rid, `after ${Date.now() - start} ms`, e);
 		});
 	}
-	async #updateCVRConfig(lc, cvr, clientID, customQueryTransformMode, fn) {
+	async #updateCVRConfig(lc, cvr, clientID, customQueryTransformMode, ctx, fn) {
 		const updater = new CVRConfigDrivenUpdater(this.#cvrStore, cvr, this.#shard);
 		updater.ensureClient(clientID);
 		const patches = fn(updater);
@@ -420,21 +450,24 @@ var ViewSyncerService = class {
 				await pokers.end(newCVR.version);
 			});
 		}
-		if (this.#pipelinesSynced) await this.#syncQueryPipelineSet(lc, this.#cvr, customQueryTransformMode);
+		if (this.#pipelinesSynced) await this.#syncQueryPipelineSet(lc, this.#cvr, customQueryTransformMode, ctx);
 		return this.#cvr;
 	}
 	/**
 	* Runs the given `fn` to process the `msg` from within the `#lock`,
 	* optionally adding the `newClient` if supplied.
 	*/
-	#runInLockForClient(ctx, msg, fn, newClient) {
+	#runInLockForClient(selector, msg, fn, newClient) {
 		this.#lc.debug?.("viewSyncer.#runInLockForClient");
-		const { clientID, wsID } = ctx;
+		const { clientID, wsID } = selector;
 		const [cmd, body] = msg;
 		if (newClient || !this.#clients.has(clientID)) this.#lastConnectTime = Date.now();
-		return startAsyncSpan(tracer, `vs.#runInLockForClient(${cmd})`, async () => {
+		return startAsyncSpan(tracer, `vs.#runInLockForClient(${cmd})`, async (span) => {
+			span.setAttribute("clientGroupID", this.id);
+			span.setAttribute("clientID", clientID);
 			let client;
 			let result;
+			let ctx;
 			try {
 				await this.#runInLockWithCVR(async (lc, cvr) => {
 					lc = lc.withContext("clientID", clientID).withContext("wsID", wsID).withContext("cmd", cmd);
@@ -444,6 +477,7 @@ var ViewSyncerService = class {
 						lc.debug?.("mismatched wsID", client?.wsID, wsID);
 						return;
 					}
+					ctx = this.contextManager.getConnectionContext(selector);
 					if (newClient) {
 						assert(newClient === client, "newClient must match existing client");
 						checkClientAndCVRVersions(client.version(), cvr.version);
@@ -452,12 +486,8 @@ var ViewSyncerService = class {
 					result = await fn(lc, clientID, body, cvr);
 				});
 			} catch (e) {
-				const lc = this.#lc.withContext("clientID", clientID).withContext("wsID", wsID).withContext("cmd", cmd);
-				lc[getLogLevel(e)]?.(`closing connection with error`, e);
-				if (isTransformAuthFailure(e)) {
-					lc.debug?.("Auth failure detected in transform response");
-					this.clearAuth();
-				}
+				this.#lc.withContext("clientID", clientID).withContext("wsID", wsID).withContext("cmd", cmd)[getLogLevel(e)]?.(`closing connection with error`, e);
+				if (ctx) this.contextManager.failConnection(selector, ctx.revision);
 				if (client) client.fail(e);
 				else throw e;
 			}
@@ -468,10 +498,10 @@ var ViewSyncerService = class {
 		const clients = [...this.#clients.values()];
 		return atVersion ? clients.filter((c) => cmpVersions(c.version() ?? EMPTY_CVR_VERSION, atVersion) === 0) : clients;
 	}
-	#handleConfigUpdate = (lc, clientID, { clientSchema, deleted, desiredQueriesPatch, activeClients }, cvr, customQueryTransformMode, profileID) => startAsyncSpan(tracer, "vs.#handleConfigUpdate", async () => {
+	#handleConfigUpdate = (lc, clientID, { clientSchema, deleted, desiredQueriesPatch, activeClients }, cvr, customQueryTransformMode, profileID, ctx) => startAsyncSpan(tracer, "vs.#handleConfigUpdate", async () => {
 		const deletedClientIDs = [];
 		const deletedClientGroupIDs = [];
-		cvr = await this.#updateCVRConfig(lc, cvr, clientID, customQueryTransformMode, (updater) => {
+		cvr = await this.#updateCVRConfig(lc, cvr, clientID, customQueryTransformMode, ctx, (updater) => {
 			const { ttlClock } = cvr;
 			const patches = [];
 			if (clientSchema) updater.setClientSchema(lc, clientSchema);
@@ -568,16 +598,17 @@ var ViewSyncerService = class {
 		let customHashMismatchCount = 0;
 		let otherHashMismatchCount = 0;
 		if (customQueries.size > 0 && !this.#customQueryTransformer) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
+		const backgroundContext = this.contextManager.mustGetBackgroundConnectionContext();
 		const customQueryTransformer = this.#customQueryTransformer;
 		if (customQueryTransformer && customQueries.size > 0) {
-			const transformedCustomQueries = await this.#runPriorityOp(lc, "#hydrateUnchangedQueries transforming custom queries", () => customQueryTransformer.transform(this.#getHeaderOptions(this.#queryConfig.forwardCookies), customQueries.values(), this.userQueryURL));
-			if (Array.isArray(transformedCustomQueries)) for (const q of transformedCustomQueries) if ("error" in q) customErrorCount++;
+			const transformedCustomQueries = await this.#runPriorityOp(lc, "#hydrateUnchangedQueries transforming custom queries", () => customQueryTransformer.transform(backgroundContext, customQueries.values()));
+			if (!transformedCustomQueries.cached) this.contextManager.validateConnection(backgroundContext, backgroundContext.revision);
+			if (Array.isArray(transformedCustomQueries.result)) for (const q of transformedCustomQueries.result) if ("error" in q) customErrorCount++;
 			else if (q.transformationHash !== customQueries.get(q.id)?.transformationHash) customHashMismatchCount++;
 			else transformedQueries.push(q);
 		}
 		for (const q of otherQueries) {
-			const auth = this.#authSession.auth;
-			const transformed = transformAndHashQuery(lc, q.id, q.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, auth?.type === "jwt" ? auth : void 0, q.type === "internal");
+			const transformed = transformAndHashQuery(lc, q.id, q.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, backgroundContext.auth?.type === "jwt" ? backgroundContext.auth : void 0, q.type === "internal");
 			if (transformed.transformationHash === q.transformationHash) transformedQueries.push(transformed);
 			else otherHashMismatchCount++;
 		}
@@ -650,8 +681,9 @@ var ViewSyncerService = class {
 	*
 	* This must be called from within the #lock.
 	*/
-	#syncQueryPipelineSet(lc, cvr, customQueryTransformMode) {
-		return startAsyncSpan(tracer, "vs.#syncQueryPipelineSet", async () => {
+	#syncQueryPipelineSet(lc, cvr, customQueryTransformMode, ctx) {
+		return startAsyncSpan(tracer, "vs.#syncQueryPipelineSet", async (span) => {
+			span.setAttribute("clientGroupID", this.id);
 			assert(this.#pipelines.initialized(), "pipelines must be initialized (syncQueryPipelineSet)");
 			if (this.#ttlClock === void 0) this.#ttlClock = cvr.ttlClock;
 			const now = Date.now();
@@ -660,6 +692,7 @@ var ViewSyncerService = class {
 			const customQueries = /* @__PURE__ */ new Map();
 			const otherQueries = [];
 			const transformedQueries = [];
+			const resolvedContext = ctx ?? this.contextManager.mustGetBackgroundConnectionContext();
 			for (const [id, query] of cvrQueryEntires) if (query.type === "custom") {
 				assert(id === query.id, "custom query id mismatch");
 				customQueries.set(id, query);
@@ -669,8 +702,7 @@ var ViewSyncerService = class {
 			});
 			for (const { id, query: origQuery } of otherQueries) {
 				assert(id === origQuery.id, "query id mismatch");
-				const auth = this.#authSession.auth;
-				const transformed = transformAndHashQuery(lc, origQuery.id, origQuery.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, auth?.type === "jwt" ? auth : void 0, origQuery.type === "internal");
+				const transformed = transformAndHashQuery(lc, origQuery.id, origQuery.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, resolvedContext.auth?.type === "jwt" ? resolvedContext.auth : void 0, origQuery.type === "internal");
 				transformedQueries.push({
 					id,
 					origQuery,
@@ -685,9 +717,12 @@ var ViewSyncerService = class {
 				const transformStart = performance.now();
 				let transformedCustomQueries;
 				try {
-					transformedCustomQueries = await this.#runPriorityOp(lc, "#syncQueryPipelineSet transforming custom queries", () => customQueryTransformer.transform(this.#getHeaderOptions(true), customQueriesToTransform, this.userQueryURL));
-					if (!Array.isArray(transformedCustomQueries) && transformedCustomQueries.kind === "TransformFailed") throw new ProtocolErrorWithLevel(transformedCustomQueries, "warn");
-					else this.#queryTransformations.add(1, { result: "success" });
+					transformedCustomQueries = await this.#runPriorityOp(lc, "#syncQueryPipelineSet transforming custom queries", () => customQueryTransformer.transform(resolvedContext, customQueriesToTransform));
+					if ("kind" in transformedCustomQueries.result) throw new ProtocolErrorWithLevel(transformedCustomQueries.result, "warn");
+					else {
+						if (!transformedCustomQueries.cached) this.contextManager.validateConnection(resolvedContext, resolvedContext.revision);
+						this.#queryTransformations.add(1, { result: "success" });
+					}
 				} catch (e) {
 					this.#queryTransformations.add(1, { result: "error" });
 					throw e;
@@ -696,7 +731,7 @@ var ViewSyncerService = class {
 					this.#queryTransformationTime.record(transformDuration);
 				}
 				const successfullyTransformedCustomQueries = /* @__PURE__ */ new Map();
-				erroredQueryIDs = this.#processTransformedCustomQueries(lc, transformedCustomQueries, (q) => {
+				erroredQueryIDs = this.#processTransformedCustomQueries(lc, transformedCustomQueries.result, (q) => {
 					const origQuery = customQueries.get(q.id);
 					if (origQuery) {
 						successfullyTransformedCustomQueries.set(q.id, q);
@@ -720,7 +755,7 @@ var ViewSyncerService = class {
 					}
 				}
 			}
-			const removeQueriesQueryIds = new Set(Object.values(cvr.queries).filter((q) => expired(ttlClock, q)).map((q) => q.id).concat(erroredQueryIDs || []));
+			const removeQueriesQueryIds = new Set([...Object.values(cvr.queries).filter((q) => expired(ttlClock, q)).map((q) => q.id), ...erroredQueryIDs || []]);
 			const addQueries = transformedQueries.map(({ id, transformed }) => ({
 				id,
 				ast: transformed.transformedAst,
@@ -731,7 +766,7 @@ var ViewSyncerService = class {
 				const orig = cvr.queries[q.id];
 				lc.debug?.("ViewSyncer adding query", q.ast, "transformed from", orig.type === "custom" ? orig.name : orig.ast);
 			}
-			if (addQueries.length > 0 || removeQueriesQueryIds.size > 0) await this.#addAndRemoveQueries(lc, cvr, addQueries, [...removeQueriesQueryIds].map((id) => ({ id })));
+			if (addQueries.length > 0 || removeQueriesQueryIds.size > 0) await this.#addAndRemoveQueries(lc, cvr, addQueries, Array.from(removeQueriesQueryIds, (id) => ({ id })));
 			else await this.#catchupClients(lc, cvr);
 		});
 	}
@@ -804,11 +839,13 @@ var ViewSyncerService = class {
 				hydrationTime.record(totalProcessTime / 1e3);
 			}
 			await this.#processChanges(lc, timer, generateRowChanges(this.#slowHydrateThreshold), updater, pokers);
-			for (const patch of await updater.deleteUnreferencedRows(lc)) await pokers.addPatch(patch);
+			await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.deleteUnreferencedRows", async () => {
+				for (const patch of await updater.deleteUnreferencedRows(lc)) await pokers.addPatch(patch);
+			});
 			this.#cvr = await this.#flushUpdater(lc, updater);
 			const finalVersion = this.#cvr.version;
 			await this.#catchupClients(lc, cvr, finalVersion, addQueries.map((q) => q.id), pokers);
-			await pokers.end(finalVersion);
+			await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.pokeEnd", () => pokers.end(finalVersion));
 			const wallTime = performance.now() - start;
 			lc.info?.(`finished processing queries (process: ${totalProcessTime} ms, wall: ${wallTime} ms)`);
 		});
@@ -889,7 +926,10 @@ var ViewSyncerService = class {
 				total += rows.size;
 				lc.debug?.(`processing ${rows.size} (of ${total}) rows (${wallElapsed} ms)`);
 				const patches = await updater.received(lc, rows);
-				for (const patch of patches) await pokers.addPatch(patch);
+				await startAsyncSpan(tracer, "processBatch.flushToClient", async (span) => {
+					span.setAttribute("patches", patches.length);
+					for (const patch of patches) await pokers.addPatch(patch);
+				});
 				rows.clear();
 			});
 			await startAsyncSpan(tracer, "loopingChanges", async (span) => {
@@ -944,7 +984,8 @@ var ViewSyncerService = class {
 	* Returns false if the advancement failed due to a schema change.
 	*/
 	#advancePipelines(lc, cvr) {
-		return startAsyncSpan(tracer, "vs.#advancePipelines", async () => {
+		return startAsyncSpan(tracer, "vs.#advancePipelines", async (span) => {
+			span.setAttribute("clientGroupID", this.id);
 			assert(this.#pipelines.initialized(), "pipelines must be initialized (advancePipelines");
 			const start = performance.now();
 			const timer = new TimeSliceTimer(lc);
@@ -964,7 +1005,7 @@ var ViewSyncerService = class {
 			}
 			this.#cvr = await this.#flushUpdater(lc, updater);
 			const finalVersion = this.#cvr.version;
-			await pokers.end(finalVersion);
+			await startAsyncSpan(tracer, "vs.#advancePipelines.pokeEnd", () => pokers.end(finalVersion));
 			const wallTime = performance.now() - start;
 			const totalProcessTime = timer.totalElapsed();
 			lc.info?.(`finished processing advancement of ${numChanges} changes ((process: ${totalProcessTime} ms, wall: ${wallTime} ms))`);
@@ -972,14 +1013,86 @@ var ViewSyncerService = class {
 			return "success";
 		});
 	}
-	async inspect(context, msg) {
-		await this.#runInLockForClient(context, msg, this.#handleInspect);
+	async inspect(selector, msg) {
+		await this.#runInLockForClient(selector, msg, this.#handleInspect);
 	}
 	#handleInspect = async (lc, clientID, body, cvr) => {
 		const client = must(this.#clients.get(clientID));
-		const auth = this.#authSession.auth;
-		return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this.#cvrStore, this.#config, this.#getHeaderOptions(this.#queryConfig.forwardCookies ?? false), this.userQueryURL, auth?.type === "jwt" ? auth : void 0);
+		const ctx = this.contextManager.mustGetConnectionContext({
+			clientID,
+			wsID: client.wsID
+		});
+		return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this.#cvrStore, this.#config, ctx);
 	};
+	async #runBackgroundRetransform(lc) {
+		const attemptRetransform = async (connection) => {
+			await this.#syncQueryPipelineSet(lc, must(this.#cvr, "cvr missing during auth maintenance retransform"), "all", connection);
+			this.contextManager.markBackgroundRetransformSuccess({
+				clientID: connection.clientID,
+				wsID: connection.wsID
+			}, connection.revision);
+		};
+		let backgroundConnection = this.contextManager.getBackgroundConnectionContext();
+		if (!backgroundConnection) {
+			lc.debug?.("Skipping background retransform with no selected connection");
+			return;
+		}
+		for (;;) {
+			try {
+				await attemptRetransform(backgroundConnection);
+				return;
+			} catch (e) {
+				if (isProtocolError(e)) {
+					if (isAuthErrorBody(e.errorBody)) {
+						lc.warn?.("Background retransform auth failed; failing connection and searching for replacement", {
+							clientID: backgroundConnection.clientID,
+							message: e.message
+						});
+						this.#failMaintenanceConnection(backgroundConnection, e);
+					} else if (isTransformFailedError(e)) {
+						lc.warn?.("Background retransform failed; deferring auth maintenance", {
+							clientID: backgroundConnection.clientID,
+							message: e.message
+						});
+						this.contextManager.deferMaintenance("retransform");
+						return;
+					}
+				} else throw e;
+			}
+			const replacement = this.contextManager.getBackgroundConnectionContext();
+			if (!replacement) {
+				lc.debug?.("No replacement connection available for background retransform");
+				return;
+			}
+			lc.debug?.("Retrying background retransform with replacement connection", {
+				clientID: replacement.clientID,
+				wsID: replacement.wsID
+			});
+			backgroundConnection = replacement;
+		}
+	}
+	async #validateConnection(ctx) {
+		try {
+			if (this.#customQueryTransformer) {
+				const validation = await this.#customQueryTransformer.validate(ctx);
+				if (validation !== void 0) throw new ProtocolErrorWithLevel(validation, "warn");
+			}
+			this.contextManager.validateConnection(ctx, ctx.revision);
+			return true;
+		} catch (e) {
+			if (isProtocolError(e) && isAuthErrorBody(e.errorBody)) {
+				this.#failMaintenanceConnection(ctx, e);
+				return false;
+			}
+			throw e;
+		}
+	}
+	#failMaintenanceConnection(ctx, error) {
+		if (!this.contextManager.failConnection(ctx, ctx.revision)) return;
+		const wrapped = wrapWithProtocolError(error);
+		const client = this.#clients.get(ctx.clientID);
+		if (client?.wsID === ctx.wsID) client.fail(wrapped);
+	}
 	stop() {
 		this.#lc.info?.("stopping view syncer");
 		this.#initialized.reject("shut down before initialization completed");
@@ -987,9 +1100,9 @@ var ViewSyncerService = class {
 		return this.#stopped.promise;
 	}
 	async #cleanup(err) {
-		this.clearAuth();
 		this.#stopTTLClockInterval();
 		this.#stopExpireTimer();
+		this.#stopAuthMaintenanceTimer();
 		for (const client of this.#clients.values()) if (err) client.fail(err);
 		else client.close(`closed clientGroupID=${this.id}`);
 		await this.#lock.withLock(() => {});
@@ -1025,9 +1138,8 @@ function checkClientAndCVRVersions(client, cvr) {
 		origin: ZeroCache
 	});
 }
-function isTransformAuthFailure(error) {
-	if (!isProtocolError(error)) return false;
-	return error.errorBody.kind === "TransformFailed" && error.errorBody.reason === "http" && (error.errorBody.status === 401 || error.errorBody.status === 403);
+function isTransformFailedError(error) {
+	return error.errorBody.kind === "TransformFailed" && !isAuthErrorBody(error.errorBody);
 }
 /**
 * A query must be expired for all clients in order to be considered