@rocicorp/zero 1.11.3 → 1.11.5

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -14,6 +14,7 @@ import { transformAndHashQuery } from "../../auth/read-authorizer.js";
 import { randInt } from "../../../../shared/src/rand.js";
 import { LogThrottle } from "../../observability/log-throttle.js";
 import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateNativeHistogram, getOrCreateUpDownCounter, getOrCreateValueHistogram } from "../../observability/metrics.js";
+import { clientGroupLogID } from "../../types/client-group-log-id.js";
 import { rowIDString } from "../../types/row-key.js";
 import "../replicator/schema/replication-state.js";
 import { planWarningMessage } from "./plan-warnings.js";
@@ -29,8 +30,8 @@ import { isAuthErrorBody } from "../../auth/auth.js";
 import { AdvancementMetrics } from "./advancement-metrics.js";
 import { ClientHandler, startPoke } from "./client-handler.js";
 import { tracer } from "./tracer.js";
-import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
-import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, classifyQueriesForHydration, nextEvictionTime } from "./cvr.js";
+import { CVRStore, ClientNotFoundError, PinnedUserMismatchError } from "./cvr-store.js";
+import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, CVRUpdater, classifyQueriesForHydration, nextEvictionTime } from "./cvr.js";
 import { E2EServingLagTracker } from "./e2e-serving-lag.js";
 import { HydrationBudget } from "./hydration-budget.js";
 import { HydrationCircuitBreaker } from "./hydration-circuit-breaker.js";
@@ -335,7 +336,7 @@ var ViewSyncerService = class {
 				}, "info");
 			}
 			if (await this.#checkForShutdownConditionsInLock()) {
-				this.#lc.info?.(`closing clientGroupID=${this.id}`);
+				this.#lc.info?.(`closing clientGroupID=${clientGroupLogID(this.id)}`);
 				this.#initialized.reject(shutdownBeforeInitializationError());
 				this.#stateChanges.cancel();
 				return;
@@ -355,6 +356,12 @@ var ViewSyncerService = class {
 				};
 			}
 			try {
+				if (reloaded && this.#cvr.pinnedUser !== null) try {
+					this.connContextManager.restorePinnedUser(this.#cvr.pinnedUser);
+				} catch (e) {
+					this.#stateChanges.fail(e instanceof Error ? e : new Error(String(e)));
+					throw e;
+				}
 				if (reloaded && this.#resetPipelinesIfBehindCVR(lc, this.#cvr)) {
 					const connCtx = this.connContextManager.getBackgroundConnectionContext();
 					if (connCtx) await this.#maybeHydratePipelines(lc, this.#cvr, connCtx);
@@ -362,6 +369,7 @@ var ViewSyncerService = class {
 				await fn(lc, this.#cvr);
 			} catch (e) {
 				this.#cvr = void 0;
+				if (e instanceof PinnedUserMismatchError) this.#stateChanges.fail(e);
 				throw e;
 			} finally {
 				this.#scheduleAuthMaintenance(lc);
@@ -410,13 +418,13 @@ var ViewSyncerService = class {
 		try {
 			this.#scheduleShutdown(this.#keepaliveMs);
 			if (await this.readyState() === "draining") {
-				this.#lc.debug?.(`draining view-syncer ${this.id} before running`);
+				this.#lc.debug?.(`draining view-syncer ${clientGroupLogID(this.id)} before running`);
 				this.stop();
 			}
 			for await (const replicaState of this.#stateChanges) {
 				const { state } = replicaState;
 				if (this.#drainCoordinator.shouldDrain()) {
-					this.#lc.debug?.(`draining view-syncer ${this.id} (elective)`);
+					this.#lc.debug?.(`draining view-syncer ${clientGroupLogID(this.id)} (elective)`);
 					break;
 				}
 				assert(state === "version-ready", "state should be version-ready");
@@ -460,11 +468,11 @@ var ViewSyncerService = class {
 			if (this.#drainCoordinator.shouldDrain()) this.#drainCoordinator.drainNextIn(this.#totalHydrationTimeMs());
 			await this.#cleanup();
 		} catch (e) {
-			this.#lc[getLogLevel(e)]?.(`stopping view-syncer ${this.id}: ${String(e)}`, e);
+			this.#lc[getLogLevel(e)]?.(`stopping view-syncer ${clientGroupLogID(this.id)}: ${String(e)}`, e);
 			await this.#cleanup(e);
 		} finally {
 			await this.#cvrStore.flushed(this.#lc).catch((e) => this.#lc[getLogLevel(e)]?.(e));
-			this.#lc.info?.(`view-syncer ${this.id} finished`);
+			this.#lc.info?.(`view-syncer ${clientGroupLogID(this.id)} finished`);
 			this.#stopped.resolve();
 		}
 	}
@@ -632,11 +640,12 @@ var ViewSyncerService = class {
 		return this.#clients.size === 0;
 	}
 	#deleteClientDueToDisconnect(clientID, client) {
-		this.connContextManager.closeConnection({
+		const c = this.#clients.get(clientID);
+		if (c === client || c?.wsID !== client.wsID) this.connContextManager.closeConnection({
 			clientID,
 			wsID: client.wsID
 		});
-		if (this.#clients.get(clientID) === client) {
+		if (c === client) {
 			this.#clients.delete(clientID);
 			if (this.#clients.size === 0) {
 				if (this.#ttlClock !== void 0) this.#updateTTLClockInCVRWithoutLock(this.#lc);
@@ -725,16 +734,19 @@ var ViewSyncerService = class {
 				this.#ttlClockBase = now;
 			}
 			const newClient = new ClientHandler(lc, this.id, connCtx.clientID, connCtx.wsID, this.#shard, connCtx.baseCookie, downstream);
-			this.#clients.get(connCtx.clientID)?.close(`replaced by wsID: ${connCtx.wsID}`);
+			const replaced = this.#clients.get(connCtx.clientID);
 			this.#clients.set(connCtx.clientID, newClient);
+			replaced?.close(`replaced by wsID: ${connCtx.wsID}`);
 			startAsyncSpan(tracer, "vs.initConnection.async", () => this.#runInLockForClient(connCtx, initConnectionMessage, async (lc, clientID, msg, cvr) => {
 				if (cvr.clientSchema === null && !msg.clientSchema) throw new ProtocolErrorWithLevel({
 					kind: InvalidConnectionRequest,
 					message: "The initConnection message for a new client group must include client schema.",
 					origin: ZeroCache
 				}, "warn");
-				if (!await this.#validateConnection(connCtx)) return;
-				await this.#handleConfigUpdate(lc, clientID, msg, cvr, "all", connCtx.profileID ?? `cg${this.id}`, connCtx);
+				const validatedConnCtx = await this.#validateCurrentConnection(connCtx);
+				if (!validatedConnCtx) return;
+				newClient.markValidated();
+				await this.#handleConfigUpdate(lc, clientID, msg, cvr, "all", validatedConnCtx.profileID ?? `cg${this.id}`, validatedConnCtx);
 				this.#initialized.resolve("initialized");
 			}, newClient)).catch((e) => newClient.fail(e));
 			return downstream;
@@ -750,11 +762,9 @@ var ViewSyncerService = class {
 				return;
 			}
 			lc.debug?.("Auth changed, re-validating and re-transforming queries");
-			const connCtx = this.connContextManager.mustGetConnectionContext(selector);
-			if (!this.#pipelinesHydrated) {
-				if (!await this.#validateConnection(connCtx)) return;
-			}
-			return await this.#handleConfigUpdate(lc, clientID, {}, cvr, "all", void 0, connCtx);
+			const validatedConnCtx = await this.#validateCurrentConnection(selector);
+			if (!validatedConnCtx) return;
+			return await this.#handleConfigUpdate(lc, clientID, {}, cvr, "all", void 0, validatedConnCtx);
 		});
 	}
 	async deleteClients(selector, msg) {
@@ -779,6 +789,8 @@ var ViewSyncerService = class {
 	*/
 	#flushUpdater(lc, updater, patchesPoked = false) {
 		return startAsyncSpan(tracer, "vs.#flushUpdater", () => this.#runPriorityOp(lc, "flushing cvr", async () => {
+			const { pinnedUser } = this.connContextManager.getGroupState();
+			if (pinnedUser !== void 0) updater.setPinnedUser(pinnedUser);
 			const now = Date.now();
 			const ttlClock = this.#getTTLClock(now);
 			const { cvr, flushed } = await updater.flush(lc, this.#lastConnectTime, now, ttlClock, patchesPoked);
@@ -851,7 +863,7 @@ var ViewSyncerService = class {
 		const [cmd, body] = msg;
 		if (newClient || !this.#clients.has(clientID)) this.#lastConnectTime = Date.now();
 		return startAsyncSpan(tracer, `vs.#runInLockForClient(${cmd})`, async (span) => {
-			span.setAttribute("clientGroupID", this.id);
+			span.setAttribute("clientGroupID", clientGroupLogID(this.id));
 			span.setAttribute("clientID", clientID);
 			let client;
 			let result;
@@ -861,31 +873,44 @@ var ViewSyncerService = class {
 					lc = lc.withContext("clientID", clientID).withContext("wsID", wsID).withContext("cmd", cmd);
 					lc.debug?.("acquired lock for cvr");
 					client = this.#clients.get(clientID);
-					if (client?.wsID !== wsID) {
-						lc.debug?.("mismatched wsID", client?.wsID, wsID);
+					if (client?.wsID !== wsID || newClient && client !== newClient) {
+						lc.debug?.("mismatched or superseded client", client?.wsID, wsID);
 						return;
 					}
 					connCtx = this.connContextManager.getConnectionContext(selector);
-					if (newClient) {
-						assert(newClient === client, "newClient must match existing client");
-						checkClientAndCVRVersions(client.version(), cvr.version);
-					} else if (!this.#clients.has(clientID)) lc.warn?.(`Processing ${cmd} before initConnection was received`);
+					if (newClient) checkClientAndCVRVersions(client.version(), cvr.version);
+					else if (!this.#clients.has(clientID)) lc.warn?.(`Processing ${cmd} before initConnection was received`);
 					lc.debug?.(cmd, body);
 					result = await fn(lc, clientID, body, cvr);
 				});
 			} catch (e) {
 				this.#lc.withContext("clientID", clientID).withContext("wsID", wsID).withContext("cmd", cmd)[getLogLevel(e)]?.(`closing connection with error`, e);
-				if (connCtx) this.connContextManager.failConnection(selector, connCtx.revision);
+				const current = connCtx && this.#currentRegistration(connCtx);
+				if (current) this.connContextManager.failConnection(selector, current.revision);
 				if (client) client.fail(e);
 				else throw e;
 			}
 			return result;
 		});
 	}
+	/**
+	* The clients that pokes and catch-up are sent to. A client whose
+	* connection has not been validated yet is left out: its `initConnection`
+	* validates it and then catches it up from its baseCookie, so lock-held
+	* work for other clients that runs in the meantime must not send it data.
+	*/
 	#getClients(atVersion) {
-		const clients = [...this.#clients.values()];
+		const clients = [...this.#clients.values()].filter((c) => c.validated());
 		return atVersion ? clients.filter((c) => cmpVersions(c.version() ?? EMPTY_CVR_VERSION, atVersion) === 0) : clients;
 	}
+	/**
+	* The client to send a targeted message (e.g. a query error) to, or
+	* `undefined` while its connection is still unvalidated; see #getClients().
+	*/
+	#getValidatedClient(clientID) {
+		const client = this.#clients.get(clientID);
+		return client?.validated() ? client : void 0;
+	}
 	#handleConfigUpdate = (lc, clientID, { clientSchema, deleted, desiredQueriesPatch, activeClients }, cvr, customQueryTransformMode, profileID, connCtx) => startAsyncSpan(tracer, "vs.#handleConfigUpdate", async () => {
 		const deletedClientIDs = [];
 		const deletedClientGroupIDs = [];
@@ -1112,12 +1137,12 @@ var ViewSyncerService = class {
 			return clientIds;
 		};
 		if ("queryIDs" in errorOrErrors) {
-			for (const clientId of getAffectedClientIDs(errorOrErrors.queryIDs)) this.#clients.get(clientId)?.sendQueryTransformFailedError(errorOrErrors);
+			for (const clientId of getAffectedClientIDs(errorOrErrors.queryIDs)) this.#getValidatedClient(clientId)?.sendQueryTransformFailedError(errorOrErrors);
 			return;
 		}
 		const appErrorGroups = /* @__PURE__ */ new Map();
 		for (const err of errorOrErrors) for (const clientId of getAffectedClientIDs([err.id])) getOrInsertComputed(appErrorGroups, clientId, newArray).push(err);
-		for (const [clientId, errors] of appErrorGroups) this.#clients.get(clientId)?.sendQueryTransformApplicationErrors(errors);
+		for (const [clientId, errors] of appErrorGroups) this.#getValidatedClient(clientId)?.sendQueryTransformApplicationErrors(errors);
 	}
 	#addQueryMaterializationServerMetric(queryID, elapsed) {
 		this.#inspectorDelegate.addMetric("query-materialization-server", elapsed, queryID);
@@ -1136,7 +1161,7 @@ var ViewSyncerService = class {
 	}
 	#logQueryCoverageShadowSummary(lc, hydrationPath, totalHydratedQueries, coveredHydratedQueries, firstCoveredQuery) {
 		if (!this.#config.enableQueryCovering || totalHydratedQueries === 0) return;
-		let coverageLC = lc.withContext("appID", this.#shard.appID).withContext("shardNum", this.#shard.shardNum).withContext("clientGroupID", this.id).withContext("queryCoverageMode", "shadow").withContext("hydrationPath", hydrationPath).withContext("totalHydratedQueries", totalHydratedQueries).withContext("coveredHydratedQueries", coveredHydratedQueries).withContext("uncoveredHydratedQueries", totalHydratedQueries - coveredHydratedQueries);
+		let coverageLC = lc.withContext("appID", this.#shard.appID).withContext("shardNum", this.#shard.shardNum).withContext("clientGroupID", clientGroupLogID(this.id)).withContext("queryCoverageMode", "shadow").withContext("hydrationPath", hydrationPath).withContext("totalHydratedQueries", totalHydratedQueries).withContext("coveredHydratedQueries", coveredHydratedQueries).withContext("uncoveredHydratedQueries", totalHydratedQueries - coveredHydratedQueries);
 		if (firstCoveredQuery) {
 			coverageLC = coverageLC.withContext("firstCoveredQueryHash", firstCoveredQuery.coveredQueryHash).withContext("firstCoveredTransformationHash", firstCoveredQuery.coveredTransformationHash).withContext("firstCoveringQueryHash", firstCoveredQuery.coveringQueryHash).withContext("firstCoveringTransformationHash", firstCoveredQuery.coveringTransformationHash);
 			if (firstCoveredQuery.coveredQueryName !== void 0) coverageLC = coverageLC.withContext("firstCoveredQueryName", firstCoveredQuery.coveredQueryName);
@@ -1158,7 +1183,7 @@ var ViewSyncerService = class {
 	}, previousQueries) {
 		return startAsyncSpan(tracer, "vs.#syncQueryPipelineSet", async (span) => {
 			const start = performance.now();
-			span.setAttribute("clientGroupID", this.id);
+			span.setAttribute("clientGroupID", clientGroupLogID(this.id));
 			assert(this.#pipelines.initialized(), "pipelines must be initialized (syncQueryPipelineSet)");
 			if (this.#ttlClock === void 0) this.#ttlClock = cvr.ttlClock;
 			const now = Date.now();
@@ -1328,7 +1353,7 @@ var ViewSyncerService = class {
 		if (!lc.warn) return;
 		const shape = queryShape(query.ast);
 		lc.warn("Query hydration aborted for exceeding the hydration timeout", {
-			clientGroupID: this.id,
+			clientGroupID: clientGroupLogID(this.id),
 			queryHash: query.id,
 			transformationHash: query.transformationHash,
 			...query.name !== void 0 && { queryName: query.name },
@@ -1356,7 +1381,7 @@ var ViewSyncerService = class {
 		if (suppressed === void 0) return;
 		lc.warn("Slow query materialization", {
 			zeroEvent: "query-slow-hydration",
-			clientGroupID: this.id,
+			clientGroupID: clientGroupLogID(this.id),
 			queryHash: query.id,
 			transformationHash: query.transformationHash,
 			...query.name !== void 0 && { queryName: query.name },
@@ -1375,7 +1400,7 @@ var ViewSyncerService = class {
 	#recordCircuitBreakerRejection(lc, query) {
 		this.#hydrationCircuitBreakerRejections.add(1);
 		lc.warn?.("Query rejected by its open hydration circuit breaker", {
-			clientGroupID: this.id,
+			clientGroupID: clientGroupLogID(this.id),
 			queryHash: query.id,
 			transformationHash: query.transformationHash,
 			...query.name !== void 0 && { queryName: query.name },
@@ -1417,7 +1442,7 @@ var ViewSyncerService = class {
 			};
 			for (const clientID of Object.keys(query.clientState)) getOrInsertComputed(errorsByClient, clientID, newArray).push(error);
 		}
-		for (const [clientID, errors] of errorsByClient) this.#clients.get(clientID)?.sendQueryTransformApplicationErrors(errors);
+		for (const [clientID, errors] of errorsByClient) this.#getValidatedClient(clientID)?.sendQueryTransformApplicationErrors(errors);
 	}
 	#recordHydrationBudgetExhaustion(lc, hydrationBudget, activeHydratedQueries, inactiveHydratedQueries, inactiveEvictedQueryIDs) {
 		const elapsedMs = hydrationBudget.exhaustedAtMs;
@@ -1428,7 +1453,7 @@ var ViewSyncerService = class {
 		this.#hydrationBudgetOvershoot.record(Math.max(0, elapsedMs - hydrationBudget.limitMs));
 		if (inactiveEvictedQueryIDs.length > 0) this.#queryEvictions.add(inactiveEvictedQueryIDs.length, { reason: "hydration-budget" });
 		lc.info?.("view-syncer hydration budget exhausted", {
-			clientGroupID: this.id,
+			clientGroupID: clientGroupLogID(this.id),
 			hydrationBudgetMs: hydrationBudget.limitMs,
 			hydrationElapsedMs: elapsedMs,
 			activeHydratedQueries,
@@ -1620,7 +1645,8 @@ var ViewSyncerService = class {
 				if (!row.refCounts) patch = {
 					type: "row",
 					op: "del",
-					id
+					id,
+					catchup: true
 				};
 				else {
 					const { contents } = contentsAndVersion(must(this.#pipelines.getRow(table, rowKey), `Missing row in ${table} keyed by ${Object.keys(rowKey).join()}`));
@@ -1718,7 +1744,7 @@ var ViewSyncerService = class {
 	*/
 	#advancePipelines(lc, cvr) {
 		return startAsyncSpan(tracer, "vs.#advancePipelines", async (span) => {
-			span.setAttribute("clientGroupID", this.id);
+			span.setAttribute("clientGroupID", clientGroupLogID(this.id));
 			assert(this.#pipelines.initialized(), "pipelines must be initialized (advancePipelines");
 			const start = performance.now();
 			const timer = this.#timer(lc, "batch");
@@ -1938,7 +1964,7 @@ var ViewSyncerService = class {
 		const now = this.#now();
 		if (log.hydrationTimeMs < REBUILD_LOG_MIN_MS || now - this.#rebuildsLoggedAt < REBUILD_LOG_INTERVAL_MS) return;
 		lc.info?.("Client group hydrated queries again in its batches of changes instead of pushing the changes to them", {
-			clientGroupID: this.id,
+			clientGroupID: clientGroupLogID(this.id),
 			...log
 		});
 		this.#rebuildsLoggedAt = now;
@@ -2006,7 +2032,7 @@ var ViewSyncerService = class {
 		const now = this.#now();
 		if (log.waitedMs < BATCH_WAIT_LOG_MIN_MS || now - log.loggedAt < BATCH_WAIT_LOG_INTERVAL_MS) return;
 		this.#lc.info?.("Client group is waiting between batches: its sync worker is over its share target", {
-			clientGroupID: this.id,
+			clientGroupID: clientGroupLogID(this.id),
 			waits: log.waits,
 			waitedMs: log.waitedMs,
 			lastWaitMs: waitedMs,
@@ -2028,7 +2054,7 @@ var ViewSyncerService = class {
 			clientID,
 			wsID: client.wsID
 		});
-		return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this, this.#cvrStore, this.#config, connCtx);
+		return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this.#cvrStore, this.#config, connCtx);
 	};
 	async #runBackgroundRetransform(lc) {
 		const attemptRetransform = async (connCtx) => {
@@ -2085,7 +2111,8 @@ var ViewSyncerService = class {
 				if (response.kind === "TransformFailed") throw new ProtocolErrorWithLevel(response, "warn");
 				validation = response.validation;
 			} else validation = { kind: "client-fallback" };
-			this.connContextManager.validateConnection(connCtx, connCtx.revision, validation);
+			if (!this.#currentRegistration(connCtx)) return false;
+			if (this.connContextManager.validateConnection(connCtx, connCtx.revision, validation) && this.#cvr?.pinnedUser === null) this.#cvr = await this.#flushUpdater(this.#lc, new CVRUpdater(this.#cvrStore, this.#cvr, this.#cvr.replicaVersion));
 			return true;
 		} catch (e) {
 			if (isProtocolError(e) && isAuthErrorBody(e.errorBody)) {
@@ -2095,12 +2122,38 @@ var ViewSyncerService = class {
 					revision: connCtx.revision,
 					message: e.message
 				});
-				this.#failMaintenanceConnection(connCtx, e);
+				if (this.#currentRegistration(connCtx)) this.#failMaintenanceConnection(connCtx, e);
 				return false;
 			}
 			throw e;
 		}
 	}
+	/**
+	* The live context registered for `connCtx`'s selector, or `undefined` if
+	* that socket is gone or has been replaced by one reusing the selector
+	* (which starts over at revision 0, so revisions alone cannot tell).
+	*/
+	#currentRegistration(connCtx) {
+		const current = this.connContextManager.getConnectionContext(connCtx);
+		return current?.insertionOrder === connCtx.insertionOrder ? current : void 0;
+	}
+	/**
+	* Validates the connection's credential as it stands once validation ends.
+	* An `updateAuth` handled while a validation is in flight moves the
+	* connection to a newer revision, which makes a failure of the older one
+	* stale, as is its success; the newer credential is then validated in its
+	* place, so the `initConnection` waiting on it is not dropped.
+	*/
+	async #validateCurrentConnection(selector) {
+		let connCtx = this.connContextManager.getConnectionContext(selector);
+		while (connCtx) {
+			const valid = await this.#validateConnection(connCtx);
+			const current = this.#currentRegistration(connCtx);
+			if (!current) return;
+			if (current.revision === connCtx.revision) return valid ? connCtx : void 0;
+			connCtx = current;
+		}
+	}
 	#failMaintenanceConnection(connCtx, error) {
 		if (!this.connContextManager.failConnection(connCtx, connCtx.revision)) return;
 		const wrapped = wrapWithProtocolError(error);
@@ -2124,10 +2177,9 @@ var ViewSyncerService = class {
 		this.#stopShutdownTimer();
 		this.#customQueryTransformer?.destroy();
 		for (const client of this.#clients.values()) if (err) client.fail(err);
-		else client.close(`closed clientGroupID=${this.id}`);
+		else client.close(`closed clientGroupID=${clientGroupLogID(this.id)}`);
 		await this.#lock.withLock(() => {});
 		this.#pipelines.destroy();
-		this.#inspectorDelegate.clearAuthenticated(this.id, this);
 	}
 	/**
 	* Test helper: Manually mark initialization as complete.