@rocicorp/zero 1.12.0-canary.12 → 1.12.0-canary.13

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -17,6 +17,7 @@ import { ZERO_VERSION_COLUMN_NAME } from "../replicator/schema/constants.js";
 import { rowIDString } from "../../types/row-key.js";
 import "../replicator/schema/replication-state.js";
 import { ProtocolErrorWithLevel, getLogLevel, wrapWithProtocolError } from "../../types/error-with-level.js";
+import { clientGroupLogID } from "../../types/client-group-log-id.js";
 import { Subscription } from "../../types/subscription.js";
 import { CustomKeyMap } from "../../../../shared/src/custom-key-map.js";
 import { ttlClockAsNumber, ttlClockFromNumber } from "./ttl-clock.js";
@@ -30,8 +31,8 @@ import { ResetPipelinesSignal } from "./snapshotter.js";
 import { AdvancementMetrics } from "./advancement-metrics.js";
 import { ClientHandler, startPoke } from "./client-handler.js";
 import { tracer } from "./tracer.js";
-import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
-import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, classifyQueriesForHydration, nextEvictionTime } from "./cvr.js";
+import { CVRStore, ClientNotFoundError, PinnedUserMismatchError } from "./cvr-store.js";
+import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, CVRUpdater, classifyQueriesForHydration, nextEvictionTime } from "./cvr.js";
 import { E2EServingLagTracker } from "./e2e-serving-lag.js";
 import { HydrationBudget } from "./hydration-budget.js";
 import { HydrationCircuitBreaker } from "./hydration-circuit-breaker.js";
@@ -147,8 +148,8 @@ var ViewSyncerService = class {
 	*   updates are processed via `#syncQueryPipelineSet`.
 	*
 	* Resets to `false` if `#advancePipelines` returns a `ResetPipelinesSignal`
-	* and pipelines must be reset and rehydrated, or if a reloaded CVR is ahead
-	* of the pipelines (see `#resetPipelinesIfBehindCVR`).
+	* and pipelines must be reset and rehydrated, or if the CVR is reloaded
+	* (see `#resetPipelinesForReloadedCVR`).
 	*/
 	#pipelinesHydrated = false;
 	#servedVersion = null;
@@ -335,7 +336,7 @@ var ViewSyncerService = class {
 				}, "info");
 			}
 			if (await this.#checkForShutdownConditionsInLock()) {
-				this.#lc.info?.(`closing clientGroupID=${this.id}`);
+				this.#lc.info?.(`closing clientGroupID=${clientGroupLogID(this.id)}`);
 				this.#initialized.reject(shutdownBeforeInitializationError());
 				this.#stateChanges.cancel();
 				return;
@@ -355,13 +356,20 @@ var ViewSyncerService = class {
 				};
 			}
 			try {
-				if (reloaded && this.#resetPipelinesIfBehindCVR(lc, this.#cvr)) {
+				if (reloaded && this.#cvr.pinnedUser !== null) try {
+					this.connContextManager.restorePinnedUser(this.#cvr.pinnedUser);
+				} catch (e) {
+					this.#stateChanges.fail(e instanceof Error ? e : new Error(String(e)));
+					throw e;
+				}
+				if (reloaded && this.#resetPipelinesForReloadedCVR(lc, this.#cvr)) {
 					const connCtx = this.connContextManager.getBackgroundConnectionContext();
 					if (connCtx) await this.#maybeHydratePipelines(lc, this.#cvr, connCtx);
 				}
 				await fn(lc, this.#cvr);
 			} catch (e) {
 				this.#cvr = void 0;
+				if (e instanceof PinnedUserMismatchError) this.#stateChanges.fail(e);
 				throw e;
 			} finally {
 				this.#scheduleAuthMaintenance(lc);
@@ -369,26 +377,29 @@ var ViewSyncerService = class {
 		});
 	}
 	/**
-	* The CVR is reloaded after an error clears the cached copy. By then another
-	* view-syncer may have taken over the client group and flushed the CVR at a
-	* version ahead of this instance's hydrated pipelines (e.g. its replica was
-	* further ahead). The pipelines can no longer be diffed against the CVR, so
-	* reset them and let `#maybeHydratePipelines` rehydrate once the replica has
-	* caught up to the CVR.
+	* The CVR is reloaded after an error clears the cached copy. The pipelines
+	* may no longer correspond to it:
+	*
+	* - Another view-syncer may have taken over the client group and flushed
+	*   the CVR, adding or removing queries (e.g. evicting them when its
+	*   hydration budget ran out), or at a version ahead of the pipelines.
+	* - The failed operation may have changed the pipelines before the CVR
+	*   flush that would have recorded the change, e.g. a hydration that failed
+	*   after some of its queries were added.
 	*
-	* A CVR behind the pipelines is expected (advancements that do not change
-	* the CVR are not flushed) and is handled by the normal update path.
+	* Rather than reconcile them, reset the pipelines and let
+	* `#maybeHydratePipelines` rehydrate them from the reloaded CVR once the
+	* replica has caught up to it, as when a view-syncer takes over a client
+	* group.
 	*
 	* Returns whether the pipelines were reset.
 	*
 	* Must be called from within the #lock.
 	*/
-	#resetPipelinesIfBehindCVR(lc, cvr) {
+	#resetPipelinesForReloadedCVR(lc, cvr) {
 		if (!this.#pipelinesHydrated) return false;
-		const pipelineVersion = this.#pipelines.currentVersion();
-		if (pipelineVersion >= cvr.version.stateVersion) return false;
-		lc.info?.(`resetting pipelines: pipelines@${pipelineVersion} are behind reloaded cvr@${versionString(cvr.version)}`);
-		this.#pipelineResets.add(1, { reason: "behind-cvr" });
+		lc.info?.(`resetting pipelines@${this.#pipelines.currentVersion()} for reloaded cvr@${versionString(cvr.version)}`);
+		this.#pipelineResets.add(1, { reason: "cvr-reloaded" });
 		this.#pipelinesHydrated = false;
 		this.connContextManager.setSharedRetransformReady(false);
 		this.#pipelines.reset(must(cvr.clientSchema, "cvr.clientSchema missing after initialization"));
@@ -410,13 +421,13 @@ var ViewSyncerService = class {
 		try {
 			this.#scheduleShutdown(this.#keepaliveMs);
 			if (await this.readyState() === "draining") {
-				this.#lc.debug?.(`draining view-syncer ${this.id} before running`);
+				this.#lc.debug?.(`draining view-syncer ${clientGroupLogID(this.id)} before running`);
 				this.stop();
 			}
 			for await (const replicaState of this.#stateChanges) {
 				const { state } = replicaState;
 				if (this.#drainCoordinator.shouldDrain()) {
-					this.#lc.debug?.(`draining view-syncer ${this.id} (elective)`);
+					this.#lc.debug?.(`draining view-syncer ${clientGroupLogID(this.id)} (elective)`);
 					break;
 				}
 				assert(state === "version-ready", "state should be version-ready");
@@ -460,11 +471,11 @@ var ViewSyncerService = class {
 			if (this.#drainCoordinator.shouldDrain()) this.#drainCoordinator.drainNextIn(this.#totalHydrationTimeMs());
 			await this.#cleanup();
 		} catch (e) {
-			this.#lc[getLogLevel(e)]?.(`stopping view-syncer ${this.id}: ${String(e)}`, e);
+			this.#lc[getLogLevel(e)]?.(`stopping view-syncer ${clientGroupLogID(this.id)}: ${String(e)}`, e);
 			await this.#cleanup(e);
 		} finally {
 			await this.#cvrStore.flushed(this.#lc).catch((e) => this.#lc[getLogLevel(e)]?.(e));
-			this.#lc.info?.(`view-syncer ${this.id} finished`);
+			this.#lc.info?.(`view-syncer ${clientGroupLogID(this.id)} finished`);
 			this.#stopped.resolve();
 		}
 	}
@@ -740,6 +751,7 @@ var ViewSyncerService = class {
 				}, "warn");
 				const validatedConnCtx = await this.#validateCurrentConnection(connCtx);
 				if (!validatedConnCtx) return;
+				newClient.markValidated();
 				await this.#handleConfigUpdate(lc, clientID, msg, cvr, "all", validatedConnCtx.profileID ?? `cg${this.id}`, validatedConnCtx);
 				this.#initialized.resolve("initialized");
 			}, newClient)).catch((e) => newClient.fail(e));
@@ -756,11 +768,9 @@ var ViewSyncerService = class {
 				return;
 			}
 			lc.debug?.("Auth changed, re-validating and re-transforming queries");
-			const connCtx = this.connContextManager.mustGetConnectionContext(selector);
-			if (!this.#pipelinesHydrated) {
-				if (!await this.#validateConnection(connCtx)) return;
-			}
-			return await this.#handleConfigUpdate(lc, clientID, {}, cvr, "all", void 0, connCtx);
+			const validatedConnCtx = await this.#validateCurrentConnection(selector);
+			if (!validatedConnCtx) return;
+			return await this.#handleConfigUpdate(lc, clientID, {}, cvr, "all", void 0, validatedConnCtx);
 		});
 	}
 	async deleteClients(selector, msg) {
@@ -785,6 +795,8 @@ var ViewSyncerService = class {
 	*/
 	#flushUpdater(lc, updater, patchesPoked = false) {
 		return startAsyncSpan(tracer, "vs.#flushUpdater", () => this.#runPriorityOp(lc, "flushing cvr", async () => {
+			const { pinnedUser } = this.connContextManager.getGroupState();
+			if (pinnedUser !== void 0) updater.setPinnedUser(pinnedUser);
 			const now = Date.now();
 			const ttlClock = this.#getTTLClock(now);
 			const { cvr, flushed } = await updater.flush(lc, this.#lastConnectTime, now, ttlClock, patchesPoked);
@@ -863,7 +875,7 @@ var ViewSyncerService = class {
 		const [cmd, body] = msg;
 		if (newClient || !this.#clients.has(clientID)) this.#lastConnectTime = Date.now();
 		return startAsyncSpan(tracer, `vs.#runInLockForClient(${cmd})`, async (span) => {
-			span.setAttribute("clientGroupID", this.id);
+			span.setAttribute("clientGroupID", clientGroupLogID(this.id));
 			span.setAttribute("clientID", clientID);
 			let client;
 			let result;
@@ -885,17 +897,32 @@ var ViewSyncerService = class {
 				});
 			} catch (e) {
 				this.#lc.withContext("clientID", clientID).withContext("wsID", wsID).withContext("cmd", cmd)[getLogLevel(e)]?.(`closing connection with error`, e);
-				if (connCtx) this.connContextManager.failConnection(selector, connCtx.revision);
+				const current = connCtx && this.#currentRegistration(connCtx);
+				if (current) this.connContextManager.failConnection(selector, current.revision);
 				if (client) client.fail(e);
 				else throw e;
 			}
 			return result;
 		});
 	}
+	/**
+	* The clients that pokes and catch-up are sent to. A client whose
+	* connection has not been validated yet is left out: its `initConnection`
+	* validates it and then catches it up from its baseCookie, so lock-held
+	* work for other clients that runs in the meantime must not send it data.
+	*/
 	#getClients(atVersion) {
-		const clients = [...this.#clients.values()];
+		const clients = [...this.#clients.values()].filter((c) => c.validated());
 		return atVersion ? clients.filter((c) => cmpVersions(c.version() ?? EMPTY_CVR_VERSION, atVersion) === 0) : clients;
 	}
+	/**
+	* The client to send a targeted message (e.g. a query error) to, or
+	* `undefined` while its connection is still unvalidated; see #getClients().
+	*/
+	#getValidatedClient(clientID) {
+		const client = this.#clients.get(clientID);
+		return client?.validated() ? client : void 0;
+	}
 	#handleConfigUpdate = (lc, clientID, { clientSchema, deleted, desiredQueriesPatch, activeClients }, cvr, customQueryTransformMode, profileID, connCtx) => startAsyncSpan(tracer, "vs.#handleConfigUpdate", async () => {
 		const deletedClientIDs = [];
 		const deletedClientGroupIDs = [];
@@ -1122,12 +1149,12 @@ var ViewSyncerService = class {
 			return clientIds;
 		};
 		if ("queryIDs" in errorOrErrors) {
-			for (const clientId of getAffectedClientIDs(errorOrErrors.queryIDs)) this.#clients.get(clientId)?.sendQueryTransformFailedError(errorOrErrors);
+			for (const clientId of getAffectedClientIDs(errorOrErrors.queryIDs)) this.#getValidatedClient(clientId)?.sendQueryTransformFailedError(errorOrErrors);
 			return;
 		}
 		const appErrorGroups = /* @__PURE__ */ new Map();
 		for (const err of errorOrErrors) for (const clientId of getAffectedClientIDs([err.id])) getOrInsertComputed(appErrorGroups, clientId, newArray).push(err);
-		for (const [clientId, errors] of appErrorGroups) this.#clients.get(clientId)?.sendQueryTransformApplicationErrors(errors);
+		for (const [clientId, errors] of appErrorGroups) this.#getValidatedClient(clientId)?.sendQueryTransformApplicationErrors(errors);
 	}
 	#addQueryMaterializationServerMetric(queryID, elapsed) {
 		this.#inspectorDelegate.addMetric("query-materialization-server", elapsed, queryID);
@@ -1146,7 +1173,7 @@ var ViewSyncerService = class {
 	}
 	#logQueryCoverageShadowSummary(lc, hydrationPath, totalHydratedQueries, coveredHydratedQueries, firstCoveredQuery) {
 		if (!this.#config.enableQueryCovering || totalHydratedQueries === 0) return;
-		let coverageLC = lc.withContext("appID", this.#shard.appID).withContext("shardNum", this.#shard.shardNum).withContext("clientGroupID", this.id).withContext("queryCoverageMode", "shadow").withContext("hydrationPath", hydrationPath).withContext("totalHydratedQueries", totalHydratedQueries).withContext("coveredHydratedQueries", coveredHydratedQueries).withContext("uncoveredHydratedQueries", totalHydratedQueries - coveredHydratedQueries);
+		let coverageLC = lc.withContext("appID", this.#shard.appID).withContext("shardNum", this.#shard.shardNum).withContext("clientGroupID", clientGroupLogID(this.id)).withContext("queryCoverageMode", "shadow").withContext("hydrationPath", hydrationPath).withContext("totalHydratedQueries", totalHydratedQueries).withContext("coveredHydratedQueries", coveredHydratedQueries).withContext("uncoveredHydratedQueries", totalHydratedQueries - coveredHydratedQueries);
 		if (firstCoveredQuery) {
 			coverageLC = coverageLC.withContext("firstCoveredQueryHash", firstCoveredQuery.coveredQueryHash).withContext("firstCoveredTransformationHash", firstCoveredQuery.coveredTransformationHash).withContext("firstCoveringQueryHash", firstCoveredQuery.coveringQueryHash).withContext("firstCoveringTransformationHash", firstCoveredQuery.coveringTransformationHash);
 			if (firstCoveredQuery.coveredQueryName !== void 0) coverageLC = coverageLC.withContext("firstCoveredQueryName", firstCoveredQuery.coveredQueryName);
@@ -1168,7 +1195,7 @@ var ViewSyncerService = class {
 	}, previousQueries) {
 		return startAsyncSpan(tracer, "vs.#syncQueryPipelineSet", async (span) => {
 			const start = performance.now();
-			span.setAttribute("clientGroupID", this.id);
+			span.setAttribute("clientGroupID", clientGroupLogID(this.id));
 			assert(this.#pipelines.initialized(), "pipelines must be initialized (syncQueryPipelineSet)");
 			if (this.#ttlClock === void 0) this.#ttlClock = cvr.ttlClock;
 			const now = Date.now();
@@ -1338,7 +1365,7 @@ var ViewSyncerService = class {
 		if (!lc.warn) return;
 		const shape = queryShape(query.ast);
 		lc.warn("Query hydration aborted for exceeding the hydration timeout", {
-			clientGroupID: this.id,
+			clientGroupID: clientGroupLogID(this.id),
 			queryHash: query.id,
 			transformationHash: query.transformationHash,
 			...query.name !== void 0 && { queryName: query.name },
@@ -1366,7 +1393,7 @@ var ViewSyncerService = class {
 		if (suppressed === void 0) return;
 		lc.warn("Slow query materialization", {
 			zeroEvent: "query-slow-hydration",
-			clientGroupID: this.id,
+			clientGroupID: clientGroupLogID(this.id),
 			queryHash: query.id,
 			transformationHash: query.transformationHash,
 			...query.name !== void 0 && { queryName: query.name },
@@ -1385,7 +1412,7 @@ var ViewSyncerService = class {
 	#recordCircuitBreakerRejection(lc, query) {
 		this.#hydrationCircuitBreakerRejections.add(1);
 		lc.warn?.("Query rejected by its open hydration circuit breaker", {
-			clientGroupID: this.id,
+			clientGroupID: clientGroupLogID(this.id),
 			queryHash: query.id,
 			transformationHash: query.transformationHash,
 			...query.name !== void 0 && { queryName: query.name },
@@ -1427,7 +1454,7 @@ var ViewSyncerService = class {
 			};
 			for (const clientID of Object.keys(query.clientState)) getOrInsertComputed(errorsByClient, clientID, newArray).push(error);
 		}
-		for (const [clientID, errors] of errorsByClient) this.#clients.get(clientID)?.sendQueryTransformApplicationErrors(errors);
+		for (const [clientID, errors] of errorsByClient) this.#getValidatedClient(clientID)?.sendQueryTransformApplicationErrors(errors);
 	}
 	#recordHydrationBudgetExhaustion(lc, hydrationBudget, activeHydratedQueries, inactiveHydratedQueries, inactiveEvictedQueryIDs) {
 		const elapsedMs = hydrationBudget.exhaustedAtMs;
@@ -1438,7 +1465,7 @@ var ViewSyncerService = class {
 		this.#hydrationBudgetOvershoot.record(Math.max(0, elapsedMs - hydrationBudget.limitMs));
 		if (inactiveEvictedQueryIDs.length > 0) this.#queryEvictions.add(inactiveEvictedQueryIDs.length, { reason: "hydration-budget" });
 		lc.info?.("view-syncer hydration budget exhausted", {
-			clientGroupID: this.id,
+			clientGroupID: clientGroupLogID(this.id),
 			hydrationBudgetMs: hydrationBudget.limitMs,
 			hydrationElapsedMs: elapsedMs,
 			activeHydratedQueries,
@@ -1630,7 +1657,8 @@ var ViewSyncerService = class {
 				if (!row.refCounts) patch = {
 					type: "row",
 					op: "del",
-					id
+					id,
+					catchup: true
 				};
 				else {
 					const { contents } = contentsAndVersion(must(this.#pipelines.getRow(table, rowKey), `Missing row in ${table} keyed by ${Object.keys(rowKey).join()}`));
@@ -1728,7 +1756,7 @@ var ViewSyncerService = class {
 	*/
 	#advancePipelines(lc, cvr) {
 		return startAsyncSpan(tracer, "vs.#advancePipelines", async (span) => {
-			span.setAttribute("clientGroupID", this.id);
+			span.setAttribute("clientGroupID", clientGroupLogID(this.id));
 			assert(this.#pipelines.initialized(), "pipelines must be initialized (advancePipelines");
 			const start = performance.now();
 			const timer = this.#timer(lc, "batch");
@@ -1948,7 +1976,7 @@ var ViewSyncerService = class {
 		const now = this.#now();
 		if (log.hydrationTimeMs < REBUILD_LOG_MIN_MS || now - this.#rebuildsLoggedAt < REBUILD_LOG_INTERVAL_MS) return;
 		lc.info?.("Client group hydrated queries again in its batches of changes instead of pushing the changes to them", {
-			clientGroupID: this.id,
+			clientGroupID: clientGroupLogID(this.id),
 			...log
 		});
 		this.#rebuildsLoggedAt = now;
@@ -2016,7 +2044,7 @@ var ViewSyncerService = class {
 		const now = this.#now();
 		if (log.waitedMs < BATCH_WAIT_LOG_MIN_MS || now - log.loggedAt < BATCH_WAIT_LOG_INTERVAL_MS) return;
 		this.#lc.info?.("Client group is waiting between batches: its sync worker is over its share target", {
-			clientGroupID: this.id,
+			clientGroupID: clientGroupLogID(this.id),
 			waits: log.waits,
 			waitedMs: log.waitedMs,
 			lastWaitMs: waitedMs,
@@ -2038,7 +2066,7 @@ var ViewSyncerService = class {
 			clientID,
 			wsID: client.wsID
 		});
-		return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this, this.#cvrStore, this.#config, connCtx);
+		return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this.#cvrStore, this.#config, connCtx);
 	};
 	async #runBackgroundRetransform(lc) {
 		const attemptRetransform = async (connCtx) => {
@@ -2095,7 +2123,8 @@ var ViewSyncerService = class {
 				if (response.kind === "TransformFailed") throw new ProtocolErrorWithLevel(response, "warn");
 				validation = response.validation;
 			} else validation = { kind: "client-fallback" };
-			this.connContextManager.validateConnection(connCtx, connCtx.revision, validation);
+			if (!this.#currentRegistration(connCtx)) return false;
+			if (this.connContextManager.validateConnection(connCtx, connCtx.revision, validation) && this.#cvr?.pinnedUser === null) this.#cvr = await this.#flushUpdater(this.#lc, new CVRUpdater(this.#cvrStore, this.#cvr, this.#cvr.replicaVersion));
 			return true;
 		} catch (e) {
 			if (isProtocolError(e) && isAuthErrorBody(e.errorBody)) {
@@ -2105,13 +2134,22 @@ var ViewSyncerService = class {
 					revision: connCtx.revision,
 					message: e.message
 				});
-				this.#failMaintenanceConnection(connCtx, e);
+				if (this.#currentRegistration(connCtx)) this.#failMaintenanceConnection(connCtx, e);
 				return false;
 			}
 			throw e;
 		}
 	}
 	/**
+	* The live context registered for `connCtx`'s selector, or `undefined` if
+	* that socket is gone or has been replaced by one reusing the selector
+	* (which starts over at revision 0, so revisions alone cannot tell).
+	*/
+	#currentRegistration(connCtx) {
+		const current = this.connContextManager.getConnectionContext(connCtx);
+		return current?.insertionOrder === connCtx.insertionOrder ? current : void 0;
+	}
+	/**
 	* Validates the connection's credential as it stands once validation ends.
 	* An `updateAuth` handled while a validation is in flight moves the
 	* connection to a newer revision, which makes a failure of the older one
@@ -2122,8 +2160,9 @@ var ViewSyncerService = class {
 		let connCtx = this.connContextManager.getConnectionContext(selector);
 		while (connCtx) {
 			const valid = await this.#validateConnection(connCtx);
-			const current = this.connContextManager.getConnectionContext(selector);
-			if (current?.revision === connCtx.revision) return valid ? connCtx : void 0;
+			const current = this.#currentRegistration(connCtx);
+			if (!current) return;
+			if (current.revision === connCtx.revision) return valid ? connCtx : void 0;
 			connCtx = current;
 		}
 	}
@@ -2150,10 +2189,9 @@ var ViewSyncerService = class {
 		this.#stopShutdownTimer();
 		this.#customQueryTransformer?.destroy();
 		for (const client of this.#clients.values()) if (err) client.fail(err);
-		else client.close(`closed clientGroupID=${this.id}`);
+		else client.close(`closed clientGroupID=${clientGroupLogID(this.id)}`);
 		await this.#lock.withLock(() => {});
 		this.#pipelines.destroy();
-		this.#inspectorDelegate.clearAuthenticated(this.id, this);
 	}
 	/**
 	* Test helper: Manually mark initialization as complete.