@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.10

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -1,8 +1,10 @@
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { must } from "../../../../shared/src/must.js";
+import { h64 } from "../../../../shared/src/hash.js";
 import { Internal, InvalidConnectionRequest, InvalidConnectionRequestBaseCookie, Rehome } from "../../../../zero-protocol/src/error-kind-enum.js";
 import { ZeroCache } from "../../../../zero-protocol/src/error-origin-enum.js";
 import { ProtocolError, isProtocolError } from "../../../../zero-protocol/src/error.js";
+import { normalizeClientSchema } from "../../../../zero-protocol/src/client-schema.js";
 import { MAX_TTL_MS, clampTTL } from "../../../../zql/src/query/ttl.js";
 import { stringify } from "../../../../shared/src/bigint-json.js";
 import { manualSpan, startAsyncSpan, startSpan } from "../../../../otel/src/span.js";
@@ -25,6 +27,7 @@ import { ClientHandler, startPoke } from "./client-handler.js";
 import { tracer } from "./tracer.js";
 import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
 import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, nextEvictionTime } from "./cvr.js";
+import { E2EServingLagTracker } from "./e2e-serving-lag.js";
 import { handleInspect } from "./inspect-handler.js";
 import { QueryCoveringIndex } from "./query-covering.js";
 import { resolver } from "@rocicorp/resolver";
@@ -90,6 +93,7 @@ var ViewSyncerService = class {
 	#cvr;
 	#pipelinesSynced = false;
 	#servedVersion = null;
+	#e2eServingLagTracker = new E2EServingLagTracker();
 	#expiredQueriesTimer = 0;
 	#authMaintenanceTimer = 0;
 	#setTimeout;
@@ -102,6 +106,14 @@ var ViewSyncerService = class {
 		description: "Time from ViewSyncer query sync requiring hydration to output for a client group. Includes query transformation, query materialization, CVR flush, catchup, and pokeEnd.",
 		unit: "s"
 	});
+	#e2eServingLag = getOrCreateNativeHistogram("sync", "e2e_serving_lag", {
+		description: "End-to-end lag from upstream commit to ViewSyncer output. Spans the whole pipeline: the upstream transaction commit, replication to the replica, IVM advancement, CVR flush, and pokeEnd. Recorded once per advancement, not sampled, so each observation is the completion latency of real replicated work. An advancement that produced no changes for this client group still counts: the group is genuinely current as of that commit, and excluding it would make the metric measure the time since the group last received data instead of the pipeline latency.",
+		unit: "s"
+	});
+	#e2eServingLagClamps = getOrCreateCounter("sync", "e2e_serving_lag_clamps", {
+		description: "Observations of sync.e2e_serving_lag that came out negative and were clamped to zero. Non-zero means the upstream database clock is running ahead of this pod by more than the entire pipeline latency, so sync.e2e_serving_lag is biased low and reads healthier than reality. See replication.upstream_clock_skew for the magnitude.",
+		unit: "{observation}"
+	});
 	#transactionAdvanceTime = getOrCreateLatencyHistogram("sync", "advance-time", "Time to advance all queries for a given client group after applying a new transaction to the replica.");
 	#queryTransformations = getOrCreateCounter("sync", "query.transformations", "Number of query transformations performed");
 	#queryTransformationTime = getOrCreateLatencyHistogram("sync", "query.transformation-time", "Time to transform custom queries via API server.");
@@ -186,12 +198,14 @@ var ViewSyncerService = class {
 				this.#lc.debug?.(`draining view-syncer ${this.id} before running`);
 				this.stop();
 			}
-			for await (const { state } of this.#stateChanges) {
+			for await (const replicaState of this.#stateChanges) {
+				const { state } = replicaState;
 				if (this.#drainCoordinator.shouldDrain()) {
 					this.#lc.debug?.(`draining view-syncer ${this.id} (elective)`);
 					break;
 				}
 				assert(state === "version-ready", "state should be version-ready");
+				this.#e2eServingLagTracker.onVersionReady(replicaState);
 				await this.#runInLockWithCVR(async (lc, cvr) => {
 					const clientSchema = must(cvr.clientSchema, "cvr.clientSchema missing after initialization");
 					if (!this.#pipelines.initialized()) this.#pipelines.init(clientSchema);
@@ -247,6 +261,28 @@ var ViewSyncerService = class {
 	get queryCount() {
 		return this.#pipelines.initialized() ? this.#pipelines.queries().size : 0;
 	}
+	pipelineHashes() {
+		if (!this.#pipelines.initialized()) return [];
+		const cvrQueries = this.#cvr?.queries;
+		if (cvrQueries === void 0) return [];
+		const hashes = [];
+		for (const [queryID, { transformationHash, queryName }] of this.#pipelines.queries().entries()) hashes.push({
+			transformationHash,
+			internal: cvrQueries[queryID]?.type === "internal",
+			queryName
+		});
+		return hashes;
+	}
+	#clientSchemaKeyCache;
+	get clientSchemaKey() {
+		const schema = this.#cvr?.clientSchema;
+		if (!schema) return;
+		if (this.#clientSchemaKeyCache?.schema !== schema) this.#clientSchemaKeyCache = {
+			schema,
+			key: h64(JSON.stringify(normalizeClientSchema(schema))).toString(36)
+		};
+		return this.#clientSchemaKeyCache.key;
+	}
 	get rowCount() {
 		return this.#cvrStore.rowCount;
 	}
@@ -256,8 +292,30 @@ var ViewSyncerService = class {
 	get servingLagEligible() {
 		return this.#clients.size > 0 && this.connContextManager.getBackgroundConnectionContext() !== void 0;
 	}
-	#markVersionServed(version) {
-		this.#servedVersion = version.stateVersion;
+	/**
+	* Records that this client group is caught up through `stateVersion`, i.e.
+	* everything the replica had at that version has been poked to clients.
+	*
+	* This is the *replica* state version that was processed, not the CVR
+	* version. The two diverge whenever an advancement produces no writes for
+	* this client group: `CVRUpdater.flush()` returns the pre-update snapshot on
+	* a no-op flush, so the CVR version stays where it was even though the group
+	* is fully current with the replica. Marking the CVR version here would
+	* leave every client group that did not happen to match a transaction
+	* looking permanently unserved, and both `sync.serving_lag_stats` and
+	* `sync.e2e_serving_lag` would then report the time since the group's last
+	* *data* change as though it were lag.
+	*
+	* Monotonic: `#catchupClients` may pass a CVR that is not the current one.
+	*/
+	#markVersionServed(stateVersion) {
+		if (this.#servedVersion !== null && stateVersion <= this.#servedVersion) return;
+		this.#servedVersion = stateVersion;
+		const observation = this.#e2eServingLagTracker.onVersionServed(stateVersion, Date.now());
+		if (observation !== null) {
+			this.#e2eServingLag.recordMs(observation.lagMs);
+			if (observation.clamped) this.#e2eServingLagClamps.add(1);
+		}
 	}
 	#keepAliveUntil = 0;
 	/**
@@ -694,8 +752,12 @@ var ViewSyncerService = class {
 		const appQueryErrors = [];
 		for (const q of transformedCustomQueries) {
 			if ("error" in q) {
-				const errorMessage = `Error transforming custom query ${q.name}: ${q.error}${q.details ? ` ${JSON.stringify(q.details)}` : ""}`;
-				lc.warn?.(errorMessage, q);
+				lc.warn?.("Error transforming custom query", {
+					id: q.id,
+					name: q.name,
+					error: q.error,
+					hasDetails: q.details !== void 0
+				});
 				appQueryErrors.push(q);
 				continue;
 			}
@@ -956,7 +1018,7 @@ var ViewSyncerService = class {
 			const finalVersion = this.#cvr.version;
 			await this.#catchupClients(lc, cvr, finalVersion, addQueries.map((q) => q.id), pokers);
 			await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.pokeEnd", () => pokers.end(finalVersion));
-			this.#markVersionServed(finalVersion);
+			this.#markVersionServed(stateVersion);
 			const wallTime = performance.now() - start;
 			lc.info?.(`finished processing queries (process: ${totalProcessTime} ms, wall: ${wallTime} ms)`);
 		});
@@ -1026,7 +1088,7 @@ var ViewSyncerService = class {
 			for (const patch of await configPatches) await pokers.addPatch(patch);
 			if (!usePokers) {
 				await pokers.end(cvr.version);
-				this.#markVersionServed(cvr.version);
+				this.#markVersionServed(cvr.version.stateVersion);
 			}
 		});
 	}
@@ -1120,7 +1182,7 @@ var ViewSyncerService = class {
 			this.#cvr = await this.#flushUpdater(lc, updater);
 			const finalVersion = this.#cvr.version;
 			await startAsyncSpan(tracer, "vs.#advancePipelines.pokeEnd", () => pokers.end(finalVersion));
-			this.#markVersionServed(finalVersion);
+			this.#markVersionServed(version);
 			const wallTime = performance.now() - start;
 			const totalProcessTime = timer.totalElapsed();
 			lc.debug?.(`finished processing advancement of ${numChanges} changes ((process: ${totalProcessTime} ms, wall: ${wallTime} ms))`);