@rocicorp/zero 1.8.0 → 1.9.0-canary.0

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -26,6 +26,7 @@ import { tracer } from "./tracer.js";
 import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
 import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, nextEvictionTime } from "./cvr.js";
 import { handleInspect } from "./inspect-handler.js";
+import { QueryCoveringIndex } from "./query-covering.js";
 import { resolver } from "@rocicorp/resolver";
 import { Lock } from "@rocicorp/lock";
 //#region ../zero-cache/src/services/view-syncer/view-syncer.ts
@@ -109,6 +110,7 @@ var ViewSyncerService = class {
 	#lockWaitTime = getOrCreateLatencyHistogram("sync", "lock-wait-time", "Time spent waiting to acquire the ViewSyncer lock.");
 	#pipelineResets = getOrCreateCounter("sync", "pipeline-resets", "Number of pipeline resets");
 	#rowSetSignatureDrifts = getOrCreateCounter("sync", "query.row-set-signature-drifts", "Number of times re-hydration of an unchanged query produced a different row-set signature than what is stored in the CVR (forcing a configVersion bump and full re-execution). Expected to be near-zero in steady state; persistent non-zero values indicate non-deterministic query execution (e.g. Cap operator picking different N-row subsets).");
+	#sameHashRehydrationVersionBumps = getOrCreateCounter("sync", "query.same-hash-rehydrations-forced-bump", "Number of times query-set reconciliation forced a configVersion bump for already-gotten same-transformation-hash query rehydration because trackQueries would not otherwise bump. Expected to be near-zero; non-zero values indicate pipeline/CVR row-set drift reached query-set reconciliation.");
 	#inspectorDelegate;
 	#config;
 	#runPriorityOp;
@@ -251,6 +253,9 @@ var ViewSyncerService = class {
 	get servedVersion() {
 		return this.#servedVersion;
 	}
+	get servingLagEligible() {
+		return this.#clients.size > 0 && this.connContextManager.getBackgroundConnectionContext() !== void 0;
+	}
 	#markVersionServed(version) {
 		this.#servedVersion = version.stateVersion;
 	}
@@ -630,9 +635,19 @@ var ViewSyncerService = class {
 		}
 		lc.info?.(`hydrateUnchangedQueries: ${gotQueries.length} got queries, ${inactivatedCount} inactivated, ${customErrorCount} custom transform errors, ${customHashMismatchCount} custom hash mismatches, ${otherHashMismatchCount} other hash mismatches, ${transformedQueries.length} hydrated`);
 		const driftedQueryIDs = /* @__PURE__ */ new Set();
+		const queryCoveringIndex = this.#config.enableQueryCovering ? new QueryCoveringIndex(this.#pipelines.queries()) : void 0;
+		let totalHydratedQueries = 0;
+		let coveredHydratedQueries = 0;
+		let firstCoveredQuery;
 		for (const { id: queryID, transformationHash, transformedAst } of transformedQueries) {
 			const query = cvr.queries[queryID];
 			const queryName = query.type === "custom" ? query.name : void 0;
+			const covered = queryCoveringIndex ? this.#findQueryCoverageShadowHit(queryCoveringIndex, queryID, transformationHash, transformedAst, queryName) : void 0;
+			totalHydratedQueries++;
+			if (covered) {
+				coveredHydratedQueries++;
+				firstCoveredQuery ??= covered;
+			}
 			const timer = new TimeSliceTimer(lc);
 			let count = 0;
 			await startAsyncSpan(tracer, "vs.#hydrateUnchangedQueries.addQuery", async (span) => {
@@ -649,6 +664,7 @@ var ViewSyncerService = class {
 			this.#addQueryMaterializationServerMetric(transformationHash, elapsed);
 			this.#inspectorDelegate.addQuery(transformationHash, transformedAst);
 			lc.debug?.(`hydrated ${count} rows for ${queryID} (${elapsed} ms)`);
+			let drifted = false;
 			const storedSigHex = cvr.queries[queryID]?.rowSetSignature;
 			if (storedSigHex !== void 0 && storedSigHex !== null) {
 				const priorSig = parseSignature(storedSigHex);
@@ -658,9 +674,16 @@ var ViewSyncerService = class {
 					this.#rowSetSignatureDrifts.add(1);
 					this.#pipelines.removeQuery(queryID);
 					driftedQueryIDs.add(queryID);
+					drifted = true;
 				}
 			}
+			if (!drifted && queryCoveringIndex) queryCoveringIndex.add(queryID, {
+				transformedAst,
+				transformationHash,
+				...queryName !== void 0 && { queryName }
+			});
 		}
+		this.#logQueryCoverageShadowSummary(lc, "hydrate-unchanged", totalHydratedQueries, coveredHydratedQueries, firstCoveredQuery);
 		return driftedQueryIDs;
 	}
 	#processTransformedCustomQueries(lc, transformedCustomQueries, cb, customQueryMap) {
@@ -706,6 +729,28 @@ var ViewSyncerService = class {
 	#addQueryMaterializationServerMetric(queryID, elapsed) {
 		this.#inspectorDelegate.addMetric("query-materialization-server", elapsed, queryID);
 	}
+	#findQueryCoverageShadowHit(queryCoveringIndex, queryID, transformationHash, ast, queryName) {
+		const covering = queryCoveringIndex.findCoveringQuery(queryID, ast);
+		if (!covering) return;
+		return {
+			coveredQueryHash: queryID,
+			coveredTransformationHash: transformationHash,
+			...queryName !== void 0 && { coveredQueryName: queryName },
+			coveringQueryHash: covering.queryID,
+			coveringTransformationHash: covering.transformationHash,
+			...covering.queryName !== void 0 && { coveringQueryName: covering.queryName }
+		};
+	}
+	#logQueryCoverageShadowSummary(lc, hydrationPath, totalHydratedQueries, coveredHydratedQueries, firstCoveredQuery) {
+		if (!this.#config.enableQueryCovering || totalHydratedQueries === 0) return;
+		let coverageLC = lc.withContext("appID", this.#shard.appID).withContext("shardNum", this.#shard.shardNum).withContext("clientGroupID", this.id).withContext("queryCoverageMode", "shadow").withContext("hydrationPath", hydrationPath).withContext("totalHydratedQueries", totalHydratedQueries).withContext("coveredHydratedQueries", coveredHydratedQueries).withContext("uncoveredHydratedQueries", totalHydratedQueries - coveredHydratedQueries);
+		if (firstCoveredQuery) {
+			coverageLC = coverageLC.withContext("firstCoveredQueryHash", firstCoveredQuery.coveredQueryHash).withContext("firstCoveredTransformationHash", firstCoveredQuery.coveredTransformationHash).withContext("firstCoveringQueryHash", firstCoveredQuery.coveringQueryHash).withContext("firstCoveringTransformationHash", firstCoveredQuery.coveringTransformationHash);
+			if (firstCoveredQuery.coveredQueryName !== void 0) coverageLC = coverageLC.withContext("firstCoveredQueryName", firstCoveredQuery.coveredQueryName);
+			if (firstCoveredQuery.coveringQueryName !== void 0) coverageLC = coverageLC.withContext("firstCoveringQueryName", firstCoveredQuery.coveringQueryName);
+		}
+		coverageLC.info?.("query coverage shadow summary");
+	}
 	/**
 	* Adds and/or removes queries to/from the PipelineDriver to bring it
 	* in sync with the set of queries in the CVR (both got and desired).
@@ -843,7 +888,15 @@ var ViewSyncerService = class {
 			lc = lc.withContext("stateVersion", stateVersion);
 			lc.info?.(`hydrating ${addQueries.length} queries`);
 			const updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, stateVersion, this.#pipelines.replicaVersion, (queryID) => this.#pipelines.rowSetSignature(queryID));
-			if (addQueries.some((q) => driftedQueryIDs.has(q.id))) updater.ensureNewVersion();
+			const sameHashRehydratedQueryIDs = addQueries.filter((q) => cvr.queries[q.id]?.transformationHash === q.transformationHash).map((q) => q.id);
+			const trackQueriesWillBumpVersion = stateVersion > cvr.version.stateVersion || removeQueries.length > 0 || addQueries.some((q) => cvr.queries[q.id]?.transformationHash !== q.transformationHash);
+			if (sameHashRehydratedQueryIDs.length > 0 && !trackQueriesWillBumpVersion) {
+				const drifted = sameHashRehydratedQueryIDs.filter((id) => driftedQueryIDs.has(id)).length;
+				const missing = sameHashRehydratedQueryIDs.length - drifted;
+				const reason = drifted && missing ? "mixed" : drifted ? "row-set-signature-drift" : "missing-pipeline";
+				this.#sameHashRehydrationVersionBumps.add(1, { reason });
+				updater.ensureNewVersion();
+			}
 			const { queryPatches, newVersion } = updater.trackQueries(lc, addQueries, removeQueries);
 			const pokers = startPoke(this.#getClients(), newVersion);
 			for (const patch of queryPatches) await pokers.addPatch(patch);
@@ -857,6 +910,10 @@ var ViewSyncerService = class {
 			const pipelines = this.#pipelines;
 			const hydrations = this.#hydrations;
 			const hydrationTime = this.#hydrationTime;
+			const queryCoveringIndex = this.#config.enableQueryCovering ? new QueryCoveringIndex(this.#pipelines.queries()) : void 0;
+			let totalHydratedQueries = 0;
+			let coveredHydratedQueries = 0;
+			let firstCoveredQuery;
 			const self = this;
 			await yieldProcess(lc);
 			function* generateRowChanges(slowHydrateThreshold) {
@@ -864,11 +921,22 @@ var ViewSyncerService = class {
 					let queryLC = lc.withContext("hash", q.id).withContext("queryHash", q.id).withContext("transformationHash", q.transformationHash);
 					if (q.name !== void 0) queryLC = queryLC.withContext("queryName", q.name);
 					queryLC.debug?.(`adding pipeline for query`, q.ast);
+					const covered = queryCoveringIndex ? self.#findQueryCoverageShadowHit(queryCoveringIndex, q.id, q.transformationHash, q.ast, q.name) : void 0;
+					totalHydratedQueries++;
+					if (covered) {
+						coveredHydratedQueries++;
+						firstCoveredQuery ??= covered;
+					}
 					yield* pipelines.addQuery(q.transformationHash, q.id, q.ast, timer.startWithoutYielding(), q.name, "query-set-sync");
 					const elapsed = timer.stop();
 					totalProcessTime += elapsed;
 					self.#addQueryMaterializationServerMetric(q.id, elapsed);
 					self.#inspectorDelegate.addQuery(q.id, q.ast);
+					queryCoveringIndex?.add(q.id, {
+						transformedAst: q.ast,
+						transformationHash: q.transformationHash,
+						...q.name !== void 0 && { queryName: q.name }
+					});
 					if (elapsed > slowHydrateThreshold) queryLC.warn?.("Slow query materialization", elapsed, q.ast);
 					manualSpan(tracer, "vs.addAndConsumeQuery", elapsed, {
 						hash: q.id,
@@ -880,6 +948,7 @@ var ViewSyncerService = class {
 				hydrationTime.recordMs(totalProcessTime);
 			}
 			await this.#processChanges(lc, timer, generateRowChanges(this.#slowHydrateThreshold), updater, pokers);
+			this.#logQueryCoverageShadowSummary(lc, "add", totalHydratedQueries, coveredHydratedQueries, firstCoveredQuery);
 			await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.deleteUnreferencedRows", async () => {
 				for (const patch of await updater.deleteUnreferencedRows(lc)) await pokers.addPatch(patch);
 			});
@@ -1159,6 +1228,7 @@ var ViewSyncerService = class {
 		this.#stopTTLClockInterval();
 		this.#stopExpireTimer();
 		this.#stopAuthMaintenanceTimer();
+		this.#customQueryTransformer?.destroy();
 		for (const client of this.#clients.values()) if (err) client.fail(err);
 		else client.close(`closed clientGroupID=${this.id}`);
 		await this.#lock.withLock(() => {});