@rocicorp/zero 1.11.0-canary.13 → 1.11.0-canary.15

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -12,9 +12,12 @@ import { manualSpan, startAsyncSpan, startSpan } from "../../../../otel/src/span
 import { ZERO_VERSION_COLUMN_NAME } from "../replicator/schema/constants.js";
 import { transformAndHashQuery } from "../../auth/read-authorizer.js";
 import { randInt } from "../../../../shared/src/rand.js";
+import { LogThrottle } from "../../observability/log-throttle.js";
 import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateNativeHistogram, getOrCreateUpDownCounter, getOrCreateValueHistogram } from "../../observability/metrics.js";
 import { rowIDString } from "../../types/row-key.js";
 import "../replicator/schema/replication-state.js";
+import { planWarningMessage } from "./plan-warnings.js";
+import { queryShape } from "./query-shape.js";
 import { parseSignature } from "./row-set-signature.js";
 import { ResetPipelinesSignal } from "./snapshotter.js";
 import { Subscription } from "../../types/subscription.js";
@@ -53,6 +56,7 @@ function shutdownBeforeInitializationError() {
 * use a timer to update the ttlClock every minute.
 */
 var TTL_CLOCK_INTERVAL = 6e4;
+var slowHydrationLogThrottle = new LogThrottle({ windowMs: 5 * 6e4 });
 var ViewSyncerService = class {
 	id;
 	createdAtMs = Date.now();
@@ -948,6 +952,7 @@ var ViewSyncerService = class {
 			if (timedOut) {
 				this.#recordHydrationTimeout(lc, {
 					id: queryID,
+					ast: transformedAst,
 					transformationHash,
 					name: queryName
 				}, elapsed, count);
@@ -959,6 +964,12 @@ var ViewSyncerService = class {
 			this.#addQueryMaterializationServerMetric(queryID, elapsed);
 			this.#inspectorDelegate.addQuery(queryID, transformedAst);
 			lc.debug?.(`hydrated ${count} rows for ${queryID} (${elapsed} ms)`);
+			if (elapsed > this.#slowHydrateThreshold) this.#logSlowHydration(lc, {
+				id: queryID,
+				ast: transformedAst,
+				transformationHash,
+				name: queryName
+			}, elapsed);
 			let drifted = false;
 			const storedSigHex = cvr.queries[queryID]?.rowSetSignature;
 			if (storedSigHex !== void 0 && storedSigHex !== null) {
@@ -1227,15 +1238,50 @@ var ViewSyncerService = class {
 	#recordHydrationTimeout(lc, query, elapsedMs, rowCount) {
 		this.#hydrationCircuitBreaker.trip(query.transformationHash);
 		this.#hydrationTimeouts.add(1);
-		lc.warn?.("Query hydration aborted for exceeding the hydration timeout", {
+		if (!lc.warn) return;
+		const shape = queryShape(query.ast);
+		lc.warn("Query hydration aborted for exceeding the hydration timeout", {
 			clientGroupID: this.id,
 			queryHash: query.id,
 			transformationHash: query.transformationHash,
 			...query.name !== void 0 && { queryName: query.name },
+			queryShape: shape.hash,
 			hydrationTimeoutMs: this.#hydrationCircuitBreaker.timeoutMs,
 			hydrationElapsedMs: elapsedMs,
 			...rowCount !== void 0 && { hydrationRowCount: rowCount },
-			circuitBreakerOpenMs: this.#hydrationCircuitBreaker.openMs
+			circuitBreakerOpenMs: this.#hydrationCircuitBreaker.openMs,
+			zql: shape.zql
+		});
+	}
+	/**
+	* Logs a hydration that exceeded the slow hydration threshold.
+	*
+	* The query is logged as its {@link queryShape}, with literal values
+	* redacted, and each shape is logged at most once per
+	* {@link SLOW_HYDRATION_LOG_WINDOW_MS}. The next log of a shape reports how
+	* many slow hydrations of it were suppressed in the meantime.
+	*/
+	#logSlowHydration(lc, query, elapsedMs) {
+		if (!lc.warn) return;
+		const stats = this.#pipelines.hydrationStats(query.id);
+		const shape = stats?.shape ?? queryShape(query.ast);
+		const suppressed = slowHydrationLogThrottle.admit(`${query.name ?? ""}:${shape.hash}`);
+		if (suppressed === void 0) return;
+		lc.warn("Slow query materialization", {
+			zeroEvent: "query-slow-hydration",
+			clientGroupID: this.id,
+			queryHash: query.id,
+			transformationHash: query.transformationHash,
+			...query.name !== void 0 && { queryName: query.name },
+			queryShape: shape.hash,
+			hydrationTimeMs: elapsedMs,
+			...stats && {
+				hydrationRowCount: stats.rowCount,
+				hydrationRowsRead: stats.rowsRead
+			},
+			...stats !== void 0 && stats.planWarnings.length > 0 && { planWarnings: stats.planWarnings.map(planWarningMessage) },
+			...suppressed > 0 && { suppressedSinceLastLog: suppressed },
+			zql: shape.zql
 		});
 	}
 	/** Records that `query` was rejected without hydration by its open breaker. */
@@ -1396,7 +1442,7 @@ var ViewSyncerService = class {
 						transformationHash: q.transformationHash,
 						...q.name !== void 0 && { queryName: q.name }
 					});
-					if (elapsed > slowHydrateThreshold) queryLC.warn?.("Slow query materialization", elapsed, q.ast);
+					if (elapsed > slowHydrateThreshold) self.#logSlowHydration(queryLC, q, elapsed);
 					manualSpan(tracer, "vs.addAndConsumeQuery", elapsed, {
 						hash: q.id,
 						transformationHash: q.transformationHash,