@rocicorp/zero 1.11.0-canary.13 → 1.11.0-canary.14

package/out/zero-cache/src/services/view-syncer/pipeline-driver.js

@@ -15,13 +15,18 @@ import { resolveSimpleScalarSubqueries } from "../../../../zqlite/src/resolve-sc
 import { randInt } from "../../../../shared/src/rand.js";
 import { createSQLiteCostModel } from "../../../../zqlite/src/sqlite-cost-model.js";
 import { reloadPermissionsIfChanged } from "../../auth/load-permissions.js";
+import { LogThrottle } from "../../observability/log-throttle.js";
 import { getOrCreateCounter, getOrCreateLatencyHistogram } from "../../observability/metrics.js";
 import "../../types/row-key.js";
 import { getSubscriptionState } from "../replicator/schema/replication-state.js";
 import { checkClientSchema } from "./client-schema.js";
+import { planWarningMessage } from "./plan-warnings.js";
+import { queryShape } from "./query-shape.js";
 import { rowIDSignatureUnit } from "./row-set-signature.js";
 import { ResetPipelinesSignal } from "./snapshotter.js";
 //#region ../zero-cache/src/services/view-syncer/pipeline-driver.ts
+/** The most queries a log of an advancement timeout lists. */
+var ADVANCE_TIMEOUT_LOG_MAX_QUERIES = 3;
 /**
 * No matter how fast hydration is, advancement is given at least this long to
 * complete before doing a pipeline reset.
@@ -34,6 +39,8 @@ var MIN_PROJECTED_ADVANCEMENT_SAMPLE_MS = 5;
 var MIN_PROJECTED_ADVANCEMENT_CHANGES = 16;
 var PROJECTED_ADVANCEMENT_RESET_MULTIPLIER = 1.5;
 var LATE_ADVANCEMENT_FINISH_PROGRESS = .8;
+var planWarningLogThrottle = new LogThrottle({ windowMs: 60 * 6e4 });
+var slowAdvanceLogThrottle = new LogThrottle({ windowMs: 5 * 6e4 });
 function randomID() {
 	return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);
 }
@@ -78,9 +85,11 @@ var PipelineDriver = class {
 	#shardID;
 	#logConfig;
 	#config;
+	#deferredWrites;
 	#tableSpecs = /* @__PURE__ */ new Map();
 	#allTableNames = /* @__PURE__ */ new Set();
 	#costModels;
+	#planWarningThresholds;
 	#yieldThresholdMs;
 	#streamer = null;
 	#hydrateContext = null;
@@ -90,17 +99,34 @@ var PipelineDriver = class {
 	#permissions = null;
 	#advanceTime = getOrCreateLatencyHistogram("sync", "ivm.advance-time", "Time to advance all queries for a given client group in response to a single change.");
 	#conflictRowsDeleted = getOrCreateCounter("sync", "ivm.conflict-rows-deleted", "Number of rows deleted because they conflicted with added row");
+	#deferredWritesFallbacks = getOrCreateCounter("sync", "ivm.deferred-writes-fallbacks", "Number of advancements written through to the replica snapshot because their changes did not fit in the deferred IVM writes budget, from the start or partway (because of the bytes held, a change that needed more rows than were left, or more rows held than reserved)");
 	#inspectorDelegate;
-	constructor(lc, logConfig, snapshotter, shardID, storage, clientGroupID, inspectorDelegate, yieldThresholdMs, enablePlanner, config) {
+	#queryStats;
+	/**
+	* Passes the pipelines' metrics on to the inspector, and accounts the time
+	* spent pushing to each query to the advancement in progress.
+	*/
+	#metricsDelegate = { addMetric: (metric, value, ...args) => {
+		this.#inspectorDelegate.addMetric(metric, value, ...args);
+		if (metric === "query-update-server") this.#recordAdvancePush(args[0], value);
+	} };
+	constructor(lc, logConfig, snapshotter, shardID, storage, clientGroupID, inspectorDelegate, yieldThresholdMs, enablePlanner, config, deferredWrites, queryStats) {
 		this.#lc = lc.withContext("clientGroupID", clientGroupID);
 		this.#snapshotter = snapshotter;
 		this.#storage = storage;
 		this.#shardID = shardID;
 		this.#logConfig = logConfig;
 		this.#config = config;
+		this.#deferredWrites = deferredWrites;
 		this.#inspectorDelegate = inspectorDelegate;
 		this.#costModels = enablePlanner ? /* @__PURE__ */ new WeakMap() : void 0;
+		const planWarningThresholds = {
+			rows: logConfig.planWarningRowThreshold,
+			cost: logConfig.planWarningCostThreshold
+		};
+		this.#planWarningThresholds = planWarningThresholds.rows > 0 || planWarningThresholds.cost > 0 ? planWarningThresholds : void 0;
 		this.#yieldThresholdMs = yieldThresholdMs;
+		this.#queryStats = queryStats;
 	}
 	/**
 	* Initializes the PipelineDriver to the current head of the database.
@@ -219,18 +245,76 @@ var PipelineDriver = class {
 	queries() {
 		return this.#pipelines;
 	}
+	/**
+	* Stats from the hydration of the pipeline for `queryID`, or `undefined` if
+	* the query has no pipeline.
+	*/
+	hydrationStats(queryID) {
+		const pipeline = this.#pipelines.get(queryID);
+		return pipeline ? {
+			rowCount: pipeline.hydrationRowCount,
+			rowsRead: pipeline.hydrationRowsRead,
+			planWarnings: pipeline.planWarnings,
+			shape: pipeline.shape
+		} : void 0;
+	}
+	/**
+	* Logs what the planner warned about the plan it chose for `query`, at
+	* most once per query shape per {@link PLAN_WARNING_LOG_WINDOW_MS}.
+	*/
+	#logPlanWarnings(shape, { queryHash, transformationHash, queryName }, warnings) {
+		if (!this.#lc.warn) return;
+		const suppressed = planWarningLogThrottle.admit(`${queryName ?? ""}:${shape.hash}`);
+		if (suppressed === void 0) return;
+		const messages = warnings.map(planWarningMessage);
+		this.#lc.warn(`Query plan warning${queryName === void 0 ? "" : ` for ${queryName}`}: ` + messages.join(" "), {
+			zeroEvent: "query-plan-warning",
+			queryHash,
+			transformationHash,
+			...queryName !== void 0 && { queryName },
+			queryShape: shape.hash,
+			warnings,
+			...suppressed > 0 && { suppressedSinceLastLog: suppressed },
+			zql: shape.zql
+		});
+	}
+	#totalRowsRead() {
+		let total = 0;
+		for (const table of this.#tables.values()) total += table.rowsRead;
+		return total;
+	}
 	totalHydrationTimeMs() {
 		let total = 0;
 		for (const pipeline of this.#pipelines.values()) total += pipeline.hydrationTimeMs;
 		return total;
 	}
-	#logQueryPipelineLifecycle({ zeroEvent, pipelineRunID, queryHash, transformationHash, queryName, hydrationReason, stopReason, hydrationTimeMs, hydrationRowCount, pipelineLifetimeMs }) {
+	/**
+	* The rows the sources hold in memory for the advancement in progress.
+	* The advancement's reservation from the {@link DeferredWritesBudget}
+	* bounds this.
+	*/
+	get pendingRows() {
+		let total = 0;
+		for (const source of this.#tables.values()) total += source.pendingRows;
+		return total;
+	}
+	/**
+	* The estimated bytes of {@link pendingRows}, which are added to the
+	* {@link DeferredWritesBudget}'s held bytes as the advancement goes.
+	*/
+	get pendingBytes() {
+		let total = 0;
+		for (const source of this.#tables.values()) total += source.pendingBytes;
+		return total;
+	}
+	#logQueryPipelineLifecycle({ zeroEvent, pipelineRunID, queryHash, transformationHash, queryName, hydrationReason, stopReason, hydrationTimeMs, hydrationRowCount, hydrationRowsRead, pipelineLifetimeMs }) {
 		let lc = this.#lc.withContext("zeroEvent", zeroEvent).withContext("pipelineRunID", pipelineRunID).withContext("queryHash", queryHash).withContext("transformationHash", transformationHash);
 		if (queryName !== void 0) lc = lc.withContext("queryName", queryName);
 		if (hydrationReason !== void 0) lc = lc.withContext("hydrationReason", hydrationReason);
 		if (stopReason !== void 0) lc = lc.withContext("stopReason", stopReason);
 		if (hydrationTimeMs !== void 0) lc = lc.withContext("hydrationTimeMs", hydrationTimeMs);
 		if (hydrationRowCount !== void 0) lc = lc.withContext("hydrationRowCount", hydrationRowCount);
+		if (hydrationRowsRead !== void 0) lc = lc.withContext("hydrationRowsRead", hydrationRowsRead);
 		if (pipelineLifetimeMs !== void 0) lc = lc.withContext("pipelineLifetimeMs", pipelineLifetimeMs);
 		lc.info?.("query pipeline lifecycle");
 	}
@@ -312,6 +396,7 @@ var PipelineDriver = class {
 		assert(this.initialized(), "Pipeline driver must be initialized before adding queries");
 		this.removeQuery(queryID, "replace-query");
 		const pipelineRunID = randomID();
+		const shape = queryShape(query);
 		this.#logQueryPipelineLifecycle({
 			zeroEvent: "query-pipeline-hydrate-start",
 			pipelineRunID,
@@ -324,9 +409,12 @@ var PipelineDriver = class {
 		const costModel = this.#ensureCostModelExistsIfEnabled(this.#snapshotter.current().db.db);
 		assert(this.#advanceContext === null, "Cannot hydrate while advance is in progress");
 		this.#hydrateContext = { timer };
+		const rowsReadAtStart = this.#totalRowsRead();
 		let hydrationFinished = false;
 		let hydrationFailed = false;
 		let hydrationRowCount = 0;
+		let planWarnings = [];
+		const planWarningThresholds = this.#planWarningThresholds;
 		let builtInputs = [];
 		try {
 			const { ast: resolvedQuery, companionRows, companions: companionMeta, companionInputs, ignoredScalarHints } = this.#resolveScalarSubqueries(query);
@@ -339,12 +427,23 @@ var PipelineDriver = class {
 				enablePlannerAwarePushdown: this.#config?.enablePlannerAwarePushdown !== false,
 				getSource: (name) => this.#getSource(name),
 				createStorage: () => this.#createStorage(),
-				decorateSourceInput: (input, _queryID) => new MeasurePushOperator(new QueryFailureLoggingOperator(this.#lc, input, queryID, transformationHash, queryName), queryID, this.#inspectorDelegate, "query-update-server"),
+				decorateSourceInput: (input, _queryID) => new MeasurePushOperator(new QueryFailureLoggingOperator(this.#lc, input, queryID, transformationHash, queryName), queryID, this.#metricsDelegate, "query-update-server"),
 				decorateInput: (input) => input,
 				addEdge() {},
-				decorateFilterInput: (input) => input
+				decorateFilterInput: (input) => input,
+				planWarnings: planWarningThresholds && {
+					thresholds: planWarningThresholds,
+					report: (warnings) => {
+						planWarnings = warnings;
+					}
+				}
 			}, queryID, costModel);
 			builtInputs.push(input);
+			if (planWarnings.length > 0) this.#logPlanWarnings(shape, {
+				queryHash: queryID,
+				transformationHash,
+				queryName
+			}, planWarnings);
 			const schema = input.getSchema();
 			input.setOutput({ push: (change) => this.#streamPushed(queryID, schema, change) });
 			for (const change of hydrateInternal(input, queryID, must(this.#primaryKeys), this.#tableSpecs)) {
@@ -363,6 +462,7 @@ var PipelineDriver = class {
 				};
 			}
 			const hydrationTimeMs = timer.totalElapsed();
+			const hydrationRowsRead = this.#totalRowsRead() - rowsReadAtStart;
 			if (runtimeDebugFlags.trackRowCountsVended) {
 				if (hydrationTimeMs > this.#logConfig.slowHydrateThreshold) {
 					let totalRowsConsidered = 0;
@@ -408,16 +508,29 @@ var PipelineDriver = class {
 				input,
 				hydrationTimeMs,
 				hydrationRowCount,
+				hydrationRowsRead,
+				planWarnings,
 				hydrationReason,
 				pipelineRunID,
 				pipelineReadyAtMs,
 				transformedAst: resolvedQuery,
 				originalAst: query,
+				shape,
 				transformationHash,
 				...queryName !== void 0 && { queryName },
 				companions: liveCompanions
 			});
 			hydrationFinished = true;
+			this.#queryStats?.recordHydration({
+				queryName,
+				shape
+			}, {
+				outcome: "finished",
+				timeMs: hydrationTimeMs,
+				rowCount: hydrationRowCount,
+				rowsRead: hydrationRowsRead,
+				planWarnings
+			});
 			this.#logQueryPipelineLifecycle({
 				zeroEvent: "query-pipeline-hydrate-finish",
 				pipelineRunID,
@@ -426,10 +539,18 @@ var PipelineDriver = class {
 				queryName,
 				hydrationReason,
 				hydrationTimeMs,
-				hydrationRowCount
+				hydrationRowCount,
+				hydrationRowsRead
 			});
 		} catch (e) {
 			hydrationFailed = true;
+			this.#queryStats?.recordHydration({
+				queryName,
+				shape
+			}, {
+				outcome: "failed",
+				timeMs: timer.totalElapsed()
+			});
 			this.#logQueryPipelineLifecycle({
 				zeroEvent: "query-pipeline-hydrate-failed",
 				pipelineRunID,
@@ -447,16 +568,25 @@ var PipelineDriver = class {
 			}, "query hydration failed", e);
 			throw e;
 		} finally {
-			if (!hydrationFinished && !hydrationFailed) this.#logQueryPipelineLifecycle({
-				zeroEvent: "query-pipeline-hydrate-aborted",
-				pipelineRunID,
-				queryHash: queryID,
-				transformationHash,
-				queryName,
-				hydrationReason,
-				hydrationTimeMs: timer.totalElapsed(),
-				hydrationRowCount
-			});
+			if (!hydrationFinished && !hydrationFailed) {
+				this.#queryStats?.recordHydration({
+					queryName,
+					shape
+				}, {
+					outcome: "aborted",
+					timeMs: timer.totalElapsed()
+				});
+				this.#logQueryPipelineLifecycle({
+					zeroEvent: "query-pipeline-hydrate-aborted",
+					pipelineRunID,
+					queryHash: queryID,
+					transformationHash,
+					queryName,
+					hydrationReason,
+					hydrationTimeMs: timer.totalElapsed(),
+					hydrationRowCount
+				});
+			}
 			if (!hydrationFinished) {
 				for (const input of builtInputs) input.destroy();
 				this.#pruneUnusedTables();
@@ -559,25 +689,38 @@ var PipelineDriver = class {
 	*#advance(diff, timer, numChanges) {
 		assert(this.#hydrateContext === null, "Cannot advance while hydration is in progress");
 		const totalHydrationTimeMs = this.totalHydrationTimeMs();
-		this.#advanceContext = {
+		const advanceContext = {
 			timer,
 			totalHydrationTimeMs,
 			numChanges,
+			reservedRows: void 0,
+			heldBytes: 0,
 			currentChangeStartMs: void 0,
-			pos: 0
+			pos: 0,
+			currentTable: void 0,
+			queryStats: /* @__PURE__ */ new Map()
 		};
-		this.#lc.debug?.(`starting pipeline advancement of ${numChanges} changes with an advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`);
+		this.#advanceContext = advanceContext;
+		let timedOut = false;
 		try {
-			for (const { table, prevValues, nextValue } of diff) {
+			advanceContext.reservedRows = this.#reserveDeferredWrites(numChanges);
+			const deferWrites = advanceContext.reservedRows !== void 0;
+			if (deferWrites) diff.setPrevWrites("none");
+			for (const table of this.#tables.values()) table.setDeferWrites(deferWrites);
+			this.#lc.debug?.(`starting pipeline advancement of ${numChanges} changes with an advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms (${deferWrites ? "deferred" : "write-through"}).`);
+			for (const { table, prevValues: probedPrevValues, nextValue, rowKey } of diff) {
 				if (this.#shouldAdvanceYieldMaybeAbortAdvance()) yield "yield";
 				const start = timer.totalElapsed();
-				const advanceContext = must(this.#advanceContext);
 				advanceContext.currentChangeStartMs = start;
+				advanceContext.currentTable = table;
+				let holds = "fits";
 				try {
 					try {
 						const tableSource = this.#tables.get(table);
 						if (!tableSource) continue;
 						const primaryKey = mustGetPrimaryKey(this.#primaryKeys, table);
+						if (nextValue !== null && !this.#reserveSecondRow(advanceContext, rowKey, primaryKey)) holds = "rows";
+						const prevValues = tableSource.reconcilePendingConflicts(probedPrevValues, nextValue, rowKey, this.#tableSpecs.get(table)?.tableSpec.uniqueKeys ?? []);
 						let editOldRow = void 0;
 						for (const prevValue of prevValues) if (nextValue && deepEqual(getRowKey(primaryKey, prevValue), getRowKey(primaryKey, nextValue))) editOldRow = prevValue;
 						else {
@@ -590,20 +733,188 @@ var PipelineDriver = class {
 						advanceContext.pos++;
 					}
 					this.#shouldAdvanceYieldMaybeAbortAdvance(false);
+					if (holds === "fits") holds = this.#holdPendingRows(advanceContext);
 				} finally {
 					advanceContext.currentChangeStartMs = void 0;
 				}
 				const elapsed = timer.totalElapsed() - start;
 				this.#advanceTime.recordMs(elapsed, { table });
+				if (holds !== "fits") yield* this.#writeThrough(advanceContext, diff, holds);
 			}
 			const { curr } = diff;
 			for (const table of this.#tables.values()) table.setDB(curr.db.db);
 			this.#ensureCostModelExistsIfEnabled(curr.db.db);
 			this.#lc.debug?.(`Advanced to ${curr.version}`);
+			this.#logSlowAdvances(advanceContext);
+		} catch (e) {
+			if (e instanceof ResetPipelinesSignal && e.reason === "advancement-timeout") {
+				timedOut = true;
+				this.#logAdvanceTimeout(advanceContext, e);
+			}
+			throw e;
 		} finally {
+			if (advanceContext.reservedRows !== void 0) for (const table of this.#tables.values()) table.discardPendingChanges();
+			this.#releaseDeferredWrites(advanceContext);
 			this.#advanceContext = null;
+			this.#recordAdvanceStats(advanceContext, timedOut);
+		}
+	}
+	/** Accounts the time each query took in an advancement to its shape. */
+	#recordAdvanceStats({ queryStats }, timedOut) {
+		if (this.#queryStats === void 0) return;
+		for (const [queryID, { timeMs, changes }] of queryStats) {
+			const pipeline = this.#pipelines.get(queryID);
+			if (pipeline !== void 0) this.#queryStats.recordAdvance(pipeline, {
+				timeMs,
+				changes,
+				timedOut
+			});
+		}
+	}
+	/**
+	* Decides how the sources apply an advancement's changes: in memory if
+	* they fit in the worker's {@link DeferredWritesBudget}, and otherwise
+	* written through to the `prev` snapshot. Returns the number of rows
+	* reserved if they are held in memory.
+	*/
+	#reserveDeferredWrites(numChanges) {
+		const budget = this.#deferredWrites;
+		if (!budget) return;
+		if (budget.tryReserve(numChanges)) return numChanges;
+		this.#deferredWritesFallbacks.add(1, { stage: "start" });
+		this.#lc.debug?.(`writing through ${numChanges} changes: ${budget.reservedRows} rows of the deferred writes budget are reserved`);
+	}
+	/**
+	* If the change log identifies the rows of a table by a key other than its
+	* primary key here, an entry that sets a row can change its primary key,
+	* which is two rows to a source: the old and the new. (An entry that removes
+	* a row removes the one row its key identifies.) Reserves the second row for
+	* such an entry, if the advancement holds its changes in memory, and
+	* returns whether it fit.
+	*/
+	#reserveSecondRow(advanceContext, rowKey, primaryKey) {
+		if (advanceContext.reservedRows === void 0 || isKeyedBy(rowKey, primaryKey)) return true;
+		if (!must(this.#deferredWrites).tryReserveMore(1)) return false;
+		advanceContext.reservedRows++;
+		return true;
+	}
+	/**
+	* Writes the changes that an advancement holds in memory through to the
+	* `prev` snapshot, and the rest of its changes after them, when they no
+	* longer fit (see {@link HeldRows}). The advancement then continues as if
+	* it had written through from the start.
+	*/
+	*#writeThrough(advanceContext, diff, reason) {
+		const budget = must(this.#deferredWrites);
+		this.#deferredWritesFallbacks.add(1, {
+			stage: "partway",
+			reason
+		});
+		this.#lc.debug?.(`writing through (${reason}) at ${advanceContext.pos} of ${advanceContext.numChanges} changes, holding ${advanceContext.heldBytes} of the ${budget.heldBytes} estimated bytes held on this worker (budget: ${budget.maxBytes})`);
+		diff.setPrevWrites("divergent");
+		for (const source of this.#tables.values()) yield* source.writePendingChanges();
+		this.#releaseDeferredWrites(advanceContext);
+	}
+	#releaseDeferredWrites(advanceContext) {
+		const { reservedRows, heldBytes } = advanceContext;
+		if (reservedRows !== void 0) {
+			must(this.#deferredWrites).release(reservedRows, heldBytes);
+			advanceContext.reservedRows = void 0;
+			advanceContext.heldBytes = 0;
+		}
+	}
+	#recordAdvancePush(queryID, timeMs) {
+		const advance = this.#advanceContext;
+		if (advance === null) return;
+		const stats = getOrInsertComputed(advance.queryStats, queryID, () => ({
+			timeMs: 0,
+			changes: 0,
+			lastPos: -1,
+			timeMsByTable: /* @__PURE__ */ new Map()
+		}));
+		stats.timeMs += timeMs;
+		if (stats.lastPos !== advance.pos) {
+			stats.lastPos = advance.pos;
+			stats.changes++;
+		}
+		const table = advance.currentTable;
+		if (table !== void 0) stats.timeMsByTable.set(table, (stats.timeMsByTable.get(table) ?? 0) + timeMs);
+	}
+	/**
+	* The identity of the query of the pipeline for `queryID` and its
+	* {@link queryShape}, for logging.
+	*/
+	#queryForLog(queryID) {
+		const pipeline = this.#pipelines.get(queryID);
+		if (pipeline === void 0) return;
+		const { transformationHash, queryName, shape } = pipeline;
+		return {
+			queryName,
+			shape,
+			fields: {
+				queryHash: queryID,
+				transformationHash,
+				...queryName !== void 0 && { queryName },
+				queryShape: shape.hash
+			}
+		};
+	}
+	/**
+	* Logs each query whose pushes took longer than the slow advance
+	* threshold, at most once per query shape per
+	* {@link SLOW_ADVANCE_LOG_WINDOW_MS}.
+	*/
+	#logSlowAdvances({ queryStats, numChanges }) {
+		if (!this.#lc.warn) return;
+		for (const [queryID, stats] of queryStats) {
+			if (stats.timeMs <= this.#logConfig.slowAdvanceThreshold) continue;
+			const query = this.#queryForLog(queryID);
+			if (query === void 0) continue;
+			const suppressed = slowAdvanceLogThrottle.admit(`${query.queryName ?? ""}:${query.shape.hash}`);
+			if (suppressed === void 0) continue;
+			this.#lc.warn(`Slow query advancement${query.queryName === void 0 ? "" : ` for ${query.queryName}`}: ${Math.round(stats.timeMs)} ms for ${stats.changes} of ${numChanges} changes`, {
+				zeroEvent: "query-slow-advance",
+				...query.fields,
+				advanceTimeMs: stats.timeMs,
+				changes: stats.changes,
+				advancementChanges: numChanges,
+				timeMsByTable: Object.fromEntries(stats.timeMsByTable),
+				...suppressed > 0 && { suppressedSinceLastLog: suppressed },
+				zql: query.shape.zql
+			});
 		}
 	}
+	/**
+	* Logs the queries that took the most time in an advancement that timed
+	* out, since the reset that follows would otherwise not say what was slow.
+	* Throttled per query shape of the slowest query.
+	*/
+	#logAdvanceTimeout({ queryStats, numChanges }, reset) {
+		if (!this.#lc.warn) return;
+		const slowest = [...queryStats].toSorted(([, a], [, b]) => b.timeMs - a.timeMs).slice(0, ADVANCE_TIMEOUT_LOG_MAX_QUERIES).flatMap(([queryID, stats]) => {
+			const query = this.#queryForLog(queryID);
+			return query ? [{
+				query,
+				stats
+			}] : [];
+		});
+		if (slowest.length === 0) return;
+		const suppressed = slowAdvanceLogThrottle.admit(`timeout:${slowest[0].query.queryName ?? ""}:${slowest[0].query.shape.hash}`);
+		if (suppressed === void 0) return;
+		const summary = slowest.map(({ query, stats }) => `${query.queryName ?? query.fields.queryHash} (${Math.round(stats.timeMs)} ms)`).join(", ");
+		this.#lc.warn(`Advancement of ${numChanges} changes timed out. The queries that took the most time: ${summary}`, {
+			zeroEvent: "query-advance-timeout",
+			reason: reset.message,
+			advancementChanges: numChanges,
+			queries: slowest.map(({ query, stats }) => ({
+				...query.fields,
+				advanceTimeMs: stats.timeMs,
+				changes: stats.changes,
+				zql: query.shape.zql
+			})),
+			...suppressed > 0 && { suppressedSinceLastLog: suppressed }
+		});
+	}
 	/** Implements `BuilderDelegate.getSource()` */
 	#getSource(tableName) {
 		return getOrInsertComputed(this.#tables, tableName, (tableName) => {
@@ -637,6 +948,28 @@ var PipelineDriver = class {
 		if (!shouldFinish && elapsed > MIN_ADVANCEMENT_TIME_LIMIT_MS && (elapsed > totalHydrationTimeMs || elapsed > totalHydrationTimeMs / 2 && pos <= numChanges / 2)) throw new ResetPipelinesSignal(`Advancement exceeded timeout at ${pos} of ${numChanges} changes after ${elapsed} ms. Advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`, "advancement-timeout");
 		return checkYield && advanceTimer.elapsedLap() > this.#yieldThresholdMs();
 	}
+	/**
+	* The rows an advancement holds in memory are bounded by its reservation,
+	* but their width is not known in advance. So the bytes they are estimated
+	* to hold are added to those held by the other advancements on this worker.
+	* Says whether that takes them past the budget, or if the rows are not
+	* bounded by the reservation after all.
+	*/
+	#holdPendingRows(advanceContext) {
+		const { reservedRows, heldBytes, pos, numChanges } = advanceContext;
+		if (reservedRows === void 0) return "fits";
+		const budget = must(this.#deferredWrites);
+		const rows = this.pendingRows;
+		const bytes = this.pendingBytes;
+		advanceContext.heldBytes = bytes;
+		const bytesFit = budget.holdBytes(bytes - heldBytes);
+		if (rows > reservedRows) {
+			budget.recordRowOverrun();
+			this.#lc.error?.(`Advancement holds ${rows} rows at ${pos} of ${numChanges} changes, more than the ${reservedRows} it reserved. Writing through the rest.`);
+			return "row-overrun";
+		}
+		return bytesFit ? "fits" : "bytes";
+	}
 	#throwSlowCurrentChangeReset(pos, numChanges, elapsed, currentChangeElapsedMs, totalHydrationTimeMs) {
 		throw new ResetPipelinesSignal(`Advancement exceeded timeout processing current change at ${pos} of ${numChanges} changes after ${currentChangeElapsedMs} ms (${elapsed} ms total). Advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`, "advancement-timeout");
 	}
@@ -875,6 +1208,15 @@ function* toAdds(nodes) {
 function getRowKey(cols, row) {
 	return Object.fromEntries(cols.map((col) => [col, must(row[col])]));
 }
+/** Whether `rowKey` has exactly the columns of `primaryKey`. */
+function isKeyedBy(rowKey, primaryKey) {
+	let cols = 0;
+	for (const col in rowKey) {
+		if (!primaryKey.includes(col)) return false;
+		cols++;
+	}
+	return cols === primaryKey.length;
+}
 /**
 * Core hydration logic used by {@link PipelineDriver#addQuery}, extracted to a
 * function for reuse by the analyze-query RPC path so that analysis hydrates