@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.40

package/out/zero-cache/src/services/view-syncer/pipeline-driver.js

@@ -1,6 +1,8 @@
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { deepEqual } from "../../../../shared/src/json.js";
+import { getOrInsertComputed } from "../../../../shared/src/map.js";
 import { must } from "../../../../shared/src/must.js";
+import { assignProperty } from "../../../../shared/src/objects.js";
 import { skipYields } from "../../../../zql/src/ivm/skip-yields.js";
 import { throwOutput } from "../../../../zql/src/ivm/operator.js";
 import { MeasurePushOperator } from "../../../../zql/src/query/measure-push-operator.js";
@@ -14,13 +16,20 @@ import { resolveSimpleScalarSubqueries } from "../../../../zqlite/src/resolve-sc
 import { randInt } from "../../../../shared/src/rand.js";
 import { createSQLiteCostModel } from "../../../../zqlite/src/sqlite-cost-model.js";
 import { reloadPermissionsIfChanged } from "../../auth/load-permissions.js";
+import { LogThrottle } from "../../observability/log-throttle.js";
 import { getOrCreateCounter, getOrCreateLatencyHistogram } from "../../observability/metrics.js";
 import "../../types/row-key.js";
 import { getSubscriptionState } from "../replicator/schema/replication-state.js";
 import { checkClientSchema } from "./client-schema.js";
+import { planWarningMessage } from "./plan-warnings.js";
+import { PushCutoff } from "./push-cutoff.js";
+import { PushCostEstimator, ReadCostEstimator, shouldRebuildInsteadOfReading, shouldStartOnRebuild } from "./push-cost.js";
+import { queryShape } from "./query-shape.js";
 import { rowIDSignatureUnit } from "./row-set-signature.js";
 import { ResetPipelinesSignal } from "./snapshotter.js";
 //#region ../zero-cache/src/services/view-syncer/pipeline-driver.ts
+/** The most queries a log of an advancement timeout lists. */
+var ADVANCE_TIMEOUT_LOG_MAX_QUERIES = 3;
 /**
 * No matter how fast hydration is, advancement is given at least this long to
 * complete before doing a pipeline reset.
@@ -33,6 +42,8 @@ var MIN_PROJECTED_ADVANCEMENT_SAMPLE_MS = 5;
 var MIN_PROJECTED_ADVANCEMENT_CHANGES = 16;
 var PROJECTED_ADVANCEMENT_RESET_MULTIPLIER = 1.5;
 var LATE_ADVANCEMENT_FINISH_PROGRESS = .8;
+var planWarningLogThrottle = new LogThrottle({ windowMs: 60 * 6e4 });
+var slowAdvanceLogThrottle = new LogThrottle({ windowMs: 5 * 6e4 });
 function randomID() {
 	return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);
 }
@@ -46,9 +57,11 @@ function advancementResetTimeLimitMs(totalHydrationTimeMs) {
 function minProjectedAdvancementSampleChanges(numChanges) {
 	return Math.max(MIN_PROJECTED_ADVANCEMENT_SAMPLE_CHANGES, Math.min(MAX_PROJECTED_ADVANCEMENT_SAMPLE_CHANGES, Math.ceil(numChanges * PROJECTED_ADVANCEMENT_SAMPLE_FRACTION)));
 }
-function shouldResetProjectedAdvancement(elapsedMs, projectedTotalTimeMs, processedChanges, numChanges, totalHydrationTimeMs) {
-	if (projectedTotalTimeMs === void 0 || numChanges < MIN_PROJECTED_ADVANCEMENT_CHANGES || processedChanges < minProjectedAdvancementSampleChanges(numChanges) || elapsedMs < MIN_PROJECTED_ADVANCEMENT_SAMPLE_MS) return false;
-	return projectedTotalTimeMs > advancementResetTimeLimitMs(totalHydrationTimeMs) * PROJECTED_ADVANCEMENT_RESET_MULTIPLIER;
+function canProjectAdvancement(elapsedMs, projectedTotalTimeMs, processedChanges, numChanges) {
+	return projectedTotalTimeMs !== void 0 && numChanges >= MIN_PROJECTED_ADVANCEMENT_CHANGES && processedChanges >= minProjectedAdvancementSampleChanges(numChanges) && elapsedMs >= MIN_PROJECTED_ADVANCEMENT_SAMPLE_MS;
+}
+function shouldResetProjectedAdvancement(elapsedMs, projectedTotalTimeMs, totalHydrationTimeMs) {
+	return projectedTotalTimeMs - elapsedMs > advancementResetTimeLimitMs(totalHydrationTimeMs) * PROJECTED_ADVANCEMENT_RESET_MULTIPLIER;
 }
 function shouldFinishLateAdvancement(processedChanges, numChanges) {
 	return numChanges > 0 && processedChanges / numChanges >= LATE_ADVANCEMENT_FINISH_PROGRESS;
@@ -56,6 +69,14 @@ function shouldFinishLateAdvancement(processedChanges, numChanges) {
 function shouldResetSlowCurrentChange(currentChangeElapsedMs, totalHydrationTimeMs) {
 	return currentChangeElapsedMs > MIN_ADVANCEMENT_TIME_LIMIT_MS && currentChangeElapsedMs > advancementResetTimeLimitMs(totalHydrationTimeMs);
 }
+function formatAdvancementBreakdown(advanceContext, elapsed) {
+	const { diff, pushTimeMs, stage, currentChangeStartMs, currentTable } = advanceContext;
+	const totalPushMs = pushTimeMs + (stage === "push" && currentChangeStartMs !== void 0 ? Math.max(0, elapsed - currentChangeStartMs) : 0);
+	const stats = diff.stats;
+	const diffTimeMs = stats?.totalDiffMs ?? 0;
+	const clMs = stats ? ` [changelog: ${stats.changeLogReadMs.toFixed(1)}ms, parse: ${stats.parseMs.toFixed(1)}ms, rows: ${stats.rowReadMs.toFixed(1)}ms]` : "";
+	return `stage: ${stage}, diff: ${diffTimeMs.toFixed(1)}ms${clMs}, push: ${totalPushMs.toFixed(1)}ms` + (currentTable ? `, table: ${currentTable}` : "");
+}
 /**
 * Manages the state of IVM pipelines for a given ViewSyncer (i.e. client group).
 */
@@ -67,39 +88,83 @@ var PipelineDriver = class {
 	* query, maintained as RowChanges are yielded from {@link addQuery} and
 	* {@link advance}. ADDs / REMOVEs XOR the row's unit in (XOR is
 	* self-inverse, so one op serves both directions); EDITs are no-ops.
-	* Hydration implicitly reseeds from `0n` because {@link addQuery} calls
-	* {@link removeQuery} first, which deletes the entry.
+	* Hydration starts the query's entry at `0n` (the empty row set), so every
+	* hydrated query has one, even if it has no rows. A hydration that does not
+	* finish removes it.
 	*/
 	#rowSetSignatures = /* @__PURE__ */ new Map();
+	/**
+	* IDs of the queries whose {@link #rowSetSignatures} entry changed since the
+	* last {@link clearChangedRowSetSignatures}, so that the CVR flush only has
+	* to look at those instead of every query.
+	*/
+	#changedRowSetSignatures = /* @__PURE__ */ new Set();
 	#lc;
 	#snapshotter;
 	#storage;
 	#shardID;
 	#logConfig;
 	#config;
+	#deferredWrites;
 	#tableSpecs = /* @__PURE__ */ new Map();
 	#allTableNames = /* @__PURE__ */ new Set();
 	#costModels;
+	#planWarningThresholds;
 	#yieldThresholdMs;
 	#streamer = null;
+	/** The row changes of the push in progress, until #push yields them. */
+	#pushedRowChanges = null;
 	#hydrateContext = null;
 	#advanceContext = null;
+	/**
+	* Whether a {@link CostBasedAdvance} has been set up and has not
+	* completed. It may already have removed pipelines, but the sources only
+	* move to the new snapshot when the iteration over its changes completes,
+	* so nothing can be hydrated until then. If the iteration is given up on
+	* or throws, they never do, and nothing can be hydrated until the
+	* pipelines are {@link reset()}.
+	*/
+	#costBasedAdvanceIncomplete = false;
+	/**
+	* The queries abandoned by a {@link CostBasedAdvance} that have not been
+	* added again since.
+	*/
+	#abandoned = /* @__PURE__ */ new Map();
+	#pushCosts = new PushCostEstimator();
+	#readCosts = new ReadCostEstimator();
 	#replicaVersion = null;
 	#primaryKeys = null;
 	#permissions = null;
 	#advanceTime = getOrCreateLatencyHistogram("sync", "ivm.advance-time", "Time to advance all queries for a given client group in response to a single change.");
 	#conflictRowsDeleted = getOrCreateCounter("sync", "ivm.conflict-rows-deleted", "Number of rows deleted because they conflicted with added row");
+	#deferredWritesFallbacks = getOrCreateCounter("sync", "ivm.deferred-writes-fallbacks", "Number of advancements written through to the replica snapshot because their changes did not fit in the deferred IVM writes budget, from the start or partway (because of the bytes held, a change that needed more rows than were left, or more rows held than reserved)");
 	#inspectorDelegate;
-	constructor(lc, logConfig, snapshotter, shardID, storage, clientGroupID, inspectorDelegate, yieldThresholdMs, enablePlanner, config) {
+	#queryStats;
+	/**
+	* Passes the pipelines' metrics on to the inspector, and accounts the time
+	* spent pushing to each query to the advancement in progress.
+	*/
+	#metricsDelegate = { addMetric: (metric, value, ...args) => {
+		this.#inspectorDelegate.addMetric(metric, value, ...args);
+		if (metric === "query-update-server") this.#recordAdvancePush(args[0], value);
+	} };
+	constructor(lc, logConfig, snapshotter, shardID, storage, clientGroupID, inspectorDelegate, yieldThresholdMs, enablePlanner, config, deferredWrites, queryStats) {
 		this.#lc = lc.withContext("clientGroupID", clientGroupID);
 		this.#snapshotter = snapshotter;
 		this.#storage = storage;
 		this.#shardID = shardID;
 		this.#logConfig = logConfig;
 		this.#config = config;
+		this.#deferredWrites = deferredWrites;
 		this.#inspectorDelegate = inspectorDelegate;
 		this.#costModels = enablePlanner ? /* @__PURE__ */ new WeakMap() : void 0;
+		const planWarningThresholds = {
+			rows: logConfig.planWarningRowThreshold,
+			cost: logConfig.planWarningCostThreshold
+		};
+		this.#planWarningThresholds = planWarningThresholds.rows > 0 || planWarningThresholds.cost > 0 ? planWarningThresholds : void 0;
 		this.#yieldThresholdMs = yieldThresholdMs;
+		this.#queryStats = queryStats;
 	}
 	/**
 	* Initializes the PipelineDriver to the current head of the database.
@@ -128,9 +193,12 @@ var PipelineDriver = class {
 			this.#pipelines.delete(queryID);
 			this.#destroyPipeline(queryID, pipeline, "reset");
 		}
+		this.#abandoned.clear();
+		this.#costBasedAdvanceIncomplete = false;
 		this.#tables.clear();
 		this.#allTableNames.clear();
 		this.#rowSetSignatures.clear();
+		this.#changedRowSetSignatures.clear();
 		this.#initAndResetCommon(clientSchema);
 	}
 	#initAndResetCommon(clientSchema) {
@@ -140,11 +208,9 @@ var PipelineDriver = class {
 		checkClientSchema(this.#shardID, clientSchema, this.#tableSpecs, fullTables);
 		this.#allTableNames.clear();
 		for (const table of fullTables.keys()) this.#allTableNames.add(table);
-		const primaryKeys = this.#primaryKeys ?? /* @__PURE__ */ new Map();
+		const primaryKeys = buildRowKeys(clientSchema, this.#tableSpecs, this.#primaryKeys ?? void 0);
 		this.#primaryKeys = primaryKeys;
-		primaryKeys.clear();
-		for (const [table, spec] of this.#tableSpecs.entries()) primaryKeys.set(table, spec.tableSpec.primaryKey);
-		buildPrimaryKeys(clientSchema, primaryKeys);
+		this.#streamer ??= new Streamer(primaryKeys, this.#tableSpecs);
 		const { replicaVersion } = getSubscriptionState(db);
 		this.#replicaVersion = replicaVersion;
 	}
@@ -173,10 +239,28 @@ var PipelineDriver = class {
 		}
 		return this.#permissions;
 	}
-	advanceWithoutDiff() {
-		const { db, version } = this.#snapshotter.advanceWithoutDiff().curr;
-		for (const table of this.#tables.values()) table.setDB(db.db);
-		return version;
+	/**
+	* Advances the snapshot to the head of the database without diffing the
+	* change log, in preparation for hydrating queries at head.
+	*
+	* Throws a {@link ResetPipelinesSignal} if the change log records a
+	* schema change since the previous snapshot. The table specs (and any
+	* TableSources built from them) were computed at or before that snapshot
+	* and are stale with respect to the new head, so the caller must
+	* {@link reset()} before hydrating. ({@link advance()} detects this when
+	* the diff encounters the RESET op; this path skips the diff and so must
+	* check explicitly.)
+	*
+	* @param releasePreviousSnapshot Whether to release the snapshot that the
+	*        driver was at, which is otherwise left open until the next
+	*        advance (see `Snapshotter.releasePrevious()`).
+	*/
+	advanceWithoutDiff(releasePreviousSnapshot = false) {
+		const { prev, curr } = this.#snapshotter.advanceWithoutDiff();
+		if (curr.schemaChangedSince(prev.version)) throw new ResetPipelinesSignal(`schema changed between ${prev.version} and ${curr.version}`, "schema-change");
+		for (const table of this.#tables.values()) table.setDB(curr.db.db);
+		if (releasePreviousSnapshot) this.#snapshotter.releasePrevious();
+		return curr.version;
 	}
 	#ensureCostModelExistsIfEnabled(db) {
 		let existing = this.#costModels?.get(db);
@@ -196,7 +280,9 @@ var PipelineDriver = class {
 			this.#pipelines.delete(queryID);
 			this.#destroyPipeline(queryID, pipeline, "destroy");
 		}
+		this.#tables.clear();
 		this.#rowSetSignatures.clear();
+		this.#changedRowSetSignatures.clear();
 		this.#storage.destroy();
 		this.#snapshotter.destroy();
 	}
@@ -204,20 +290,91 @@ var PipelineDriver = class {
 	queries() {
 		return this.#pipelines;
 	}
+	/**
+	* The queries whose pipelines a {@link CostBasedAdvance} removed and that
+	* have not been hydrated again since, by query ID. They are not in
+	* {@link queries()}. A query whose hydration is started but does not
+	* finish is still one of them.
+	*/
+	abandonedQueries() {
+		return this.#abandoned;
+	}
+	/**
+	* Stats from the hydration of the pipeline for `queryID`, or `undefined` if
+	* the query has no pipeline.
+	*/
+	hydrationStats(queryID) {
+		const pipeline = this.#pipelines.get(queryID);
+		return pipeline ? {
+			rowCount: pipeline.hydrationRowCount,
+			rowsRead: pipeline.hydrationRowsRead,
+			planWarnings: pipeline.planWarnings,
+			shape: pipeline.shape
+		} : void 0;
+	}
+	/**
+	* Logs what the planner warned about the plan it chose for `query`, at
+	* most once per query shape per {@link PLAN_WARNING_LOG_WINDOW_MS}.
+	*/
+	#logPlanWarnings(shape, { queryHash, transformationHash, queryName }, warnings) {
+		if (!this.#lc.warn) return;
+		const suppressed = planWarningLogThrottle.admit(`${queryName ?? ""}:${shape.hash}`);
+		if (suppressed === void 0) return;
+		const messages = warnings.map(planWarningMessage);
+		this.#lc.warn(`Query plan warning${queryName === void 0 ? "" : ` for ${queryName}`}: ` + messages.join(" "), {
+			zeroEvent: "query-plan-warning",
+			queryHash,
+			transformationHash,
+			...queryName !== void 0 && { queryName },
+			queryShape: shape.hash,
+			warnings,
+			...suppressed > 0 && { suppressedSinceLastLog: suppressed },
+			zql: shape.zql
+		});
+	}
+	#totalRowsRead() {
+		let total = 0;
+		for (const table of this.#tables.values()) total += table.rowsRead;
+		return total;
+	}
 	totalHydrationTimeMs() {
 		let total = 0;
 		for (const pipeline of this.#pipelines.values()) total += pipeline.hydrationTimeMs;
 		return total;
 	}
-	#logQueryPipelineLifecycle({ zeroEvent, pipelineRunID, queryHash, transformationHash, queryName, hydrationReason, stopReason, hydrationTimeMs, hydrationRowCount, pipelineLifetimeMs }) {
+	/**
+	* The rows the sources hold in memory for the advancement in progress.
+	* The advancement's reservation from the {@link DeferredWritesBudget}
+	* bounds this.
+	*/
+	get pendingRows() {
+		let total = 0;
+		for (const source of this.#tables.values()) total += source.pendingRows;
+		return total;
+	}
+	/**
+	* The estimated bytes of {@link pendingRows}, which are added to the
+	* {@link DeferredWritesBudget}'s held bytes as the advancement goes.
+	*/
+	get pendingBytes() {
+		let total = 0;
+		for (const source of this.#tables.values()) total += source.pendingBytes;
+		return total;
+	}
+	#logQueryPipelineLifecycle({ zeroEvent, pipelineRunID, queryHash, transformationHash, queryName, hydrationReason, stopReason, hydrationTimeMs, hydrationRowCount, hydrationRowsRead, pipelineLifetimeMs, abandonReason, estimatedPushMs, pushTimeMs, pushedChanges, level = "info" }) {
 		let lc = this.#lc.withContext("zeroEvent", zeroEvent).withContext("pipelineRunID", pipelineRunID).withContext("queryHash", queryHash).withContext("transformationHash", transformationHash);
 		if (queryName !== void 0) lc = lc.withContext("queryName", queryName);
 		if (hydrationReason !== void 0) lc = lc.withContext("hydrationReason", hydrationReason);
 		if (stopReason !== void 0) lc = lc.withContext("stopReason", stopReason);
 		if (hydrationTimeMs !== void 0) lc = lc.withContext("hydrationTimeMs", hydrationTimeMs);
 		if (hydrationRowCount !== void 0) lc = lc.withContext("hydrationRowCount", hydrationRowCount);
+		if (hydrationRowsRead !== void 0) lc = lc.withContext("hydrationRowsRead", hydrationRowsRead);
 		if (pipelineLifetimeMs !== void 0) lc = lc.withContext("pipelineLifetimeMs", pipelineLifetimeMs);
-		lc.info?.("query pipeline lifecycle");
+		if (abandonReason !== void 0) lc = lc.withContext("abandonReason", abandonReason);
+		if (estimatedPushMs !== void 0) lc = lc.withContext("estimatedPushMs", estimatedPushMs);
+		if (pushTimeMs !== void 0) lc = lc.withContext("pushTimeMs", pushTimeMs);
+		if (pushedChanges !== void 0) lc = lc.withContext("pushedChanges", pushedChanges);
+		lc[level]?.("query pipeline lifecycle");
 	}
 	/**
 	* A `{scalar: true}` that cannot be honored degrades silently to a plain
@@ -232,35 +389,39 @@ var PipelineDriver = class {
 			this.#lc.warn?.(`Ignoring {scalar: true} on the "${table}" subquery of query ${queryID}: it does not constrain every column of any unique key ${keys.length > 0 ? `[${keys}]` : "(none on this table)"} to a literal with "=", so it is not provably limited to one row. The gate runs as a plain EXISTS.`);
 		}
 	}
-	#resolveScalarSubqueries(ast) {
-		const companionRows = [];
+	#disableCorrelatedPredicatePushdown() {
+		return this.#config?.enableCorrelatedPredicatePushdown === false;
+	}
+	#resolveScalarSubqueries(ast, tables) {
 		const companionInputs = [];
 		const executor = (subqueryAST, childField) => {
 			const input = buildPipeline(subqueryAST, {
-				getSource: (name) => this.#getSource(name),
+				enableNotExists: true,
+				disableCorrelatedPredicatePushdown: this.#disableCorrelatedPredicatePushdown(),
+				getSource: (name) => {
+					tables.add(name);
+					return this.#getSource(name);
+				},
 				createStorage: () => this.#createStorage(),
 				decorateSourceInput: (input) => input,
 				decorateInput: (input) => input,
 				addEdge() {},
 				decorateFilterInput: (input) => input
 			}, "scalar-subquery");
-			let node;
-			for (const n of skipYields(input.fetch({}))) node ??= n;
-			if (!node) {
-				companionInputs.push(input);
-				return;
-			}
-			companionRows.push({
-				table: subqueryAST.table,
-				row: node.row
-			});
 			companionInputs.push(input);
-			return node.row[childField] ?? null;
+			return fetchScalarValue(input, childField);
 		};
-		const { ast: resolved, companions, ignoredScalarHints } = resolveSimpleScalarSubqueries(ast, this.#tableSpecs, executor);
+		let resolved;
+		let companions;
+		let ignoredScalarHints;
+		try {
+			({ast: resolved, companions, ignoredScalarHints} = resolveSimpleScalarSubqueries(ast, this.#tableSpecs, executor));
+		} catch (e) {
+			for (const input of companionInputs) input.destroy();
+			throw e;
+		}
 		return {
 			ast: resolved,
-			companionRows,
 			companions,
 			companionInputs,
 			ignoredScalarHints
@@ -288,57 +449,74 @@ var PipelineDriver = class {
 		assert(this.initialized(), "Pipeline driver must be initialized before adding queries");
 		this.removeQuery(queryID, "replace-query");
 		const pipelineRunID = randomID();
+		const shape = queryShape(query);
+		const level = this.#abandoned.has(queryID) ? "debug" : "info";
 		this.#logQueryPipelineLifecycle({
 			zeroEvent: "query-pipeline-hydrate-start",
 			pipelineRunID,
 			queryHash: queryID,
 			transformationHash,
 			queryName,
-			hydrationReason
+			hydrationReason,
+			level
 		});
-		const debugDelegate = runtimeDebugFlags.trackRowsVended ? new Debug() : void 0;
+		const debugDelegate = runtimeDebugFlags.trackRowsVended ? new Debug(true) : void 0;
 		const costModel = this.#ensureCostModelExistsIfEnabled(this.#snapshotter.current().db.db);
 		assert(this.#advanceContext === null, "Cannot hydrate while advance is in progress");
+		assert(!this.#costBasedAdvanceIncomplete, "Cannot hydrate before the advance has completed");
 		this.#hydrateContext = { timer };
+		const rowsReadAtStart = this.#totalRowsRead();
 		let hydrationFinished = false;
 		let hydrationFailed = false;
 		let hydrationRowCount = 0;
+		let planWarnings = [];
+		const planWarningThresholds = this.#planWarningThresholds;
+		let builtInputs = [];
+		const tables = /* @__PURE__ */ new Set();
 		try {
-			const { ast: resolvedQuery, companionRows, companions: companionMeta, companionInputs, ignoredScalarHints } = this.#resolveScalarSubqueries(query);
+			this.#rowSetSignatures.set(queryID, 0n);
+			this.#changedRowSetSignatures.add(queryID);
+			const { ast: resolvedQuery, companions: companionMeta, companionInputs, ignoredScalarHints } = this.#resolveScalarSubqueries(query, tables);
+			builtInputs = [...companionInputs];
 			this.#warnIgnoredScalarHints(queryID, ignoredScalarHints);
 			const input = buildPipeline(resolvedQuery, {
 				debug: debugDelegate,
 				enableNotExists: true,
-				getSource: (name) => this.#getSource(name),
+				disableCorrelatedPredicatePushdown: this.#disableCorrelatedPredicatePushdown(),
+				enablePlannerAwarePushdown: this.#config?.enablePlannerAwarePushdown !== false,
+				getSource: (name) => {
+					tables.add(name);
+					return this.#getSource(name);
+				},
 				createStorage: () => this.#createStorage(),
-				decorateSourceInput: (input, _queryID) => new MeasurePushOperator(new QueryFailureLoggingOperator(this.#lc, input, queryID, transformationHash, queryName), queryID, this.#inspectorDelegate, "query-update-server"),
+				decorateSourceInput: (input, _queryID) => new MeasurePushOperator(new QueryFailureLoggingOperator(this.#lc, input, queryID, transformationHash, queryName), queryID, this.#metricsDelegate, "query-update-server"),
 				decorateInput: (input) => input,
 				addEdge() {},
-				decorateFilterInput: (input) => input
+				decorateFilterInput: (input) => input,
+				planWarnings: planWarningThresholds && {
+					thresholds: planWarningThresholds,
+					report: (warnings) => {
+						planWarnings = warnings;
+					}
+				}
 			}, queryID, costModel);
+			builtInputs.push(input);
+			if (planWarnings.length > 0) this.#logPlanWarnings(shape, {
+				queryHash: queryID,
+				transformationHash,
+				queryName
+			}, planWarnings);
 			const schema = input.getSchema();
-			input.setOutput({ push: (change) => {
-				const streamer = this.#streamer;
-				assert(streamer, "must #startAccumulating() before pushing changes");
-				streamer.accumulate(queryID, schema, [change]);
-				return [];
-			} });
-			for (const change of hydrateInternal(input, queryID, must(this.#primaryKeys), this.#tableSpecs)) {
-				if (change !== "yield") hydrationRowCount++;
-				yield change;
-			}
-			for (const { table, row } of companionRows) {
-				const primaryKey = mustGetPrimaryKey(this.#primaryKeys, table);
-				hydrationRowCount++;
-				yield {
-					type: 0,
-					queryID,
-					table,
-					rowKey: getRowKey(primaryKey, row),
-					row
-				};
+			input.setOutput({ push: (change) => this.#streamPushed(queryID, schema, change) });
+			for (let i = -1; i < companionInputs.length; i++) {
+				const pipeline = i < 0 ? input : companionInputs[i];
+				for (const change of must(this.#streamer).streamNodes(queryID, i < 0 ? schema : companionStreamSchema(pipeline, companionMeta[i]), 0, pipeline.fetch({}))) {
+					if (change !== "yield") hydrationRowCount++;
+					yield change;
+				}
 			}
 			const hydrationTimeMs = timer.totalElapsed();
+			const hydrationRowsRead = this.#totalRowsRead() - rowsReadAtStart;
 			if (runtimeDebugFlags.trackRowCountsVended) {
 				if (hydrationTimeMs > this.#logConfig.slowHydrateThreshold) {
 					let totalRowsConsidered = 0;
@@ -356,7 +534,7 @@ var PipelineDriver = class {
 			for (let i = 0; i < companionMeta.length; i++) {
 				const meta = companionMeta[i];
 				const companionInput = companionInputs[i];
-				const companionSchema = companionInput.getSchema();
+				const companionSchema = companionStreamSchema(companionInput, meta);
 				const { childField, resolvedValue } = meta;
 				companionInput.setOutput({ push: (change) => {
 					let newValue;
@@ -368,13 +546,10 @@ var PipelineDriver = class {
 						case 1:
 							newValue = void 0;
 							break;
-						case 3: return [];
+						case 3: return this.#streamPushed(queryID, companionSchema, change);
 					}
 					if (!scalarValuesEqual(newValue, resolvedValue)) throw new ResetPipelinesSignal(`Scalar subquery value changed for ${meta.ast.table}: ${String(resolvedValue)} -> ${String(newValue)}`, "scalar-subquery");
-					const streamer = this.#streamer;
-					assert(streamer, "must #startAccumulating() before pushing changes");
-					streamer.accumulate(queryID, companionSchema, [change]);
-					return [];
+					return this.#streamPushed(queryID, companionSchema, change);
 				} });
 				liveCompanions.push({
 					input: companionInput,
@@ -387,15 +562,31 @@ var PipelineDriver = class {
 				input,
 				hydrationTimeMs,
 				hydrationRowCount,
+				hydrationRowsRead,
+				planWarnings,
 				hydrationReason,
 				pipelineRunID,
 				pipelineReadyAtMs,
 				transformedAst: resolvedQuery,
+				originalAst: query,
+				shape,
 				transformationHash,
 				...queryName !== void 0 && { queryName },
-				companions: liveCompanions
+				companions: liveCompanions,
+				tables
 			});
+			this.#abandoned.delete(queryID);
 			hydrationFinished = true;
+			this.#queryStats?.recordHydration({
+				queryName,
+				shape
+			}, {
+				outcome: "finished",
+				timeMs: hydrationTimeMs,
+				rowCount: hydrationRowCount,
+				rowsRead: hydrationRowsRead,
+				planWarnings
+			});
 			this.#logQueryPipelineLifecycle({
 				zeroEvent: "query-pipeline-hydrate-finish",
 				pipelineRunID,
@@ -404,10 +595,19 @@ var PipelineDriver = class {
 				queryName,
 				hydrationReason,
 				hydrationTimeMs,
-				hydrationRowCount
+				hydrationRowCount,
+				hydrationRowsRead,
+				level
 			});
 		} catch (e) {
 			hydrationFailed = true;
+			this.#queryStats?.recordHydration({
+				queryName,
+				shape
+			}, {
+				outcome: "failed",
+				timeMs: timer.totalElapsed()
+			});
 			this.#logQueryPipelineLifecycle({
 				zeroEvent: "query-pipeline-hydrate-failed",
 				pipelineRunID,
@@ -425,16 +625,30 @@ var PipelineDriver = class {
 			}, "query hydration failed", e);
 			throw e;
 		} finally {
-			if (!hydrationFinished && !hydrationFailed) this.#logQueryPipelineLifecycle({
-				zeroEvent: "query-pipeline-hydrate-aborted",
-				pipelineRunID,
-				queryHash: queryID,
-				transformationHash,
-				queryName,
-				hydrationReason,
-				hydrationTimeMs: timer.totalElapsed(),
-				hydrationRowCount
-			});
+			if (!hydrationFinished && !hydrationFailed) {
+				this.#queryStats?.recordHydration({
+					queryName,
+					shape
+				}, {
+					outcome: "aborted",
+					timeMs: timer.totalElapsed()
+				});
+				this.#logQueryPipelineLifecycle({
+					zeroEvent: "query-pipeline-hydrate-aborted",
+					pipelineRunID,
+					queryHash: queryID,
+					transformationHash,
+					queryName,
+					hydrationReason,
+					hydrationTimeMs: timer.totalElapsed(),
+					hydrationRowCount
+				});
+			}
+			if (!hydrationFinished) {
+				for (const input of builtInputs) input.destroy();
+				this.#rowSetSignatures.delete(queryID);
+				this.#changedRowSetSignatures.delete(queryID);
+			}
 			this.#hydrateContext = null;
 		}
 	}
@@ -443,14 +657,24 @@ var PipelineDriver = class {
 	* was not added.
 	*/
 	removeQuery(queryID, stopReason = "remove-query") {
+		this.#removeQuery(queryID, stopReason);
+	}
+	/**
+	* @param abandoned If an advancement is abandoning the query's pipeline,
+	*        the record of that, which is logged.
+	*/
+	#removeQuery(queryID, stopReason, abandoned) {
 		const pipeline = this.#pipelines.get(queryID);
 		if (pipeline) {
 			this.#pipelines.delete(queryID);
-			this.#destroyPipeline(queryID, pipeline, stopReason);
+			this.#destroyPipeline(queryID, pipeline, stopReason, abandoned);
 		}
 		this.#rowSetSignatures.delete(queryID);
+		this.#changedRowSetSignatures.delete(queryID);
+		if (stopReason !== "replace-query") this.#abandoned.delete(queryID);
+		if (stopReason === "remove-query") this.#pushCosts.forget(queryID);
 	}
-	#destroyPipeline(queryID, pipeline, stopReason) {
+	#destroyPipeline(queryID, pipeline, stopReason, abandoned) {
 		this.#logQueryPipelineLifecycle({
 			zeroEvent: "query-pipeline-stop",
 			pipelineRunID: pipeline.pipelineRunID,
@@ -461,7 +685,14 @@ var PipelineDriver = class {
 			stopReason,
 			hydrationTimeMs: pipeline.hydrationTimeMs,
 			hydrationRowCount: pipeline.hydrationRowCount,
-			pipelineLifetimeMs: Date.now() - pipeline.pipelineReadyAtMs
+			pipelineLifetimeMs: Date.now() - pipeline.pipelineReadyAtMs,
+			...abandoned && {
+				abandonReason: abandoned.reason,
+				estimatedPushMs: abandoned.estimatedPushMs,
+				pushTimeMs: abandoned.pushTimeMs,
+				pushedChanges: abandoned.changes,
+				level: "debug"
+			}
 		});
 		pipeline.input.destroy();
 		for (const companion of pipeline.companions) companion.input.destroy();
@@ -475,6 +706,20 @@ var PipelineDriver = class {
 		return this.#rowSetSignatures.get(queryID);
 	}
 	/**
+	* IDs of the queries whose {@link rowSetSignature} changed (by an ADD or
+	* REMOVE from {@link addQuery} or {@link advance}) since the last call to
+	* {@link clearChangedRowSetSignatures}.
+	*/
+	changedRowSetSignatures() {
+		return this.#changedRowSetSignatures;
+	}
+	/**
+	* Called once the {@link changedRowSetSignatures} have been persisted.
+	*/
+	clearChangedRowSetSignatures() {
+		this.#changedRowSetSignatures.clear();
+	}
+	/**
 	* Wraps an iterable of RowChanges, XORing each row's unit hash into the
 	* query's signature (ADDs and REMOVEs share the same op; EDITs are no-ops).
 	* Used to intercept the yield streams from {@link addQuery} and
@@ -490,6 +735,7 @@ var PipelineDriver = class {
 					rowKey: change.rowKey
 				});
 				this.#rowSetSignatures.set(change.queryID, cur ^ unit);
+				this.#changedRowSetSignatures.add(change.queryID);
 			}
 			yield change;
 		}
@@ -510,44 +756,212 @@ var PipelineDriver = class {
 	*        measure the progress of the advancement and abort with a
 	*        {@link ResetPipelinesSignal} if it is estimated to take longer
 	*        than a hydration.
+	* @param costBased If given, the advancement is bounded query by query
+	*        instead (see {@link CostBasedAdvance}).
 	* @return The resulting row changes for all added queries. Note that the
 	*         `changes` must be iterated over in their entirety in order to
 	*         advance the database snapshot.
+	* @throws A {@link ResetPipelinesSignal} if `costBased` is given and the
+	*         changes include a schema change, a truncation or a change of
+	*         the permissions. It is thrown before any change is pushed or
+	*         any pipeline is removed, as the caller resets the pipelines
+	*         either way. Without `costBased`, it is iterating over the
+	*         `changes` that throws it, on getting to that change.
 	*/
-	advance(timer) {
+	advance(timer, costBased) {
 		assert(this.initialized(), "Pipeline driver must be initialized before advancing");
-		const diff = this.#snapshotter.advance(this.#tableSpecs, this.#allTableNames);
+		let diff;
+		let context;
+		if (costBased === void 0) diff = this.#snapshotter.advance(this.#tableSpecs, this.#allTableNames, this.#tables, "divergent");
+		else {
+			const { prev, curr } = this.#snapshotter.advanceWithoutDiff();
+			const counts = curr.countChangesSince(prev.version);
+			const reset = this.#snapshotter.pendingReset(this.#tableSpecs, counts);
+			if (reset) throw reset;
+			context = this.#startCostBasedAdvance(costBased, curr.version > prev.version, counts);
+			diff = this.#snapshotter.diff(this.#tableSpecs, this.#allTableNames, this.#tables, "divergent", counts, context.abandoned.length > 0 ? this.#tablesRead() : void 0);
+			this.#costBasedAdvanceIncomplete = true;
+		}
 		const { prev, curr, changes } = diff;
 		this.#lc.debug?.(`advance ${prev.version} => ${curr.version}: ${changes} changes`);
+		const progress = { changesProcessed: 0 };
 		return {
 			version: curr.version,
 			numChanges: changes,
-			changes: this.#trackRowSetSignatures(this.#advance(diff, timer, changes))
+			changes: this.#trackRowSetSignatures(this.#advance(diff, timer, changes, progress, context)),
+			abandoned: () => context?.abandoned ?? [],
+			estimates: () => context?.estimates ?? [],
+			progress: () => ({
+				changesScanned: diff.changesScanned,
+				changesProcessed: progress.changesProcessed,
+				changesSkipped: diff.changesSkipped
+			})
 		};
 	}
-	*#advance(diff, timer, numChanges) {
+	/** The tables that the pipelines read. */
+	#tablesRead() {
+		const tables = /* @__PURE__ */ new Set();
+		for (const pipeline of this.#pipelines.values()) for (const table of pipeline.tables) tables.add(table);
+		return tables;
+	}
+	/**
+	* Removes the pipelines of the queries that are estimated to be cheaper to
+	* hydrate again than to push the advancement's changes to: first each
+	* query for what pushing costs it, and then the ones that are left for
+	* what reading the changes costs them together.
+	*
+	* @param canAbandon Whether pipelines can be removed, by this or by the
+	*        cutoff.
+	* @param counts The rows that changed since the previous snapshot.
+	*/
+	#startCostBasedAdvance({ shouldCutOff, rebuildMargin }, canAbandon, counts) {
+		const changes = counts.byTable;
+		const estimatedPushMs = /* @__PURE__ */ new Map();
+		const abandoned = [];
+		if (canAbandon) {
+			for (const [queryID, pipeline] of this.#pipelines) {
+				const estimate = this.#pushCosts.estimate(queryID, pipeline.tables, changes);
+				if (estimate === void 0) continue;
+				if (shouldStartOnRebuild(estimate, pipeline.hydrationTimeMs, rebuildMargin)) this.#abandon(abandoned, "estimate", pipeline, {
+					queryID,
+					pushTimeMs: 0,
+					changes: 0,
+					hydrationTimeMs: pipeline.hydrationTimeMs,
+					estimatedPushMs: estimate
+				});
+				else estimatedPushMs.set(queryID, estimate);
+			}
+			this.#abandonInsteadOfReading(counts, rebuildMargin, estimatedPushMs, abandoned);
+		}
+		return {
+			cutoff: new PushCutoff(canAbandon ? shouldCutOff : () => false),
+			entries: counts.total,
+			estimatedPushMs,
+			abandoned,
+			estimates: []
+		};
+	}
+	/**
+	* Removes the pipelines that are left, if reading the advancement's
+	* changes and pushing them to those is estimated to cost more than
+	* hydrating them again. The changes are then not read at all.
+	*/
+	#abandonInsteadOfReading(counts, rebuildMargin, estimatedPushMs, abandoned) {
+		if (this.#pipelines.size === 0) return;
+		let entriesOfTablesRead = 0;
+		for (const table of this.#tablesRead()) entriesOfTablesRead += counts.byTable.get(table) ?? 0;
+		const queries = Array.from(this.#pipelines, ([queryID, pipeline]) => ({
+			queryID,
+			pipeline,
+			estimatedPushMs: estimatedPushMs.get(queryID),
+			hydrationTimeMs: pipeline.hydrationTimeMs
+		}));
+		const estimatedReadMs = this.#readCosts.estimate({
+			walk: counts.total,
+			parse: abandoned.length > 0 ? entriesOfTablesRead : counts.total,
+			lookup: entriesOfTablesRead
+		});
+		if (!shouldRebuildInsteadOfReading(estimatedReadMs, queries, rebuildMargin)) return;
+		let pushMs = 0;
+		let hydrationMs = 0;
+		for (const query of queries) {
+			pushMs += query.estimatedPushMs ?? 0;
+			hydrationMs += query.hydrationTimeMs;
+		}
+		this.#lc.debug?.("Not reading a batch of changes: hydrating its queries again is estimated to cost less", {
+			entries: counts.total,
+			entriesOfTablesRead,
+			estimatedReadMs,
+			estimatedPushMs: pushMs,
+			hydrationTimeMs: hydrationMs,
+			rebuildMargin,
+			queries: queries.length
+		});
+		for (const { queryID, pipeline, ...estimate } of queries) this.#abandon(abandoned, "read", pipeline, {
+			queryID,
+			pushTimeMs: 0,
+			changes: 0,
+			...estimate
+		});
+		estimatedPushMs.clear();
+	}
+	#abandon(abandoned, reason, pipeline, progress) {
+		const { transformationHash, transformedAst, originalAst, queryName } = pipeline;
+		const query = {
+			...progress,
+			reason,
+			transformationHash,
+			transformedAst,
+			originalAst,
+			...queryName !== void 0 && { queryName }
+		};
+		this.#removeQuery(progress.queryID, "abandon-advance", query);
+		this.#abandoned.set(progress.queryID, query);
+		abandoned.push(query);
+		this.#queryStats?.recordAbandoned(pipeline, reason);
+	}
+	*#advance(diff, timer, numChanges, progress, costBased) {
 		assert(this.#hydrateContext === null, "Cannot advance while hydration is in progress");
 		const totalHydrationTimeMs = this.totalHydrationTimeMs();
-		this.#advanceContext = {
+		const advanceContext = {
 			timer,
+			progress,
+			diff,
+			completedChanges: 0,
 			totalHydrationTimeMs,
 			numChanges,
+			reservedRows: void 0,
+			heldBytes: 0,
 			currentChangeStartMs: void 0,
-			pos: 0
+			pos: 0,
+			currentTable: void 0,
+			queryStats: /* @__PURE__ */ new Map(),
+			costBased,
+			pushTimeMs: 0,
+			stage: "diff"
 		};
-		this.#lc.debug?.(`starting pipeline advancement of ${numChanges} changes with an advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`);
+		this.#advanceContext = advanceContext;
+		let timedOut = false;
+		let readToEnd = true;
 		try {
-			for (const { table, prevValues, nextValue } of diff) {
-				if (this.#shouldAdvanceYieldMaybeAbortAdvance()) yield "yield";
+			advanceContext.reservedRows = this.#reserveDeferredWrites(numChanges);
+			const deferWrites = advanceContext.reservedRows !== void 0;
+			if (deferWrites) diff.setPrevWrites("none");
+			for (const table of this.#tables.values()) table.setDeferWrites(deferWrites);
+			this.#lc.debug?.(`starting pipeline advancement of ${numChanges} changes with an advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms (${deferWrites ? "deferred" : "write-through"}).`);
+			for (const { table, prevValues: probedPrevValues, nextValue, rowKey } of diff) {
+				advanceContext.stage = "between";
+				advanceContext.completedChanges = diff.changesScanned - 1;
+				if (this.#checkAdvance(advanceContext)) yield "yield";
+				if (costBased) {
+					this.#cutOffQueries(advanceContext, costBased, diff, table);
+					if (this.#pipelines.size === 0) {
+						readToEnd = false;
+						break;
+					}
+				}
 				const start = timer.totalElapsed();
-				const advanceContext = must(this.#advanceContext);
 				advanceContext.currentChangeStartMs = start;
-				let type;
+				advanceContext.currentTable = table;
+				advanceContext.stage = "push";
+				let holds = "fits";
+				let pushCompleted = false;
 				try {
 					try {
 						const tableSource = this.#tables.get(table);
-						if (!tableSource) continue;
+						if (!tableSource) {
+							pushCompleted = true;
+							continue;
+						}
 						const primaryKey = mustGetPrimaryKey(this.#primaryKeys, table);
+						const isUnobservableAdd = probedPrevValues.length === 0 && nextValue !== null && !tableSource.mayAcceptRow(nextValue);
+						const isUnobservableEdit = probedPrevValues.length === 1 && nextValue !== null && !tableSource.mayAcceptRow(nextValue) && !tableSource.mayAcceptRow(probedPrevValues[0]) && deepEqual(getRowKey(primaryKey, probedPrevValues[0]), rowKey);
+						if (isUnobservableAdd || isUnobservableEdit) {
+							pushCompleted = true;
+							continue;
+						}
+						if (nextValue !== null && !this.#reserveSecondRow(advanceContext, rowKey, primaryKey)) holds = "rows";
+						const prevValues = tableSource.reconcilePendingConflicts(probedPrevValues, nextValue, rowKey, this.#tableSpecs.get(table)?.tableSpec.uniqueKeys ?? []);
 						let editOldRow = void 0;
 						for (const prevValue of prevValues) if (nextValue && deepEqual(getRowKey(primaryKey, prevValue), getRowKey(primaryKey, nextValue))) editOldRow = prevValue;
 						else {
@@ -556,42 +970,318 @@ var PipelineDriver = class {
 						}
 						if (nextValue) if (editOldRow) yield* this.#push(tableSource, makeSourceChangeEdit(nextValue, editOldRow));
 						else yield* this.#push(tableSource, makeSourceChangeAdd(nextValue));
+						pushCompleted = true;
 					} finally {
 						advanceContext.pos++;
+						if (pushCompleted) {
+							progress.changesProcessed++;
+							advanceContext.completedChanges = diff.changesScanned;
+						}
 					}
-					this.#shouldAdvanceYieldMaybeAbortAdvance(false);
+					this.#checkAdvance(advanceContext, false);
+					if (holds === "fits") holds = this.#holdPendingRows(advanceContext);
 				} finally {
+					advanceContext.pushTimeMs += timer.totalElapsed() - start;
+					advanceContext.stage = "diff";
 					advanceContext.currentChangeStartMs = void 0;
 				}
 				const elapsed = timer.totalElapsed() - start;
-				this.#advanceTime.recordMs(elapsed, {
-					table,
-					type
-				});
+				this.#advanceTime.recordMs(elapsed, { table });
+				if (holds !== "fits") yield* this.#writeThrough(advanceContext, diff, holds);
 			}
+			advanceContext.completedChanges = diff.changesScanned;
 			const { curr } = diff;
 			for (const table of this.#tables.values()) table.setDB(curr.db.db);
+			this.#costBasedAdvanceIncomplete = false;
 			this.#ensureCostModelExistsIfEnabled(curr.db.db);
 			this.#lc.debug?.(`Advanced to ${curr.version}`);
+			this.#logSlowAdvances(advanceContext);
+			if (costBased) {
+				this.#recordPushCosts(advanceContext, costBased, diff.changesScannedByTable);
+				if (readToEnd && diff.changes > 0 && diff.stats) {
+					const { changeLogReadMs, parseMs, rowReadMs } = diff.stats;
+					this.#readCosts.record({
+						walk: changeLogReadMs,
+						parse: parseMs,
+						lookup: rowReadMs
+					}, {
+						walk: costBased.entries,
+						parse: diff.changesScanned,
+						lookup: diff.changesLookedUp
+					});
+				}
+				this.#snapshotter.releasePrevious();
+			}
+		} catch (e) {
+			if (e instanceof ResetPipelinesSignal && e.reason === "advancement-timeout") {
+				timedOut = true;
+				this.#logAdvanceTimeout(advanceContext, e);
+			}
+			throw e;
 		} finally {
+			if (advanceContext.reservedRows !== void 0) for (const table of this.#tables.values()) table.discardPendingChanges();
+			this.#releaseDeferredWrites(advanceContext);
 			this.#advanceContext = null;
+			this.#recordAdvanceStats(advanceContext, timedOut);
+		}
+	}
+	/**
+	* Checks the progress of the advancement. Returns whether it should yield.
+	*
+	* An advancement is bounded in one of two ways, never both. A
+	* {@link CostBasedAdvance} cuts queries off one by one between changes
+	* (see {@link #cutOffQueries}) and always runs to completion.
+	* Without one, {@link #shouldAdvanceYieldMaybeAbortAdvance} aborts the
+	* advancement with a {@link ResetPipelinesSignal} if it takes too long.
+	*/
+	#checkAdvance(advanceContext, checkYield = true) {
+		if (advanceContext.costBased === void 0) return this.#shouldAdvanceYieldMaybeAbortAdvance(checkYield);
+		return checkYield && advanceContext.timer.elapsedLap() > this.#yieldThresholdMs();
+	}
+	/**
+	* Removes the pipelines of the queries that the cutoff says to.
+	*
+	* @param nextTable The table of the change that `diff` has just yielded,
+	*        which has not been pushed yet.
+	*/
+	#cutOffQueries({ queryStats }, costBased, diff, nextTable) {
+		const cutOff = costBased.cutoff.takeCutOff((queryID) => {
+			const pipeline = this.#pipelines.get(queryID);
+			const stats = queryStats.get(queryID);
+			return pipeline === void 0 || stats === void 0 ? void 0 : {
+				queryID,
+				pushTimeMs: stats.timeMs,
+				changes: stats.changes,
+				hydrationTimeMs: pipeline.hydrationTimeMs
+			};
+		});
+		if (cutOff.length === 0) return;
+		const entriesByTable = new Map(diff.changesScannedByTable);
+		entriesByTable.set(nextTable, (entriesByTable.get(nextTable) ?? 1) - 1);
+		for (const progress of cutOff) {
+			const { queryID } = progress;
+			const pipeline = must(this.#pipelines.get(queryID));
+			this.#pushCosts.record(queryID, pipeline.tables, queryStats.get(queryID)?.timeMsByTable, entriesByTable);
+			this.#queryStats?.recordAdvance(pipeline, {
+				timeMs: progress.pushTimeMs,
+				changes: progress.changes,
+				timedOut: false
+			});
+			this.#abandon(costBased.abandoned, "cutoff", pipeline, {
+				...progress,
+				estimatedPushMs: costBased.estimatedPushMs.get(queryID)
+			});
+		}
+	}
+	/**
+	* Learns from a completed advancement what its changes cost the queries
+	* that were pushed all of them.
+	*/
+	#recordPushCosts({ queryStats }, { estimatedPushMs, estimates }, entriesByTable) {
+		for (const [queryID, pipeline] of this.#pipelines) {
+			const stats = queryStats.get(queryID);
+			this.#pushCosts.record(queryID, pipeline.tables, stats?.timeMsByTable, entriesByTable);
+			const estimate = estimatedPushMs.get(queryID);
+			if (estimate !== void 0) estimates.push({
+				queryID,
+				estimatedPushMs: estimate,
+				pushTimeMs: stats?.timeMs ?? 0
+			});
+		}
+	}
+	/** Accounts the time each query took in an advancement to its shape. */
+	#recordAdvanceStats({ queryStats }, timedOut) {
+		if (this.#queryStats === void 0) return;
+		for (const [queryID, { timeMs, changes }] of queryStats) {
+			const pipeline = this.#pipelines.get(queryID);
+			if (pipeline !== void 0) this.#queryStats.recordAdvance(pipeline, {
+				timeMs,
+				changes,
+				timedOut
+			});
+		}
+	}
+	/**
+	* Decides how the sources apply an advancement's changes: in memory if
+	* they fit in the worker's {@link DeferredWritesBudget}, and otherwise
+	* written through to the `prev` snapshot. Returns the number of rows
+	* reserved if they are held in memory.
+	*/
+	#reserveDeferredWrites(numChanges) {
+		const budget = this.#deferredWrites;
+		if (!budget) return;
+		if (budget.tryReserve(numChanges)) return numChanges;
+		this.#deferredWritesFallbacks.add(1, { stage: "start" });
+		this.#lc.debug?.(`writing through ${numChanges} changes: ${budget.reservedRows} rows of the deferred writes budget are reserved`);
+	}
+	/**
+	* If the change log identifies the rows of a table by a key other than its
+	* primary key here, an entry that sets a row can change its primary key,
+	* which is two rows to a source: the old and the new. (An entry that removes
+	* a row removes the one row its key identifies.) Reserves the second row for
+	* such an entry, if the advancement holds its changes in memory, and
+	* returns whether it fit.
+	*/
+	#reserveSecondRow(advanceContext, rowKey, primaryKey) {
+		if (advanceContext.reservedRows === void 0 || isKeyedBy(rowKey, primaryKey)) return true;
+		if (!must(this.#deferredWrites).tryReserveMore(1)) return false;
+		advanceContext.reservedRows++;
+		return true;
+	}
+	/**
+	* Writes the changes that an advancement holds in memory through to the
+	* `prev` snapshot, and the rest of its changes after them, when they no
+	* longer fit (see {@link HeldRows}). The advancement then continues as if
+	* it had written through from the start.
+	*/
+	*#writeThrough(advanceContext, diff, reason) {
+		const budget = must(this.#deferredWrites);
+		this.#deferredWritesFallbacks.add(1, {
+			stage: "partway",
+			reason
+		});
+		this.#lc.debug?.(`writing through (${reason}) at ${advanceContext.pos} of ${advanceContext.numChanges} changes, holding ${advanceContext.heldBytes} of the ${budget.heldBytes} estimated bytes held on this worker (budget: ${budget.maxBytes})`);
+		diff.setPrevWrites("divergent");
+		for (const source of this.#tables.values()) yield* source.writePendingChanges();
+		this.#releaseDeferredWrites(advanceContext);
+	}
+	#releaseDeferredWrites(advanceContext) {
+		const { reservedRows, heldBytes } = advanceContext;
+		if (reservedRows !== void 0) {
+			must(this.#deferredWrites).release(reservedRows, heldBytes);
+			advanceContext.reservedRows = void 0;
+			advanceContext.heldBytes = 0;
+		}
+	}
+	#recordAdvancePush(queryID, timeMs) {
+		const advance = this.#advanceContext;
+		if (advance === null) return;
+		const stats = getOrInsertComputed(advance.queryStats, queryID, () => ({
+			timeMs: 0,
+			changes: 0,
+			lastPos: -1,
+			timeMsByTable: /* @__PURE__ */ new Map()
+		}));
+		stats.timeMs += timeMs;
+		advance.costBased?.cutoff.recordPush(queryID);
+		if (stats.lastPos !== advance.pos) {
+			stats.lastPos = advance.pos;
+			stats.changes++;
+		}
+		const table = advance.currentTable;
+		if (table !== void 0) stats.timeMsByTable.set(table, (stats.timeMsByTable.get(table) ?? 0) + timeMs);
+	}
+	/**
+	* The identity of the query of the pipeline for `queryID` and its
+	* {@link queryShape}, for logging.
+	*/
+	#queryForLog(queryID) {
+		const pipeline = this.#pipelines.get(queryID);
+		if (pipeline === void 0) return;
+		const { transformationHash, queryName, shape } = pipeline;
+		return {
+			queryName,
+			shape,
+			fields: {
+				queryHash: queryID,
+				transformationHash,
+				...queryName !== void 0 && { queryName },
+				queryShape: shape.hash
+			}
+		};
+	}
+	/** Compute once for an admitted log, without clocks or allocations per change. */
+	#advancementLogFields({ timer, diff, progress, completedChanges, queryStats, totalHydrationTimeMs }) {
+		const advancementTimeMs = timer.totalElapsed();
+		let queryPushTimeMs = 0;
+		for (const stats of queryStats.values()) queryPushTimeMs += stats.timeMs;
+		return {
+			advancementTimeMs,
+			queryPushTimeMs,
+			outsideQueryPushTimeMs: Math.max(0, advancementTimeMs - queryPushTimeMs),
+			totalHydrationTimeMs,
+			changesScanned: diff.changesScanned,
+			changesCompleted: completedChanges,
+			changesProcessed: progress.changesProcessed,
+			changesSkipped: diff.changesSkipped
+		};
+	}
+	/**
+	* Logs each query whose pushes took longer than the slow advance
+	* threshold, at most once per query shape per
+	* {@link SLOW_ADVANCE_LOG_WINDOW_MS}.
+	*/
+	#logSlowAdvances(context) {
+		if (!this.#lc.warn) return;
+		const { queryStats, numChanges } = context;
+		let advancementFields;
+		for (const [queryID, stats] of queryStats) {
+			if (stats.timeMs <= this.#logConfig.slowAdvanceThreshold) continue;
+			const query = this.#queryForLog(queryID);
+			if (query === void 0) continue;
+			const suppressed = slowAdvanceLogThrottle.admit(`${query.queryName ?? ""}:${query.shape.hash}`);
+			if (suppressed === void 0) continue;
+			this.#lc.warn(`Slow query advancement${query.queryName === void 0 ? "" : ` for ${query.queryName}`}: ${Math.round(stats.timeMs)} ms for ${stats.changes} of ${numChanges} changes`, {
+				zeroEvent: "query-slow-advance",
+				...query.fields,
+				advanceTimeMs: stats.timeMs,
+				changes: stats.changes,
+				advancementChanges: numChanges,
+				...advancementFields ??= this.#advancementLogFields(context),
+				timeMsByTable: Object.fromEntries(stats.timeMsByTable),
+				...suppressed > 0 && { suppressedSinceLastLog: suppressed },
+				zql: query.shape.zql
+			});
 		}
 	}
+	/**
+	* Logs the queries that took the most time in an advancement that timed
+	* out, since the reset that follows would otherwise not say what was slow.
+	* Throttled per query shape of the slowest query, with a shared fallback
+	* when the timeout happens before any query push timings are recorded.
+	*/
+	#logAdvanceTimeout(context, reset) {
+		if (!this.#lc.warn) return;
+		const { queryStats, numChanges } = context;
+		const slowest = [...queryStats].toSorted(([, a], [, b]) => b.timeMs - a.timeMs).slice(0, ADVANCE_TIMEOUT_LOG_MAX_QUERIES).flatMap(([queryID, stats]) => {
+			const query = this.#queryForLog(queryID);
+			return query ? [{
+				query,
+				stats
+			}] : [];
+		});
+		const leadingQuery = slowest[0]?.query;
+		const suppressed = slowAdvanceLogThrottle.admit(leadingQuery ? `timeout:${leadingQuery.queryName ?? ""}:${leadingQuery.shape.hash}` : "timeout:outside-query-push");
+		if (suppressed === void 0) return;
+		const summary = slowest.map(({ query, stats }) => `${query.queryName ?? query.fields.queryHash} (${Math.round(stats.timeMs)} ms)`).join(", ");
+		this.#lc.warn(`Advancement of ${numChanges} changes timed out. ` + (summary ? `The queries that took the most time: ${summary}` : "No query push timings were recorded."), {
+			zeroEvent: "query-advance-timeout",
+			reason: reset.message,
+			advancementChanges: numChanges,
+			...this.#advancementLogFields(context),
+			queries: slowest.map(({ query, stats }) => ({
+				...query.fields,
+				advanceTimeMs: stats.timeMs,
+				changes: stats.changes,
+				zql: query.shape.zql
+			})),
+			...suppressed > 0 && { suppressedSinceLastLog: suppressed }
+		});
+	}
 	/** Implements `BuilderDelegate.getSource()` */
 	#getSource(tableName) {
-		let source = this.#tables.get(tableName);
-		if (source) return source;
-		const tableSpec = mustGetTableSpec(this.#tableSpecs, tableName);
-		const primaryKey = mustGetPrimaryKey(this.#primaryKeys, tableName);
-		const { db } = this.#snapshotter.current();
-		source = new TableSource(this.#lc, this.#logConfig, db.db, tableName, tableSpec.zqlSpec, primaryKey, () => this.#shouldYield());
-		this.#tables.set(tableName, source);
-		this.#lc.debug?.(`created TableSource for ${tableName}`);
-		return source;
+		return getOrInsertComputed(this.#tables, tableName, (tableName) => {
+			const tableSpec = mustGetTableSpec(this.#tableSpecs, tableName);
+			const primaryKey = mustGetPrimaryKey(this.#primaryKeys, tableName);
+			const { db } = this.#snapshotter.current();
+			const source = new TableSource(this.#lc, this.#logConfig, db.db, tableName, tableSpec.zqlSpec, primaryKey, () => this.#shouldYield(), { skipUnobservableChanges: true });
+			this.#lc.debug?.(`created TableSource for ${tableName}`);
+			return source;
+		});
 	}
 	#shouldYield() {
 		if (this.#hydrateContext) return this.#hydrateContext.timer.elapsedLap() > this.#yieldThresholdMs();
-		if (this.#advanceContext) return this.#shouldAdvanceYieldMaybeAbortAdvance();
+		if (this.#advanceContext) return this.#checkAdvance(this.#advanceContext);
 		throw new Error("shouldYield called outside of hydration or advancement");
 	}
 	/**
@@ -601,120 +1291,132 @@ var PipelineDriver = class {
 	* checks; a single pathological push always resets.
 	*/
 	#shouldAdvanceYieldMaybeAbortAdvance(checkYield = true) {
-		const { currentChangeStartMs, pos, numChanges, timer: advanceTimer, totalHydrationTimeMs } = must(this.#advanceContext);
+		const { currentChangeStartMs, completedChanges, numChanges, timer: advanceTimer, totalHydrationTimeMs } = must(this.#advanceContext);
 		const elapsed = advanceTimer.totalElapsed();
 		const currentChangeElapsedMs = currentChangeStartMs === void 0 ? void 0 : elapsed - currentChangeStartMs;
-		if (currentChangeElapsedMs !== void 0 && shouldResetSlowCurrentChange(currentChangeElapsedMs, totalHydrationTimeMs)) this.#throwSlowCurrentChangeReset(pos, numChanges, elapsed, currentChangeElapsedMs, totalHydrationTimeMs);
-		const projectedTotalTimeMs = projectedAdvancementTimeMs(elapsed, pos, numChanges);
-		const shouldFinish = shouldFinishLateAdvancement(pos, numChanges);
-		if (!shouldFinish && shouldResetProjectedAdvancement(elapsed, projectedTotalTimeMs, pos, numChanges, totalHydrationTimeMs)) this.#throwProjectedAdvancementReset(pos, numChanges, elapsed, projectedTotalTimeMs, totalHydrationTimeMs);
-		if (!shouldFinish && elapsed > MIN_ADVANCEMENT_TIME_LIMIT_MS && (elapsed > totalHydrationTimeMs || elapsed > totalHydrationTimeMs / 2 && pos <= numChanges / 2)) throw new ResetPipelinesSignal(`Advancement exceeded timeout at ${pos} of ${numChanges} changes after ${elapsed} ms. Advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`, "advancement-timeout");
+		if (currentChangeElapsedMs !== void 0 && shouldResetSlowCurrentChange(currentChangeElapsedMs, totalHydrationTimeMs)) this.#throwSlowCurrentChangeReset(completedChanges, numChanges, elapsed, currentChangeElapsedMs, totalHydrationTimeMs);
+		const projectedTotalTimeMs = projectedAdvancementTimeMs(elapsed, completedChanges, numChanges);
+		if (!shouldFinishLateAdvancement(completedChanges, numChanges)) {
+			if (canProjectAdvancement(elapsed, projectedTotalTimeMs, completedChanges, numChanges)) {
+				const projectedTotal = must(projectedTotalTimeMs);
+				if (shouldResetProjectedAdvancement(elapsed, projectedTotal, totalHydrationTimeMs)) this.#throwProjectedAdvancementReset(completedChanges, numChanges, elapsed, projectedTotal, totalHydrationTimeMs);
+			} else if (elapsed > MIN_ADVANCEMENT_TIME_LIMIT_MS && elapsed > totalHydrationTimeMs) {
+				const breakdown = formatAdvancementBreakdown(must(this.#advanceContext), elapsed);
+				this.#lc.warn?.(`Advancement exceeded timeout at ${completedChanges} of ${numChanges} changes after ${elapsed.toFixed(1)} ms (${breakdown}). Advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`);
+				throw new ResetPipelinesSignal(`Advancement exceeded timeout at ${completedChanges} of ${numChanges} changes after ${elapsed} ms. Advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`, "advancement-timeout", breakdown);
+			}
+		}
 		return checkYield && advanceTimer.elapsedLap() > this.#yieldThresholdMs();
 	}
+	/**
+	* The rows an advancement holds in memory are bounded by its reservation,
+	* but their width is not known in advance. So the bytes they are estimated
+	* to hold are added to those held by the other advancements on this worker.
+	* Says whether that takes them past the budget, or if the rows are not
+	* bounded by the reservation after all.
+	*/
+	#holdPendingRows(advanceContext) {
+		const { reservedRows, heldBytes, pos, numChanges } = advanceContext;
+		if (reservedRows === void 0) return "fits";
+		const budget = must(this.#deferredWrites);
+		const rows = this.pendingRows;
+		const bytes = this.pendingBytes;
+		advanceContext.heldBytes = bytes;
+		const bytesFit = budget.holdBytes(bytes - heldBytes);
+		if (rows > reservedRows) {
+			budget.recordRowOverrun();
+			this.#lc.error?.(`Advancement holds ${rows} rows at ${pos} of ${numChanges} changes, more than the ${reservedRows} it reserved. Writing through the rest.`);
+			return "row-overrun";
+		}
+		return bytesFit ? "fits" : "bytes";
+	}
 	#throwSlowCurrentChangeReset(pos, numChanges, elapsed, currentChangeElapsedMs, totalHydrationTimeMs) {
-		throw new ResetPipelinesSignal(`Advancement exceeded timeout processing current change at ${pos} of ${numChanges} changes after ${currentChangeElapsedMs} ms (${elapsed} ms total). Advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`, "advancement-timeout");
+		const breakdown = this.#advanceContext ? formatAdvancementBreakdown(this.#advanceContext, elapsed) : void 0;
+		this.#lc.warn?.(`Advancement slow current change at ${pos} of ${numChanges} changes after ${currentChangeElapsedMs.toFixed(1)} ms (${elapsed.toFixed(1)} ms total)` + (breakdown ? ` (${breakdown})` : "") + `. Advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`);
+		throw new ResetPipelinesSignal(`Advancement exceeded timeout processing current change at ${pos} of ${numChanges} changes after ${currentChangeElapsedMs} ms (${elapsed} ms total). Advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`, "advancement-timeout", breakdown);
 	}
 	#throwProjectedAdvancementReset(pos, numChanges, elapsed, projectedTotalTimeMs, totalHydrationTimeMs) {
+		const breakdown = this.#advanceContext ? formatAdvancementBreakdown(this.#advanceContext, elapsed) : void 0;
 		const projection = projectedTotalTimeMs === void 0 ? "" : ` Projected total advancement time is ${projectedTotalTimeMs} ms.`;
-		throw new ResetPipelinesSignal(`Advancement projected to exceed hydration time at ${pos} of ${numChanges} changes after ${elapsed} ms.` + projection + ` Advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`, "advancement-timeout");
+		this.#lc.warn?.(`Advancement projected timeout at ${pos} of ${numChanges} changes after ${elapsed.toFixed(1)} ms` + projection + (breakdown ? ` (${breakdown})` : "") + `. Advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`);
+		throw new ResetPipelinesSignal(`Advancement projected to exceed hydration time at ${pos} of ${numChanges} changes after ${elapsed} ms.` + projection + ` Advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`, "advancement-timeout", breakdown);
 	}
 	/** Implements `BuilderDelegate.createStorage()` */
 	#createStorage() {
 		return this.#storage.createStorage();
 	}
 	*#push(source, change) {
-		this.#startAccumulating();
+		assert(this.#pushedRowChanges === null, "push already in progress");
 		try {
+			this.#pushedRowChanges = [];
 			for (const val of source.genPush(change)) {
 				if (val === "yield") yield "yield";
-				for (const changeOrYield of this.#stopAccumulating().stream()) yield changeOrYield;
-				this.#startAccumulating();
+				const rowChanges = this.#pushedRowChanges;
+				this.#pushedRowChanges = [];
+				yield* rowChanges;
 			}
 		} finally {
-			if (this.#streamer !== null) this.#stopAccumulating();
+			this.#pushedRowChanges = null;
 		}
 	}
-	#startAccumulating() {
-		assert(this.#streamer === null, "Streamer already started");
-		this.#streamer = new Streamer(must(this.#primaryKeys), this.#tableSpecs, (queryID, error) => this.#logQueryFailure(queryID, "query pipeline failed", error));
-	}
-	#stopAccumulating() {
-		const streamer = this.#streamer;
-		assert(streamer, "Streamer not started");
-		this.#streamer = null;
-		return streamer;
-	}
-	#logQueryFailure(queryID, message, error) {
-		const pipeline = this.#pipelines.get(queryID);
-		const queryInfo = pipeline ? {
-			queryHash: queryID,
-			transformationHash: pipeline.transformationHash,
-			queryName: pipeline.queryName
-		} : void 0;
-		logQueryFailure(this.#lc, queryInfo, message, error);
+	/**
+	* Converts a change pushed out of a query pipeline into row changes during
+	* the push. The relationships of a pushed node are lazy, and operators such
+	* as Join compute them from the state of the push in progress: while a child
+	* change is pushed to each matching parent in turn, parents not yet pushed
+	* must not see it. Read after the push has moved on to the next parent, or
+	* after it has finished, a node's relationships show the later state, and
+	* rows that are also pushed separately are counted twice.
+	*/
+	*#streamPushed(queryID, schema, change) {
+		assert(this.#pushedRowChanges, "changes are only pushed by #push");
+		for (const rowChange of must(this.#streamer).streamChange(queryID, schema, change)) {
+			if (rowChange === "yield") {
+				yield rowChange;
+				continue;
+			}
+			must(this.#pushedRowChanges).push(rowChange);
+		}
 	}
 };
+/** Turns the output of query pipelines into the row changes sent to clients. */
 var Streamer = class {
 	#primaryKeys;
 	#tableSpecs;
-	#logQueryFailure;
-	constructor(primaryKeys, tableSpecs, logQueryFailure) {
+	constructor(primaryKeys, tableSpecs) {
 		this.#primaryKeys = primaryKeys;
 		this.#tableSpecs = tableSpecs;
-		this.#logQueryFailure = logQueryFailure;
 	}
-	#changes = [];
-	accumulate(queryID, schema, changes) {
-		this.#changes.push([
-			queryID,
-			schema,
-			changes
-		]);
-		return this;
-	}
-	*stream() {
-		for (const [queryID, schema, changes] of this.#changes) try {
-			yield* this.#streamChanges(queryID, schema, changes);
-		} catch (e) {
-			this.#logQueryFailure?.(queryID, e);
-			throw e;
-		}
-	}
-	*#streamChanges(queryID, schema, changes) {
+	/** The row changes for a change pushed out of the pipeline of `queryID`. */
+	*streamChange(queryID, schema, change) {
 		if (schema.system === "permissions") return;
-		for (const change of changes) {
-			if (change === "yield") {
-				yield change;
-				continue;
-			}
-			const type = change[0];
-			switch (type) {
-				case 1:
-				case 0:
-					yield* this.#streamNodes(queryID, schema, type, () => [change[1]]);
-					break;
-				case 3: {
-					const child = change[2];
-					const childSchema = must(schema.relationships[child.relationshipName]);
-					yield* this.#streamChanges(queryID, childSchema, [child.change]);
-					break;
-				}
-				case 2:
-					yield* this.#streamNodes(queryID, schema, type, () => [{
-						row: change[1].row,
-						relationships: {}
-					}]);
-					break;
-				default: unreachable(change[0]);
+		const type = change[0];
+		switch (type) {
+			case 1:
+			case 0:
+				yield* this.streamNodes(queryID, schema, type, [change[1]]);
+				break;
+			case 3: {
+				const child = change[2];
+				yield* this.streamChange(queryID, must(schema.relationships[child.relationshipName]), child.change);
+				break;
 			}
+			case 2:
+				yield* this.streamNodes(queryID, schema, type, [{
+					row: change[1].row,
+					relationships: {}
+				}]);
+				break;
+			default: unreachable(change[0]);
 		}
 	}
-	*#streamNodes(queryID, schema, op, nodes) {
+	/** A row change of type `op` for each of `nodes` and their descendants. */
+	*streamNodes(queryID, schema, op, nodes) {
 		const { tableName: table, system } = schema;
+		if (system === "permissions") return;
 		const primaryKey = must(this.#primaryKeys.get(table));
 		const spec = must(this.#tableSpecs.get(table)).tableSpec;
-		if (system === "permissions") return;
-		for (const node of nodes()) {
+		for (const node of nodes) {
 			if (node === "yield") {
 				yield node;
 				continue;
@@ -736,9 +1438,9 @@ var Streamer = class {
 				rowKey,
 				row: op === 1 ? void 0 : row
 			};
-			for (const [relationship, children] of Object.entries(relationships)) {
+			for (const relationship of Object.keys(relationships)) {
 				const childSchema = must(schema.relationships[relationship]);
-				yield* this.#streamNodes(queryID, childSchema, op, children);
+				if (childSchema.system !== "permissions") yield* this.streamNodes(queryID, childSchema, op, relationships[relationship]());
 			}
 		}
 	}
@@ -782,6 +1484,18 @@ var QueryFailureLoggingOperator = class {
 			throw e;
 		}
 	}
+	*reconcile(_pusher) {
+		if (this.#output.reconcile) try {
+			yield* this.#output.reconcile(this);
+		} catch (e) {
+			logQueryFailure(this.#lc, {
+				queryHash: this.#queryHash,
+				transformationHash: this.#transformationHash,
+				queryName: this.#queryName
+			}, "query pipeline failed during reconcile", e);
+			throw e;
+		}
+	}
 };
 function logQueryFailure(lc, queryInfo, message, error) {
 	if (error instanceof ResetPipelinesSignal) return;
@@ -792,43 +1506,73 @@ function logQueryFailure(lc, queryInfo, message, error) {
 	}
 	queryLC.error?.(message, error);
 }
-function* toAdds(nodes) {
-	for (const node of nodes) {
-		if (node === "yield") {
-			yield node;
-			continue;
-		}
-		yield [
-			0,
-			node,
-			null
-		];
-	}
-}
 function getRowKey(cols, row) {
-	return Object.fromEntries(cols.map((col) => [col, must(row[col])]));
+	const rowKey = {};
+	for (const col of cols) assignProperty(rowKey, col, must(row[col]));
+	return rowKey;
+}
+/** Whether `rowKey` has exactly the columns of `primaryKey`. */
+function isKeyedBy(rowKey, primaryKey) {
+	let cols = 0;
+	for (const col in rowKey) {
+		if (!primaryKey.includes(col)) return false;
+		cols++;
+	}
+	return cols === primaryKey.length;
 }
 /**
 * Core hydration logic used by {@link PipelineDriver#addQuery}, extracted to a
 * function for reuse by the analyze-query RPC path so that analysis hydrates
-* queries the same way the view-syncer does in production.
+* queries the same way the view-syncer does in production. As there, the rows
+* of a table outside `clientSchema` are keyed by its key in the replica.
+*/
+function hydrate(input, hash, clientSchema, tableSpecs) {
+	return new Streamer(buildRowKeys(clientSchema, tableSpecs), tableSpecs).streamNodes(hash, input.getSchema(), 0, input.fetch({}));
+}
+/**
+* Returns the value of `childField` in the row of `input`, the pipeline of a
+* scalar subquery: `null` if the field is `NULL`, or `undefined` if there is
+* no row. Used by {@link PipelineDriver#addQuery} and the analyze-query RPC
+* path to resolve scalar subqueries the same way.
 */
-function* hydrate(input, hash, clientSchema, tableSpecs) {
-	const res = input.fetch({});
-	yield* new Streamer(buildPrimaryKeys(clientSchema), tableSpecs).accumulate(hash, input.getSchema(), toAdds(res)).stream();
+function fetchScalarValue(input, childField) {
+	let node;
+	for (const n of skipYields(input.fetch({}))) node ??= n;
+	if (!node) return;
+	return node.row[childField] ?? null;
 }
-function* hydrateInternal(input, hash, primaryKeys, tableSpecs) {
-	const res = input.fetch({});
-	yield* new Streamer(primaryKeys, tableSpecs).accumulate(hash, input.getSchema(), toAdds(res)).stream();
+/**
+* The schema the rows of a companion pipeline are streamed with. A companion is
+* built from the bare subquery, so its schema says `'client'` even when its
+* gate came from a permission rule. Rows read for a rule are never synced, so
+* such a companion takes the gate's system and the Streamer skips it, the rows
+* behind any EXISTS nested in it included.
+*/
+function companionStreamSchema(input, { system }) {
+	const schema = input.getSchema();
+	return system === "permissions" ? {
+		...schema,
+		system
+	} : schema;
 }
 function buildPrimaryKeys(clientSchema, primaryKeys = /* @__PURE__ */ new Map()) {
 	for (const [tableName, { primaryKey }] of Object.entries(clientSchema.tables)) primaryKeys.set(tableName, primaryKey);
 	return primaryKeys;
 }
+/**
+* Sets `primaryKeys` to the columns that key the synced rows of each table:
+* the table's primary key in `clientSchema`, or, for a table outside it, its
+* key in the replica.
+*/
+function buildRowKeys(clientSchema, tableSpecs, primaryKeys = /* @__PURE__ */ new Map()) {
+	primaryKeys.clear();
+	for (const [table, spec] of tableSpecs.entries()) primaryKeys.set(table, spec.tableSpec.primaryKey);
+	return buildPrimaryKeys(clientSchema, primaryKeys);
+}
 function mustGetPrimaryKey(primaryKeys, table) {
 	const pKeys = must(primaryKeys, "primaryKey map must be non-null");
 	const rv = pKeys.get(table);
-	assert(rv, () => `table '${table}' is not one of: ${[...pKeys.keys()].sort()}. Check the spelling and ensure that the table has a primary key.`);
+	assert(rv, () => `table '${table}' is not one of: ${JSON.stringify([...pKeys.keys()].sort())}. Check the spelling and ensure that the table has a primary key.`);
 	return rv;
 }
 /**
@@ -841,6 +1585,6 @@ function scalarValuesEqual(a, b) {
 	return a === b;
 }
 //#endregion
-export { PipelineDriver, hydrate };
+export { PipelineDriver, fetchScalarValue, hydrate };
 
 //# sourceMappingURL=pipeline-driver.js.map