@rocicorp/zero 1.11.0-canary.21 → 1.11.0-canary.23

package/out/zero-cache/src/services/view-syncer/pipeline-driver.js

@@ -2,6 +2,7 @@ import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { deepEqual } from "../../../../shared/src/json.js";
 import { getOrInsertComputed } from "../../../../shared/src/map.js";
 import { must } from "../../../../shared/src/must.js";
+import { assignProperty } from "../../../../shared/src/objects.js";
 import { skipYields } from "../../../../zql/src/ivm/skip-yields.js";
 import { throwOutput } from "../../../../zql/src/ivm/operator.js";
 import { MeasurePushOperator } from "../../../../zql/src/query/measure-push-operator.js";
@@ -92,6 +93,8 @@ var PipelineDriver = class {
 	#planWarningThresholds;
 	#yieldThresholdMs;
 	#streamer = null;
+	/** The row changes of the push in progress, until #push yields them. */
+	#pushedRowChanges = null;
 	#hydrateContext = null;
 	#advanceContext = null;
 	#replicaVersion = null;
@@ -172,6 +175,7 @@ var PipelineDriver = class {
 		primaryKeys.clear();
 		for (const [table, spec] of this.#tableSpecs.entries()) primaryKeys.set(table, spec.tableSpec.primaryKey);
 		buildPrimaryKeys(clientSchema, primaryKeys);
+		this.#streamer ??= new Streamer(primaryKeys, this.#tableSpecs);
 		const { replicaVersion } = getSubscriptionState(db);
 		this.#replicaVersion = replicaVersion;
 	}
@@ -446,7 +450,7 @@ var PipelineDriver = class {
 			}, planWarnings);
 			const schema = input.getSchema();
 			input.setOutput({ push: (change) => this.#streamPushed(queryID, schema, change) });
-			for (const change of hydrateInternal(input, queryID, must(this.#primaryKeys), this.#tableSpecs)) {
+			for (const change of must(this.#streamer).streamNodes(queryID, schema, 0, input.fetch({}))) {
 				if (change !== "yield") hydrationRowCount++;
 				yield change;
 			}
@@ -675,17 +679,26 @@ var PipelineDriver = class {
 		const diff = this.#snapshotter.advance(this.#tableSpecs, this.#allTableNames, this.#tables, "divergent");
 		const { prev, curr, changes } = diff;
 		this.#lc.debug?.(`advance ${prev.version} => ${curr.version}: ${changes} changes`);
+		const progress = { changesProcessed: 0 };
 		return {
 			version: curr.version,
 			numChanges: changes,
-			changes: this.#trackRowSetSignatures(this.#advance(diff, timer, changes))
+			changes: this.#trackRowSetSignatures(this.#advance(diff, timer, changes, progress)),
+			progress: () => ({
+				changesScanned: diff.changesScanned,
+				changesProcessed: progress.changesProcessed,
+				changesSkipped: diff.changesSkipped
+			})
 		};
 	}
-	*#advance(diff, timer, numChanges) {
+	*#advance(diff, timer, numChanges, progress) {
 		assert(this.#hydrateContext === null, "Cannot advance while hydration is in progress");
 		const totalHydrationTimeMs = this.totalHydrationTimeMs();
 		const advanceContext = {
 			timer,
+			progress,
+			diff,
+			completedChanges: 0,
 			totalHydrationTimeMs,
 			numChanges,
 			reservedRows: void 0,
@@ -704,15 +717,20 @@ var PipelineDriver = class {
 			for (const table of this.#tables.values()) table.setDeferWrites(deferWrites);
 			this.#lc.debug?.(`starting pipeline advancement of ${numChanges} changes with an advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms (${deferWrites ? "deferred" : "write-through"}).`);
 			for (const { table, prevValues: probedPrevValues, nextValue, rowKey } of diff) {
+				advanceContext.completedChanges = diff.changesScanned - 1;
 				if (this.#shouldAdvanceYieldMaybeAbortAdvance()) yield "yield";
 				const start = timer.totalElapsed();
 				advanceContext.currentChangeStartMs = start;
 				advanceContext.currentTable = table;
 				let holds = "fits";
+				let pushCompleted = false;
 				try {
 					try {
 						const tableSource = this.#tables.get(table);
-						if (!tableSource) continue;
+						if (!tableSource) {
+							pushCompleted = true;
+							continue;
+						}
 						const primaryKey = mustGetPrimaryKey(this.#primaryKeys, table);
 						if (nextValue !== null && !this.#reserveSecondRow(advanceContext, rowKey, primaryKey)) holds = "rows";
 						const prevValues = tableSource.reconcilePendingConflicts(probedPrevValues, nextValue, rowKey, this.#tableSpecs.get(table)?.tableSpec.uniqueKeys ?? []);
@@ -724,8 +742,13 @@ var PipelineDriver = class {
 						}
 						if (nextValue) if (editOldRow) yield* this.#push(tableSource, makeSourceChangeEdit(nextValue, editOldRow));
 						else yield* this.#push(tableSource, makeSourceChangeAdd(nextValue));
+						pushCompleted = true;
 					} finally {
 						advanceContext.pos++;
+						if (pushCompleted) {
+							progress.changesProcessed++;
+							advanceContext.completedChanges = diff.changesScanned;
+						}
 					}
 					this.#shouldAdvanceYieldMaybeAbortAdvance(false);
 					if (holds === "fits") holds = this.#holdPendingRows(advanceContext);
@@ -736,6 +759,7 @@ var PipelineDriver = class {
 				this.#advanceTime.recordMs(elapsed, { table });
 				if (holds !== "fits") yield* this.#writeThrough(advanceContext, diff, holds);
 			}
+			advanceContext.completedChanges = diff.changesScanned;
 			const { curr } = diff;
 			for (const table of this.#tables.values()) table.setDB(curr.db.db);
 			this.#ensureCostModelExistsIfEnabled(curr.db.db);
@@ -854,13 +878,31 @@ var PipelineDriver = class {
 			}
 		};
 	}
+	/** Compute once for an admitted log, without clocks or allocations per change. */
+	#advancementLogFields({ timer, diff, progress, completedChanges, queryStats, totalHydrationTimeMs }) {
+		const advancementTimeMs = timer.totalElapsed();
+		let queryPushTimeMs = 0;
+		for (const stats of queryStats.values()) queryPushTimeMs += stats.timeMs;
+		return {
+			advancementTimeMs,
+			queryPushTimeMs,
+			outsideQueryPushTimeMs: Math.max(0, advancementTimeMs - queryPushTimeMs),
+			totalHydrationTimeMs,
+			changesScanned: diff.changesScanned,
+			changesCompleted: completedChanges,
+			changesProcessed: progress.changesProcessed,
+			changesSkipped: diff.changesSkipped
+		};
+	}
 	/**
 	* Logs each query whose pushes took longer than the slow advance
 	* threshold, at most once per query shape per
 	* {@link SLOW_ADVANCE_LOG_WINDOW_MS}.
 	*/
-	#logSlowAdvances({ queryStats, numChanges }) {
+	#logSlowAdvances(context) {
 		if (!this.#lc.warn) return;
+		const { queryStats, numChanges } = context;
+		let advancementFields;
 		for (const [queryID, stats] of queryStats) {
 			if (stats.timeMs <= this.#logConfig.slowAdvanceThreshold) continue;
 			const query = this.#queryForLog(queryID);
@@ -873,6 +915,7 @@ var PipelineDriver = class {
 				advanceTimeMs: stats.timeMs,
 				changes: stats.changes,
 				advancementChanges: numChanges,
+				...advancementFields ??= this.#advancementLogFields(context),
 				timeMsByTable: Object.fromEntries(stats.timeMsByTable),
 				...suppressed > 0 && { suppressedSinceLastLog: suppressed },
 				zql: query.shape.zql
@@ -882,10 +925,12 @@ var PipelineDriver = class {
 	/**
 	* Logs the queries that took the most time in an advancement that timed
 	* out, since the reset that follows would otherwise not say what was slow.
-	* Throttled per query shape of the slowest query.
+	* Throttled per query shape of the slowest query, with a shared fallback
+	* when the timeout happens before any query push timings are recorded.
 	*/
-	#logAdvanceTimeout({ queryStats, numChanges }, reset) {
+	#logAdvanceTimeout(context, reset) {
 		if (!this.#lc.warn) return;
+		const { queryStats, numChanges } = context;
 		const slowest = [...queryStats].toSorted(([, a], [, b]) => b.timeMs - a.timeMs).slice(0, ADVANCE_TIMEOUT_LOG_MAX_QUERIES).flatMap(([queryID, stats]) => {
 			const query = this.#queryForLog(queryID);
 			return query ? [{
@@ -893,14 +938,15 @@ var PipelineDriver = class {
 				stats
 			}] : [];
 		});
-		if (slowest.length === 0) return;
-		const suppressed = slowAdvanceLogThrottle.admit(`timeout:${slowest[0].query.queryName ?? ""}:${slowest[0].query.shape.hash}`);
+		const leadingQuery = slowest[0]?.query;
+		const suppressed = slowAdvanceLogThrottle.admit(leadingQuery ? `timeout:${leadingQuery.queryName ?? ""}:${leadingQuery.shape.hash}` : "timeout:outside-query-push");
 		if (suppressed === void 0) return;
 		const summary = slowest.map(({ query, stats }) => `${query.queryName ?? query.fields.queryHash} (${Math.round(stats.timeMs)} ms)`).join(", ");
-		this.#lc.warn(`Advancement of ${numChanges} changes timed out. The queries that took the most time: ${summary}`, {
+		this.#lc.warn(`Advancement of ${numChanges} changes timed out. ` + (summary ? `The queries that took the most time: ${summary}` : "No query push timings were recorded."), {
 			zeroEvent: "query-advance-timeout",
 			reason: reset.message,
 			advancementChanges: numChanges,
+			...this.#advancementLogFields(context),
 			queries: slowest.map(({ query, stats }) => ({
 				...query.fields,
 				advanceTimeMs: stats.timeMs,
@@ -933,14 +979,14 @@ var PipelineDriver = class {
 	* checks; a single pathological push always resets.
 	*/
 	#shouldAdvanceYieldMaybeAbortAdvance(checkYield = true) {
-		const { currentChangeStartMs, pos, numChanges, timer: advanceTimer, totalHydrationTimeMs } = must(this.#advanceContext);
+		const { currentChangeStartMs, completedChanges, numChanges, timer: advanceTimer, totalHydrationTimeMs } = must(this.#advanceContext);
 		const elapsed = advanceTimer.totalElapsed();
 		const currentChangeElapsedMs = currentChangeStartMs === void 0 ? void 0 : elapsed - currentChangeStartMs;
-		if (currentChangeElapsedMs !== void 0 && shouldResetSlowCurrentChange(currentChangeElapsedMs, totalHydrationTimeMs)) this.#throwSlowCurrentChangeReset(pos, numChanges, elapsed, currentChangeElapsedMs, totalHydrationTimeMs);
-		const projectedTotalTimeMs = projectedAdvancementTimeMs(elapsed, pos, numChanges);
-		const shouldFinish = shouldFinishLateAdvancement(pos, numChanges);
-		if (!shouldFinish && shouldResetProjectedAdvancement(elapsed, projectedTotalTimeMs, pos, numChanges, totalHydrationTimeMs)) this.#throwProjectedAdvancementReset(pos, numChanges, elapsed, projectedTotalTimeMs, totalHydrationTimeMs);
-		if (!shouldFinish && elapsed > MIN_ADVANCEMENT_TIME_LIMIT_MS && elapsed > totalHydrationTimeMs / 2 && (projectedTotalTimeMs === void 0 || projectedTotalTimeMs - elapsed > totalHydrationTimeMs)) throw new ResetPipelinesSignal(`Advancement exceeded timeout at ${pos} of ${numChanges} changes after ${elapsed} ms. Advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`, "advancement-timeout");
+		if (currentChangeElapsedMs !== void 0 && shouldResetSlowCurrentChange(currentChangeElapsedMs, totalHydrationTimeMs)) this.#throwSlowCurrentChangeReset(completedChanges, numChanges, elapsed, currentChangeElapsedMs, totalHydrationTimeMs);
+		const projectedTotalTimeMs = projectedAdvancementTimeMs(elapsed, completedChanges, numChanges);
+		const shouldFinish = shouldFinishLateAdvancement(completedChanges, numChanges);
+		if (!shouldFinish && shouldResetProjectedAdvancement(elapsed, projectedTotalTimeMs, completedChanges, numChanges, totalHydrationTimeMs)) this.#throwProjectedAdvancementReset(completedChanges, numChanges, elapsed, projectedTotalTimeMs, totalHydrationTimeMs);
+		if (!shouldFinish && elapsed > MIN_ADVANCEMENT_TIME_LIMIT_MS && elapsed > totalHydrationTimeMs / 2 && (projectedTotalTimeMs === void 0 || projectedTotalTimeMs - elapsed > totalHydrationTimeMs)) throw new ResetPipelinesSignal(`Advancement exceeded timeout at ${completedChanges} of ${numChanges} changes after ${elapsed} ms. Advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`, "advancement-timeout");
 		return checkYield && advanceTimer.elapsedLap() > this.#yieldThresholdMs();
 	}
 	/**
@@ -977,15 +1023,17 @@ var PipelineDriver = class {
 		return this.#storage.createStorage();
 	}
 	*#push(source, change) {
-		this.#startAccumulating();
+		assert(this.#pushedRowChanges === null, "push already in progress");
 		try {
+			this.#pushedRowChanges = [];
 			for (const val of source.genPush(change)) {
 				if (val === "yield") yield "yield";
-				for (const changeOrYield of this.#stopAccumulating().stream()) yield changeOrYield;
-				this.#startAccumulating();
+				const rowChanges = this.#pushedRowChanges;
+				this.#pushedRowChanges = [];
+				yield* rowChanges;
 			}
 		} finally {
-			if (this.#streamer !== null) this.#stopAccumulating();
+			this.#pushedRowChanges = null;
 		}
 	}
 	/**
@@ -998,106 +1046,54 @@ var PipelineDriver = class {
 	* rows that are also pushed separately are counted twice.
 	*/
 	*#streamPushed(queryID, schema, change) {
-		const streamer = this.#streamer;
-		assert(streamer, "must #startAccumulating() before pushing changes");
-		for (const rowChange of streamer.streamChange(queryID, schema, change)) {
+		assert(this.#pushedRowChanges, "changes are only pushed by #push");
+		for (const rowChange of must(this.#streamer).streamChange(queryID, schema, change)) {
 			if (rowChange === "yield") {
 				yield rowChange;
 				continue;
 			}
-			must(this.#streamer).add(rowChange);
+			must(this.#pushedRowChanges).push(rowChange);
 		}
 	}
-	#startAccumulating() {
-		assert(this.#streamer === null, "Streamer already started");
-		this.#streamer = new Streamer(must(this.#primaryKeys), this.#tableSpecs, (queryID, error) => this.#logQueryFailure(queryID, "query pipeline failed", error));
-	}
-	#stopAccumulating() {
-		const streamer = this.#streamer;
-		assert(streamer, "Streamer not started");
-		this.#streamer = null;
-		return streamer;
-	}
-	#logQueryFailure(queryID, message, error) {
-		const pipeline = this.#pipelines.get(queryID);
-		const queryInfo = pipeline ? {
-			queryHash: queryID,
-			transformationHash: pipeline.transformationHash,
-			queryName: pipeline.queryName
-		} : void 0;
-		logQueryFailure(this.#lc, queryInfo, message, error);
-	}
 };
+/** Turns the output of query pipelines into the row changes sent to clients. */
 var Streamer = class {
 	#primaryKeys;
 	#tableSpecs;
-	#logQueryFailure;
-	constructor(primaryKeys, tableSpecs, logQueryFailure) {
+	constructor(primaryKeys, tableSpecs) {
 		this.#primaryKeys = primaryKeys;
 		this.#tableSpecs = tableSpecs;
-		this.#logQueryFailure = logQueryFailure;
-	}
-	#changes = [];
-	/** Row changes that were already produced by {@link streamChange}. */
-	#rowChanges = [];
-	add(rowChange) {
-		this.#rowChanges.push(rowChange);
-	}
-	streamChange(queryID, schema, change) {
-		return this.#streamChanges(queryID, schema, [change]);
-	}
-	accumulate(queryID, schema, changes) {
-		this.#changes.push([
-			queryID,
-			schema,
-			changes
-		]);
-		return this;
 	}
-	*stream() {
-		yield* this.#rowChanges;
-		for (const [queryID, schema, changes] of this.#changes) try {
-			yield* this.#streamChanges(queryID, schema, changes);
-		} catch (e) {
-			this.#logQueryFailure?.(queryID, e);
-			throw e;
-		}
-	}
-	*#streamChanges(queryID, schema, changes) {
+	/** The row changes for a change pushed out of the pipeline of `queryID`. */
+	*streamChange(queryID, schema, change) {
 		if (schema.system === "permissions") return;
-		for (const change of changes) {
-			if (change === "yield") {
-				yield change;
-				continue;
-			}
-			const type = change[0];
-			switch (type) {
-				case 1:
-				case 0:
-					yield* this.#streamNodes(queryID, schema, type, () => [change[1]]);
-					break;
-				case 3: {
-					const child = change[2];
-					const childSchema = must(schema.relationships[child.relationshipName]);
-					yield* this.#streamChanges(queryID, childSchema, [child.change]);
-					break;
-				}
-				case 2:
-					yield* this.#streamNodes(queryID, schema, type, () => [{
-						row: change[1].row,
-						relationships: {}
-					}]);
-					break;
-				default: unreachable(change[0]);
+		const type = change[0];
+		switch (type) {
+			case 1:
+			case 0:
+				yield* this.streamNodes(queryID, schema, type, [change[1]]);
+				break;
+			case 3: {
+				const child = change[2];
+				yield* this.streamChange(queryID, must(schema.relationships[child.relationshipName]), child.change);
+				break;
 			}
+			case 2:
+				yield* this.streamNodes(queryID, schema, type, [{
+					row: change[1].row,
+					relationships: {}
+				}]);
+				break;
+			default: unreachable(change[0]);
 		}
 	}
-	*#streamNodes(queryID, schema, op, nodes) {
+	/** A row change of type `op` for each of `nodes` and their descendants. */
+	*streamNodes(queryID, schema, op, nodes) {
 		const { tableName: table, system } = schema;
+		if (system === "permissions") return;
 		const primaryKey = must(this.#primaryKeys.get(table));
 		const spec = must(this.#tableSpecs.get(table)).tableSpec;
-		if (system === "permissions") return;
-		for (const node of nodes()) {
+		for (const node of nodes) {
 			if (node === "yield") {
 				yield node;
 				continue;
@@ -1119,9 +1115,9 @@ var Streamer = class {
 				rowKey,
 				row: op === 1 ? void 0 : row
 			};
-			for (const [relationship, children] of Object.entries(relationships)) {
+			for (const relationship of Object.keys(relationships)) {
 				const childSchema = must(schema.relationships[relationship]);
-				yield* this.#streamNodes(queryID, childSchema, op, children);
+				if (childSchema.system !== "permissions") yield* this.streamNodes(queryID, childSchema, op, relationships[relationship]());
 			}
 		}
 	}
@@ -1187,21 +1183,10 @@ function logQueryFailure(lc, queryInfo, message, error) {
 	}
 	queryLC.error?.(message, error);
 }
-function* toAdds(nodes) {
-	for (const node of nodes) {
-		if (node === "yield") {
-			yield node;
-			continue;
-		}
-		yield [
-			0,
-			node,
-			null
-		];
-	}
-}
 function getRowKey(cols, row) {
-	return Object.fromEntries(cols.map((col) => [col, must(row[col])]));
+	const rowKey = {};
+	for (const col of cols) assignProperty(rowKey, col, must(row[col]));
+	return rowKey;
 }
 /** Whether `rowKey` has exactly the columns of `primaryKey`. */
 function isKeyedBy(rowKey, primaryKey) {
@@ -1218,12 +1203,7 @@ function isKeyedBy(rowKey, primaryKey) {
 * queries the same way the view-syncer does in production.
 */
 function hydrate(input, hash, clientSchema, tableSpecs) {
-	const res = input.fetch({});
-	return new Streamer(buildPrimaryKeys(clientSchema), tableSpecs).accumulate(hash, input.getSchema(), toAdds(res)).stream();
-}
-function hydrateInternal(input, hash, primaryKeys, tableSpecs) {
-	const res = input.fetch({});
-	return new Streamer(primaryKeys, tableSpecs).accumulate(hash, input.getSchema(), toAdds(res)).stream();
+	return new Streamer(buildPrimaryKeys(clientSchema), tableSpecs).streamNodes(hash, input.getSchema(), 0, input.fetch({}));
 }
 function buildPrimaryKeys(clientSchema, primaryKeys = /* @__PURE__ */ new Map()) {
 	for (const [tableName, { primaryKey }] of Object.entries(clientSchema.tables)) primaryKeys.set(tableName, primaryKey);