@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.41

package/out/zero-cache/src/services/view-syncer/snapshotter.js

@@ -1,7 +1,9 @@
 import { assert } from "../../../../shared/src/asserts.js";
 import { parse } from "../../../../shared/src/valita.js";
+import { getOrInsertComputed } from "../../../../shared/src/map.js";
 import { stringify } from "../../../../shared/src/bigint-json.js";
 import { Database } from "../../../../zqlite/src/db.js";
+import { recordIndexUsage } from "../../../../zqlite/src/index-usage.js";
 import { fromSQLiteTypes } from "../../../../zqlite/src/table-source.js";
 import "../replicator/schema/constants.js";
 import { normalizedKeyOrder } from "../../types/row-key.js";
@@ -74,13 +76,21 @@ var Snapshotter = class {
 	#dbFile;
 	#appID;
 	#pageCacheSizeKib;
+	#rowCache;
 	#curr;
 	#prev;
-	constructor(lc, dbFile, { appID }, pageCacheSizeKib) {
+	/**
+	* @param rowCache An optional worker-wide {@link SnapshotRowCache} through
+	*        which the row reads performed when iterating over a
+	*        {@link SnapshotDiff} are shared with the other Snapshotters
+	*        (i.e. client groups) on the worker.
+	*/
+	constructor(lc, dbFile, { appID }, pageCacheSizeKib, rowCache) {
 		this.#lc = lc;
 		this.#dbFile = dbFile;
 		this.#appID = appID;
 		this.#pageCacheSizeKib = pageCacheSizeKib;
+		this.#rowCache = rowCache;
 	}
 	/**
 	* Initializes the snapshot to the current head of the database. This must be
@@ -137,10 +147,36 @@ var Snapshotter = class {
 	* change-applying iterations, the caller must (1) create a save point
 	* on `prev` before each iteration, and (2) rollback to the save point after
 	* the iteration.
+	*
+	* @param prevWrites How the caller writes to `prev` during the iteration,
+	*        which determines which `prev` reads can be shared through the
+	*        {@link SnapshotRowCache}. See {@link PrevWrites}.
 	*/
-	advance(syncableTables, allTableNames) {
-		const { prev, curr } = this.advanceWithoutDiff();
-		return new Diff(this.#appID, syncableTables, allTableNames, prev, curr);
+	advance(syncableTables, allTableNames, observedTables, prevWrites = "uniform") {
+		this.advanceWithoutDiff();
+		return this.diff(syncableTables, allTableNames, observedTables, prevWrites);
+	}
+	/**
+	* The diff between the two snapshots of the last
+	* {@link advanceWithoutDiff()}, as {@link advance()} returns it.
+	*
+	* @param counts The counts of the change log entries between the two
+	*        snapshots, if the caller has counted them already (see
+	*        `Snapshot.countChangesSince()`). The diff then does not count
+	*        them again.
+	* @param readTables If set, only the change log entries of these tables
+	*        (and of the permissions table) are read, along with the
+	*        table-wide ops of every table, where `observedTables` reads the
+	*        entries of every table and skips those it does not have. For a
+	*        caller that does not need the changes of the other tables, this
+	*        saves reading and parsing their entries. This requires `counts`.
+	*        If they show that there is no entry of these tables and no
+	*        table-wide op, the change log is not read at all, and the
+	*        diff's `changes` is 0.
+	*/
+	diff(syncableTables, allTableNames, observedTables, prevWrites = "uniform", counts, readTables) {
+		assert(this.#prev !== void 0 && this.#curr !== void 0, "Snapshotter has not been advanced");
+		return new Diff(this.#appID, syncableTables, allTableNames, this.#prev, this.#curr, observedTables, this.#rowCache, prevWrites, counts, readTables);
 	}
 	advanceWithoutDiff() {
 		assert(this.#curr !== void 0, "Snapshotter has not been initialized");
@@ -153,23 +189,140 @@ var Snapshotter = class {
 		};
 	}
 	/**
+	* The reset of the pipelines that iterating over the changes from the
+	* previous snapshot to the current one ends in, if it does: the first of
+	* a schema change, a truncation, and a change of the permissions. It is
+	* found without iterating, so that a caller that would reset the pipelines
+	* anyway does not first process the changes that come before it.
+	*
+	* If there is none, iterating over a {@link diff()} of the two snapshots
+	* does not throw a {@link ResetPipelinesSignal}, and can be stopped at any
+	* point without missing one.
+	*
+	* @param counts The changes from the previous snapshot to the current
+	*        one, from which it is known whether there is anything to look
+	*        for.
+	*/
+	pendingReset(syncableTables, counts) {
+		assert(this.#prev !== void 0 && this.#curr !== void 0, "Snapshotter has not been advanced");
+		const prev = this.#prev;
+		const curr = this.#curr;
+		let first;
+		if (counts.tableWideOps > 0) {
+			const { stateVersion, pos, table, op } = curr.firstTableWideOpSince(prev.version);
+			first = {
+				stateVersion,
+				pos,
+				signal: op === "t" ? new ResetPipelinesSignal(`table ${table} has been truncated`, "truncation") : new ResetPipelinesSignal(`schema for table ${table} has changed`, "schema-change")
+			};
+		}
+		const permissionsTable = `${this.#appID}.permissions`;
+		const specs = syncableTables.get(permissionsTable);
+		if (specs && (counts.sampled || (counts.byTable.get(permissionsTable) ?? 0) > 0)) {
+			const { tableSpec } = specs;
+			for (const entry of curr.entriesSince(permissionsTable, prev.version)) {
+				if (first && !isBefore(entry, first)) break;
+				const rowKey = entry.rowKey && JSON.parse(entry.rowKey);
+				if (rowKey === null) continue;
+				const nextValue = entry.op === "s" ? curr.getRow(tableSpec, rowKey) : null;
+				let prevValues;
+				if (nextValue) prevValues = prev.getRows(tableSpec, tableSpec.uniqueKeys, nextValue);
+				else {
+					const prevValue = prev.getRow(tableSpec, rowKey);
+					prevValues = prevValue ? [prevValue] : [];
+				}
+				const changed = prevValues.find((prevValue) => prevValue.permissions !== nextValue?.permissions);
+				if (changed) {
+					first = {
+						...entry,
+						signal: new ResetPipelinesSignal(`Permissions have changed ${changed.hash} => ${nextValue?.hash}`, "permissions-change")
+					};
+					break;
+				}
+			}
+		}
+		return first?.signal;
+	}
+	/**
+	* Releases the previous snapshot, i.e. the `prev` of the last
+	* {@link advance()} or {@link advanceWithoutDiff()}. It is only read
+	* while the diff that it is the start of is iterated over, but its read
+	* transaction otherwise stays open until the next advance, which keeps
+	* the replica from checkpointing its write-ahead log past it. A diff that
+	* starts at it can no longer be iterated over.
+	*/
+	releasePrevious() {
+		this.#prev?.release();
+	}
+	/**
 	* Call this to close the database connections when the Snapshotter is
 	* no longer needed.
 	*/
 	destroy() {
-		this.#curr?.db.db.close();
-		this.#prev?.db.db.close();
+		this.#curr?.db.db.close({ optimize: false });
+		this.#prev?.db.db.close({ optimize: false });
 		this.#lc.debug?.("closed database connections");
 	}
 };
+function isBefore(a, b) {
+	return a.stateVersion === b.stateVersion ? a.pos < b.pos : a.stateVersion < b.stateVersion;
+}
+/**
+* The most change log entries that are read to count a batch's changes by
+* table. Reading an entry costs microseconds, and nothing else runs on the
+* sync worker in the meantime, so the cost of counting must not grow with
+* the number of changes that a client group has to catch up on.
+*/
+var CHANGE_LOG_SAMPLE_SIZE = 1e3;
 var ResetPipelinesSignal = class extends Error {
 	name = "ResetPipelinesSignal";
 	reason;
-	constructor(msg, reason) {
+	breakdown;
+	constructor(msg, reason, breakdown) {
 		super(msg);
 		this.reason = reason;
+		this.breakdown = breakdown;
 	}
 };
+var tableReads = /* @__PURE__ */ new WeakMap();
+function readsFor(table) {
+	return getOrInsertComputed(tableReads, table, () => ({
+		prefix: `SELECT ${Object.keys(table.columns).map((c) => id(c)).join(",")} FROM ${id(table.name)} WHERE `,
+		byKeyColumns: /* @__PURE__ */ new Map(),
+		byUniqueKeys: /* @__PURE__ */ new Map()
+	}));
+}
+var MAX_MEMOIZED_UNIQUE_KEYS = 30;
+/**
+* The index that SQLite creates for the `PRIMARY KEY("stateVersion", "pos")`
+* of `_zero.changeLog2` (the first constraint of its CREATE TABLE statement,
+* hence the `_1` suffix).
+*
+* For a `stateVersion > ?` query that reads a column outside of this index
+* (e.g. `op`) and has no ORDER BY, the query planner chooses between a range
+* search of the index and a full scan of the table based on `sqlite_stat1`.
+* Statistics recorded while the change log held only a few rows make the full
+* scan look cheaper, and they persist as the table grows to millions of rows:
+* the change log of a replication-manager's replica only receives entries
+* during column backfills, and view-syncers inherit its statistics when they
+* restore its backup. Such queries therefore pin this index with INDEXED BY,
+* which also makes them fail to prepare (rather than silently scan) if the
+* index is ever renamed.
+*/
+var CHANGE_LOG_STATE_VERSION_INDEX = "sqlite_autoindex__zero.changeLog2_1";
+/**
+* The index that SQLite creates for the `UNIQUE("table", "rowKey")` of
+* `_zero.changeLog2` (the second constraint of its CREATE TABLE statement).
+*/
+var CHANGE_LOG_TABLE_INDEX = "sqlite_autoindex__zero.changeLog2_2";
+/**
+* Selects the change log entries of the tables in a JSON array of names,
+* and the table-wide ops (RESET and TRUNCATE) of every table, which abort
+* the diff whatever table they are for. Binds the two ops, then the array.
+* The array is a single parameter so that the statement is the same, and is
+* prepared once, for any set of tables.
+*/
+var CHANGE_LOG_TABLES_FILTER = `("op" IN (?, ?) OR "table" IN (SELECT "value" FROM json_each(?)))`;
 var Snapshot = class Snapshot {
 	static create(lc, dbFile, appID, pageCacheSizeKib) {
 		const conn = new Database(lc, dbFile);
@@ -182,8 +335,13 @@ var Snapshot = class Snapshot {
 	db;
 	#appID;
 	version;
-	constructor(db, appID) {
-		db.beginConcurrent();
+	#reset = false;
+	/**
+	* @param begun Whether `db` is already in a `BEGIN CONCURRENT` transaction
+	*        that has not read anything.
+	*/
+	constructor(db, appID, begun = false) {
+		if (!begun) db.beginConcurrent();
 		const { stateVersion } = getReplicationState(db);
 		this.db = db;
 		this.#appID = appID;
@@ -193,61 +351,285 @@ var Snapshot = class Snapshot {
 		const { count } = this.db.get("SELECT COUNT(*) AS count FROM \"_zero.changeLog2\" WHERE stateVersion > ?", prevVersion);
 		return count;
 	}
-	changesSince(prevVersion) {
-		const cached = this.db.statementCache.get(`SELECT "stateVersion", "table", "rowKey", "op" FROM "_zero.changeLog2"
-         WHERE "stateVersion" > ? ORDER BY "stateVersion" ASC, "pos" ASC`);
+	/**
+	* Counts the change log entries after `prevVersion`. The change log keeps
+	* the latest entry of a row, so these are counts of the rows that changed,
+	* which is what a diff from `prevVersion` to this snapshot reads.
+	*
+	* The total, and the number of table-wide ops (a RESET or TRUNCATE), are
+	* exact, and are counted from the index alone. The counts by table are
+	* exact if there are at most `sampleSize` entries. Otherwise they are
+	* estimated from the newest `sampleSize` entries, so that at most that
+	* many entries are read however many there are.
+	*/
+	countChangesSince(prevVersion, sampleSize = CHANGE_LOG_SAMPLE_SIZE) {
+		const { total, tableWideOps } = this.db.get(`SELECT COUNT(*) AS total, COALESCE(SUM(pos < 0), 0) AS tableWideOps
+         FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_STATE_VERSION_INDEX}"
+         WHERE stateVersion > ?`, prevVersion);
+		const byTable = /* @__PURE__ */ new Map();
+		if (total === 0) return {
+			total,
+			tableWideOps,
+			byTable,
+			sampled: false
+		};
+		const rows = this.db.all(`SELECT "table", COUNT(*) AS count FROM (
+           SELECT "table"
+             FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_STATE_VERSION_INDEX}"
+             WHERE stateVersion > ?
+             ORDER BY stateVersion DESC, pos DESC LIMIT ?
+         ) GROUP BY "table"`, prevVersion, sampleSize);
+		const sampled = total > sampleSize;
+		const scale = sampled ? total / sampleSize : 1;
+		for (const { table, count } of rows) byTable.set(table, count * scale);
 		return {
-			changes: cached.statement.iterate(prevVersion),
+			total,
+			tableWideOps,
+			byTable,
+			sampled
+		};
+	}
+	/**
+	* Whether an entry of `table` was logged after `prevVersion`. This reads
+	* the table's entries in the change log, however old, so it is for tables
+	* with few rows.
+	*/
+	tableChangedSince(table, prevVersion) {
+		return this.db.get(`SELECT 1 FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_TABLE_INDEX}"
+         WHERE "table" = ? AND stateVersion > ? LIMIT 1`, table, prevVersion) !== void 0;
+	}
+	/**
+	* Whether a table-wide RESET op (i.e. a schema change) was logged after
+	* `prevVersion`. Table specs computed at or before `prevVersion` are
+	* stale with respect to this snapshot if so.
+	*/
+	schemaChangedSince(prevVersion) {
+		return this.db.get(`SELECT 1 FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_STATE_VERSION_INDEX}"
+         WHERE stateVersion > ? AND op = ? LIMIT 1`, prevVersion, "r") !== void 0;
+	}
+	/**
+	* Whether any table-wide op (i.e. a RESET or TRUNCATE) was logged after
+	* `prevVersion`. Iterating over a diff ending at this snapshot will
+	* abort with a {@link ResetPipelinesSignal} if so.
+	*/
+	hasTableWideOpSince(prevVersion) {
+		return this.db.get(`SELECT 1 FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_STATE_VERSION_INDEX}"
+         WHERE stateVersion > ? AND op IN (?, ?) LIMIT 1`, prevVersion, "r", "t") !== void 0;
+	}
+	/**
+	* The first table-wide op (a schema change or a truncation) logged after
+	* `prevVersion`, in the order that {@link changesSince} reads the entries.
+	*/
+	firstTableWideOpSince(prevVersion) {
+		return this.db.get(`SELECT "stateVersion", "pos", "table", "op" FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_STATE_VERSION_INDEX}"
+         WHERE "stateVersion" > ? AND "pos" < 0
+         ORDER BY "stateVersion" ASC, "pos" ASC LIMIT 1`, prevVersion);
+	}
+	/**
+	* The entries of `table` logged after `prevVersion`, in the order that
+	* {@link changesSince} reads them. For a table with few rows.
+	*/
+	entriesSince(table, prevVersion) {
+		return this.db.all(`SELECT "stateVersion", "pos", "rowKey", "op" FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_TABLE_INDEX}"
+         WHERE "table" = ? AND "stateVersion" > ?
+         ORDER BY "stateVersion" ASC, "pos" ASC`, table, prevVersion);
+	}
+	/**
+	* @param tables If set, only the entries of these tables, and the
+	*        table-wide ops of every table, are read.
+	*/
+	changesSince(prevVersion, tables) {
+		const cached = this.db.statementCache.get(tables === void 0 ? `SELECT "stateVersion", "table", "rowKey", "op" FROM "_zero.changeLog2"
+         WHERE "stateVersion" > ? ORDER BY "stateVersion" ASC, "pos" ASC` : `SELECT "stateVersion", "table", "rowKey", "op" FROM "_zero.changeLog2"
+         WHERE "stateVersion" > ? AND ${CHANGE_LOG_TABLES_FILTER}
+         ORDER BY "stateVersion" ASC, "pos" ASC`);
+		return {
+			changes: tables === void 0 ? cached.statement.iterate(prevVersion) : cached.statement.iterate(prevVersion, "r", "t", tables),
 			cleanup: () => this.db.statementCache.return(cached)
 		};
 	}
-	getRow(table, rowKey) {
+	/**
+	* Reads the row with the given `rowKey`, or `undefined` if it does not
+	* exist.
+	*
+	* @param cache An optional {@link SnapshotRowCache} through which to
+	*        perform the read, along with the `tag` that identifies the
+	*        version(s) of the replica that the result depends on.
+	*/
+	getRow(table, rowKey, cache, tag) {
 		const key = normalizedKeyOrder(rowKey);
-		const conds = Object.keys(key).map((c) => `${id(c)}=?`);
-		const cols = Object.keys(table.columns);
-		const cached = this.db.statementCache.get(`SELECT ${cols.map((c) => id(c)).join(",")} FROM ${id(table.name)} WHERE ${conds.join(" AND ")}`);
-		cached.statement.safeIntegers(true);
-		try {
-			return cached.statement.get(Object.values(key));
-		} finally {
-			this.db.statementCache.return(cached);
-		}
+		const cols = Object.keys(key);
+		const reads = readsFor(table);
+		const sql = getOrInsertComputed(reads.byKeyColumns, cols.join("\0"), () => reads.prefix + cols.map((c) => `${id(c)}=?`).join(" AND "));
+		const args = Object.values(key);
+		const read = () => this.db.statementCache.use(sql, (cached) => {
+			cached.statement.safeIntegers(true);
+			const row = cached.statement.get(args);
+			recordIndexUsage(cached.statement, table.name);
+			return row;
+		});
+		return cache ? cache.getOrRead(tag ?? "", sql, "get", args, () => {
+			const row = read();
+			if (row !== void 0) this.#assertNotAdvanced(row);
+			return row;
+		}) : read();
 	}
-	getRows(table, keys, row) {
-		const validKeys = keys.filter((key) => key.every((column) => row[column] !== null && row[column] !== void 0));
+	/**
+	* Reads the rows that conflict with `row` on any of the given unique
+	* `keys` (which includes the row with the same primary key, if any).
+	*
+	* @param cache An optional {@link SnapshotRowCache} through which to
+	*        perform the read, along with the `tag` that identifies the
+	*        version(s) of the replica that the result depends on.
+	*/
+	getRows(table, keys, row, cache, tag) {
+		let validKeys = keys;
+		let mask = (1 << keys.length) - 1;
+		for (let i = 0; i < keys.length; i++) {
+			const key = keys[i];
+			for (let j = 0; j < key.length; j++) {
+				const val = row[key[j]];
+				if (val === null || val === void 0) {
+					validKeys = [];
+					mask = 0;
+					for (let k = 0; k < keys.length; k++) {
+						const candidate = keys[k];
+						if (candidate.every((col) => row[col] !== null && row[col] !== void 0)) {
+							validKeys.push(candidate);
+							mask |= 1 << k;
+						}
+					}
+					break;
+				}
+			}
+			if (validKeys !== keys) break;
+		}
 		if (validKeys.length === 0) return [];
-		const conds = validKeys.map((key) => key.map((c) => `${id(c)}=?`));
-		const cols = Object.keys(table.columns);
-		const cached = this.db.statementCache.get(`SELECT ${cols.map((c) => id(c)).join(",")} FROM ${id(table.name)} WHERE ${conds.map((cond) => cond.join(" AND ")).join(" OR ")}`);
-		cached.statement.safeIntegers(true);
-		try {
-			return cached.statement.all(validKeys.flatMap((key) => key.map((column) => row[column])));
-		} finally {
-			this.db.statementCache.return(cached);
+		const reads = readsFor(table);
+		const buildSQL = () => reads.prefix + validKeys.map((key) => key.map((c) => `${id(c)}=?`).join(" AND ")).join(" OR ");
+		const sql = keys === table.uniqueKeys && keys.length <= MAX_MEMOIZED_UNIQUE_KEYS ? getOrInsertComputed(reads.byUniqueKeys, mask, buildSQL) : buildSQL();
+		const args = [];
+		for (let i = 0; i < validKeys.length; i++) {
+			const key = validKeys[i];
+			for (let j = 0; j < key.length; j++) args.push(row[key[j]]);
+		}
+		const read = () => this.db.statementCache.use(sql, (cached) => {
+			cached.statement.safeIntegers(true);
+			const rows = cached.statement.all(args);
+			recordIndexUsage(cached.statement, table.name);
+			return rows;
+		});
+		return cache ? cache.getOrRead(tag ?? "", sql, "all", args, () => {
+			const rows = read();
+			for (const row of rows) this.#assertNotAdvanced(row);
+			return rows;
+		}) : read();
+	}
+	/**
+	* Rows are shared with other client groups via the SnapshotRowCache, so
+	* one read from a connection that has advanced past this snapshot (i.e. a
+	* {@link SnapshotDiff} used after its Snapshotter advanced) must fail
+	* before it is cached, rather than be served to the valid Diffs of other
+	* client groups (whose own validity checks would then fail).
+	*/
+	#assertNotAdvanced(row) {
+		if ((row["_0_version"] ?? "~") > this.version) throw new InvalidDiffError(`Diff is no longer valid. db has advanced past ${this.version}.`);
+	}
+	/**
+	* Whether the snapshot has been released, or its connection reset to a
+	* newer snapshot (i.e. via {@link release()} or {@link resetToHead()}),
+	* after which reads no longer reflect {@link version}.
+	*/
+	get reset() {
+		return this.#reset;
+	}
+	/**
+	* Ends the snapshot's read transaction, after which reads no longer
+	* reflect {@link version}. An open read transaction keeps SQLite from
+	* checkpointing the write-ahead log past the snapshot, so a snapshot that
+	* is no longer read should not be left open.
+	*/
+	release() {
+		if (!this.#reset) {
+			this.#reset = true;
+			this.db.rollback();
+			this.db.beginConcurrent();
 		}
 	}
 	resetToHead() {
-		this.db.rollback();
-		return new Snapshot(this.db, this.#appID);
+		this.release();
+		return new Snapshot(this.db, this.#appID, true);
 	}
 };
 var Diff = class {
 	#permissionsTable;
 	#syncableTables;
 	#allTableNames;
+	#observedTables;
+	#rowCache;
+	/** The JSON array of the names of the tables whose entries are read. */
+	#readTables;
+	#permissionsChanged;
+	#nothingToRead = false;
+	#prevWrites;
+	#prevTagUniform;
+	#prevTagDivergent;
+	#iterated = false;
 	prev;
 	curr;
 	changes;
-	constructor(appID, syncableTables, allTableNames, prev, curr) {
+	changesScanned = 0;
+	changesScannedByTable = /* @__PURE__ */ new Map();
+	changesSkipped = 0;
+	changesLookedUp = 0;
+	stats = {
+		changeLogReadMs: 0,
+		rowReadMs: 0,
+		parseMs: 0,
+		totalDiffMs: 0
+	};
+	constructor(appID, syncableTables, allTableNames, prev, curr, observedTables, rowCache, prevWrites = "uniform", counts, readTables) {
+		this.#prevWrites = prevWrites;
 		this.#permissionsTable = `${appID}.permissions`;
+		const read = readTables && new Set(readTables).add(this.#permissionsTable);
+		this.#readTables = read && JSON.stringify([...read]);
 		this.#syncableTables = syncableTables;
 		this.#allTableNames = allTableNames;
+		this.#observedTables = observedTables;
 		this.prev = prev;
 		this.curr = curr;
-		this.changes = curr.numChangesSince(prev.version);
+		this.#prevTagUniform = `p:${prev.version}`;
+		this.#prevTagDivergent = `p:${prev.version}:${curr.version}`;
+		if (counts === void 0) {
+			assert(read === void 0, "readTables requires counts");
+			this.changes = curr.numChangesSince(prev.version);
+		} else {
+			const hasEntries = (table) => this.#hasEntries(counts, table);
+			this.#nothingToRead = counts.total === 0 || read !== void 0 && counts.tableWideOps === 0 && ![...read].some(hasEntries);
+			this.changes = this.#nothingToRead ? 0 : counts.total;
+		}
+		const hasTableWideOp = counts ? counts.tableWideOps > 0 : curr.hasTableWideOpSince(prev.version);
+		this.#rowCache = rowCache && !hasTableWideOp ? rowCache : void 0;
+	}
+	/**
+	* Whether the diff has an entry of `table`. Sampled counts cannot show
+	* that a table has none, so the answer is then `true`, except for the
+	* permissions table: it has few rows, so whether it has an entry is
+	* looked up instead.
+	*/
+	#hasEntries(counts, table) {
+		if (!counts.sampled) return (counts.byTable.get(table) ?? 0) > 0;
+		if (table !== this.#permissionsTable) return true;
+		return this.#permissionsChanged ??= this.curr.tableChangedSince(table, this.prev.version);
+	}
+	setPrevWrites(prevWrites) {
+		assert(!this.#iterated || prevWrites === "divergent", "prevWrites must be set before iterating the diff");
+		this.#prevWrites = prevWrites;
 	}
 	[Symbol.iterator]() {
-		const { changes, cleanup: done } = this.curr.changesSince(this.prev.version);
+		this.#iterated = true;
+		if (this.#nothingToRead) return [][Symbol.iterator]();
+		const { changes, cleanup: done } = this.curr.changesSince(this.prev.version, this.#readTables);
 		const cleanup = () => {
 			try {
 				changes.return?.(void 0);
@@ -257,9 +639,12 @@ var Diff = class {
 		};
 		return {
 			next: () => {
+				const nextStart = performance.now();
 				try {
 					for (;;) {
+						const clStart = performance.now();
 						const { value, done } = changes.next();
+						this.stats.changeLogReadMs += performance.now() - clStart;
 						if (done) {
 							cleanup();
 							return {
@@ -267,27 +652,52 @@ var Diff = class {
 								done: true
 							};
 						}
+						this.changesScanned++;
+						const parseStart = performance.now();
 						const { table, rowKey, op, stateVersion } = parse(value, changeLogEntrySchema);
+						this.stats.parseMs += performance.now() - parseStart;
+						this.changesScannedByTable.set(table, (this.changesScannedByTable.get(table) ?? 0) + 1);
 						if (op === "r") throw new ResetPipelinesSignal(`schema for table ${table} has changed`, "schema-change");
 						if (op === "t") throw new ResetPipelinesSignal(`table ${table} has been truncated`, "truncation");
+						if (this.#observedTables && !this.#observedTables.has(table) && table !== this.#permissionsTable) {
+							if (this.#allTableNames.has(table)) {
+								this.changesSkipped++;
+								continue;
+							}
+							throw new Error(`change for unknown table ${table}`);
+						}
 						const specs = this.#syncableTables.get(table);
 						if (!specs) {
-							if (this.#allTableNames.has(table)) continue;
+							if (this.#allTableNames.has(table)) {
+								this.changesSkipped++;
+								continue;
+							}
 							throw new Error(`change for unknown table ${table}`);
 						}
 						const { tableSpec, zqlSpec } = specs;
 						assert((tableSpec.minRowVersion ?? "") < stateVersion, () => `unexpected change @${stateVersion} for table ${table} with minRowVersion ${tableSpec.minRowVersion}: ${op}(${rowKey})`);
 						assert(rowKey !== null, "rowKey must be present for row changes");
-						const nextValue = op === "s" ? this.curr.getRow(tableSpec, rowKey) : null;
+						if (this.prev.reset) throw new InvalidDiffError(`Diff is no longer valid. prev db has advanced past ${this.prev.version}.`);
+						const cache = this.#rowCache;
+						const prevDependsOnWrites = tableSpec.uniqueKeys.length > 1 && this.#prevWrites !== "none";
+						const prevCache = prevDependsOnWrites && this.#prevWrites === "divergent" ? void 0 : cache;
+						const prevTag = prevDependsOnWrites ? this.#prevTagDivergent : this.#prevTagUniform;
+						this.changesLookedUp++;
+						const rowStart = performance.now();
+						const nextValue = op === "s" ? this.curr.getRow(tableSpec, rowKey, cache, `n:${stateVersion}`) : null;
 						let prevValues;
-						if (nextValue) prevValues = this.prev.getRows(tableSpec, tableSpec.uniqueKeys, nextValue);
+						if (nextValue) prevValues = this.prev.getRows(tableSpec, tableSpec.uniqueKeys, nextValue, prevCache, prevTag);
 						else {
-							const prevValue = this.prev.getRow(tableSpec, rowKey);
+							const prevValue = this.prev.getRow(tableSpec, rowKey, prevCache, prevTag);
 							prevValues = prevValue ? [prevValue] : [];
 						}
+						this.stats.rowReadMs += performance.now() - rowStart;
 						if (nextValue === void 0) throw new Error(`Missing value for ${table} ${stringify(rowKey)}`);
 						this.checkThatDiffIsValid(stateVersion, op, prevValues, nextValue);
-						if (prevValues.length === 0 && nextValue === null) continue;
+						if (prevValues.length === 0 && nextValue === null) {
+							this.changesSkipped++;
+							continue;
+						}
 						if (table === this.#permissionsTable && prevValues.some((prevValue) => prevValue.permissions !== nextValue.permissions)) throw new ResetPipelinesSignal(`Permissions have changed ${prevValues.find((prevValue) => prevValue.permissions !== nextValue.permissions).hash} => ${nextValue.hash}`, "permissions-change");
 						return { value: {
 							table,
@@ -299,6 +709,8 @@ var Diff = class {
 				} catch (e) {
 					cleanup();
 					throw e;
+				} finally {
+					this.stats.totalDiffMs += performance.now() - nextStart;
 				}
 			},
 			return: (value) => {