@rocicorp/zero 1.11.0-canary.2 → 1.11.0-canary.20

package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts

@@ -6,6 +6,7 @@ import type { LiteAndZqlSpec, LiteTableSpecWithKeysAndVersion } from '../../db/s
 import { StatementRunner } from '../../db/statements.ts';
 import { type RowKey, type RowValue } from '../../types/row-key.ts';
 import type { AppID } from '../../types/shards.ts';
+import type { SnapshotRowCache } from './snapshot-row-cache.ts';
 /**
  * A `Snapshotter` manages the progression of database snapshots for a
  * ViewSyncer.
@@ -67,7 +68,13 @@ import type { AppID } from '../../types/shards.ts';
  */
 export declare class Snapshotter {
     #private;
-    constructor(lc: LogContext, dbFile: string, { appID }: AppID, pageCacheSizeKib?: number);
+    /**
+     * @param rowCache An optional worker-wide {@link SnapshotRowCache} through
+     *        which the row reads performed when iterating over a
+     *        {@link SnapshotDiff} are shared with the other Snapshotters
+     *        (i.e. client groups) on the worker.
+     */
+    constructor(lc: LogContext, dbFile: string, { appID }: AppID, pageCacheSizeKib?: number, rowCache?: SnapshotRowCache);
     /**
      * Initializes the snapshot to the current head of the database. This must be
      * only be called once. The state of whether a Snapshotter has been initialized
@@ -113,8 +120,12 @@ export declare class Snapshotter {
      * change-applying iterations, the caller must (1) create a save point
      * on `prev` before each iteration, and (2) rollback to the save point after
      * the iteration.
+     *
+     * @param prevWrites How the caller writes to `prev` during the iteration,
+     *        which determines which `prev` reads can be shared through the
+     *        {@link SnapshotRowCache}. See {@link PrevWrites}.
      */
-    advance(syncableTables: Map<string, LiteAndZqlSpec>, allTableNames: Set<string>): SnapshotDiff;
+    advance(syncableTables: Map<string, LiteAndZqlSpec>, allTableNames: Set<string>, observedTables?: TableFilter | undefined, prevWrites?: PrevWrites): SnapshotDiff;
     advanceWithoutDiff(): {
         prev: Snapshot;
         curr: Snapshot;
@@ -137,6 +148,27 @@ export type Change = {
     readonly nextValue: Readonly<Row> | null;
     readonly rowKey: RowKey;
 };
+export interface TableFilter {
+    has(table: string): boolean;
+}
+/**
+ * How the caller of {@link Snapshotter.advance()} writes the changes of a
+ * {@link SnapshotDiff} to its `prev` snapshot while iterating over it.
+ *
+ * A `prev` read for a table with a single unique key (its primary key)
+ * cannot observe earlier changes of the iteration, since a row key appears
+ * once in the diff. A read for a table with other unique keys can, because
+ * the rows it returns may have been displaced or edited by earlier changes.
+ *
+ * * `uniform`: every caller applies every change, so such a read depends on
+ *   the `<prev, curr>` versions of the diff, and is shared on that basis.
+ * * `divergent`: callers apply different subsets of the changes (e.g. the
+ *   pipeline driver skips changes that no pipeline can observe), so such a
+ *   read depends on the caller and is not shared.
+ * * `none`: the caller does not write to `prev` (e.g. IVM derivation is
+ *   held in memory), so every read depends solely on `prev.version`.
+ */
+export type PrevWrites = 'uniform' | 'divergent' | 'none';
 /**
  * Represents the difference between two database Snapshots.
  * Iterating over the object will produce a sequence of {@link Change}s
@@ -165,6 +197,14 @@ export interface SnapshotDiff extends Iterable<Change> {
      *       may not be worth it for a presumable rare operation.
      */
     readonly changes: number;
+    /**
+     * Overrides the `prevWrites` passed to {@link Snapshotter.advance()}, for a
+     * caller that decides how to write to `prev` once it knows the number of
+     * {@link changes}. Must be called before the diff is iterated, except to
+     * switch to `divergent`, which can be done between any two changes: it
+     * shares only the reads that no write to `prev` can affect.
+     */
+    setPrevWrites(prevWrites: PrevWrites): void;
 }
 /**
  * Thrown during an iteration of a {@link SnapshotDiff} when a schema
@@ -184,12 +224,46 @@ declare class Snapshot {
     readonly version: string;
     constructor(db: StatementRunner, appID: string);
     numChangesSince(prevVersion: string): any;
+    /**
+     * Whether a table-wide RESET op (i.e. a schema change) was logged after
+     * `prevVersion`. Table specs computed at or before `prevVersion` are
+     * stale with respect to this snapshot if so.
+     */
+    schemaChangedSince(prevVersion: string): boolean;
+    /**
+     * Whether any table-wide op (i.e. a RESET or TRUNCATE) was logged after
+     * `prevVersion`. Iterating over a diff ending at this snapshot will
+     * abort with a {@link ResetPipelinesSignal} if so.
+     */
+    hasTableWideOpSince(prevVersion: string): boolean;
     changesSince(prevVersion: string): {
         changes: IterableIterator<unknown>;
         cleanup: () => void;
     };
-    getRow(table: LiteTableSpecWithKeysAndVersion, rowKey: JSONValue): any;
-    getRows(table: LiteTableSpecWithKeysAndVersion, keys: PrimaryKey[], row: RowValue): any[];
+    /**
+     * Reads the row with the given `rowKey`, or `undefined` if it does not
+     * exist.
+     *
+     * @param cache An optional {@link SnapshotRowCache} through which to
+     *        perform the read, along with the `tag` that identifies the
+     *        version(s) of the replica that the result depends on.
+     */
+    getRow(table: LiteTableSpecWithKeysAndVersion, rowKey: JSONValue, cache?: SnapshotRowCache, tag?: string): any;
+    /**
+     * Reads the rows that conflict with `row` on any of the given unique
+     * `keys` (which includes the row with the same primary key, if any).
+     *
+     * @param cache An optional {@link SnapshotRowCache} through which to
+     *        perform the read, along with the `tag` that identifies the
+     *        version(s) of the replica that the result depends on.
+     */
+    getRows(table: LiteTableSpecWithKeysAndVersion, keys: PrimaryKey[], row: RowValue, cache?: SnapshotRowCache, tag?: string): any[];
+    /**
+     * Whether the connection has been reset to a newer snapshot (i.e. via
+     * {@link resetToHead()}), after which reads no longer reflect
+     * {@link version}.
+     */
+    get reset(): boolean;
     resetToHead(): Snapshot;
 }
 export declare class InvalidDiffError extends Error {

package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAY,KAAK,SAAS,EAAC,MAAM,uCAAuC,CAAC;AAEhF,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,uCAAuC,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sCAAsC,CAAC;AAGrE,OAAO,KAAK,EACV,cAAc,EACd,+BAA+B,EAChC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAEL,KAAK,MAAM,EACX,KAAK,QAAQ,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,uBAAuB,CAAC;AAajD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,qBAAa,WAAW;;IAQtB,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAC,KAAK,EAAC,EAAE,KAAK,EACd,gBAAgB,CAAC,EAAE,MAAM,EAM1B;IAED;;;;OAIG;IACH,IAAI,IAAI,IAAI,CAUX;IAED,WAAW,IAAI,OAAO,CAErB;IAED,+EAA+E;IAC/E,OAAO,IAAI,QAAQ,CAGlB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoCG;IACH,OAAO,CACL,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EAC3C,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,GACzB,YAAY,CAGd;IAED,kBAAkB;QAYR,IAAI;QAAc,IAAI;MAC/B;IAED;;;OAGG;IACH,OAAO,SAIN;CACF;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,WAAW,YAAa,SAAQ,QAAQ,CAAC,MAAM,CAAC;IACpD,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IAEF;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;GAIG;AACH,MAAM,MAAM,oBAAoB,GAC5B,qBAAqB,GACrB,iBAAiB,GACjB,eAAe,GACf,YAAY,GACZ,oBAAoB,CAAC;AAEzB,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,IAAI,0BAA0B;IACvC,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAC;IAEtC,YAAY,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB,EAGpD;CACF;AAED,cAAM,QAAQ;;IACZ,MAAM,CAAC,MAAM,CACX,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,gBAAgB,EAAE,MAAM,GAAG,SAAS,YAqBrC;IAED,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;IAE7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB,YAAY,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM,EAU7C;IAED,eAAe,CAAC,WAAW,EAAE,MAAM,OAMlC;IAED,YAAY,CAAC,WAAW,EAAE,MAAM;QAQ5B,OAAO;QACP,OAAO;MAEV;IAED,MAAM,CAAC,KAAK,EAAE,+BAA+B,EAAE,MAAM,EAAE,SAAS,OAgB/D;IAED,OAAO,CACL,KAAK,EAAE,+BAA+B,EACtC,IAAI,EAAE,UAAU,EAAE,EAClB,GAAG,EAAE,QAAQ,SA8Bd;IAED,WAAW,IAAI,QAAQ,CAGtB;CACF;AA8LD,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,YAAY,GAAG,EAAE,MAAM,EAEtB;CACF"}
+{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAY,KAAK,SAAS,EAAC,MAAM,uCAAuC,CAAC;AAGhF,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,uCAAuC,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sCAAsC,CAAC;AAGrE,OAAO,KAAK,EACV,cAAc,EACd,+BAA+B,EAChC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAEL,KAAK,MAAM,EACX,KAAK,QAAQ,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,uBAAuB,CAAC;AAYjD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,yBAAyB,CAAC;AAE9D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,qBAAa,WAAW;;IAStB;;;;;OAKG;IACH,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAC,KAAK,EAAC,EAAE,KAAK,EACd,gBAAgB,CAAC,EAAE,MAAM,EACzB,QAAQ,CAAC,EAAE,gBAAgB,EAO5B;IAED;;;;OAIG;IACH,IAAI,IAAI,IAAI,CAUX;IAED,WAAW,IAAI,OAAO,CAErB;IAED,+EAA+E;IAC/E,OAAO,IAAI,QAAQ,CAGlB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwCG;IACH,OAAO,CACL,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EAC3C,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,EAC1B,cAAc,CAAC,EAAE,WAAW,GAAG,SAAS,EACxC,UAAU,GAAE,UAAsB,GACjC,YAAY,CAYd;IAED,kBAAkB;QAYR,IAAI;QAAc,IAAI;MAC/B;IAED;;;OAGG;IACH,OAAO,SAIN;CACF;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC7B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,WAAW,GAAG,MAAM,CAAC;AAE1D;;;;;;GAMG;AACH,MAAM,WAAW,YAAa,SAAQ,QAAQ,CAAC,MAAM,CAAC;IACpD,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IAEF;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB;;;;;;OAMG;IACH,aAAa,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI,CAAC;CAC7C;AAED;;;;GAIG;AACH,MAAM,MAAM,oBAAoB,GAC5B,qBAAqB,GACrB,iBAAiB,GACjB,eAAe,GACf,YAAY,GACZ,oBAAoB,CAAC;AAEzB,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,IAAI,0BAA0B;IACvC,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAC;IAEtC,YAAY,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB,EAGpD;CACF;AAqDD,cAAM,QAAQ;;IACZ,MAAM,CAAC,MAAM,CACX,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,gBAAgB,EAAE,MAAM,GAAG,SAAS,YAqBrC;IAED,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;IAE7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAGzB,YAAY,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM,EAU7C;IAED,eAAe,CAAC,WAAW,EAAE,MAAM,OAMlC;IAED;;;;OAIG;IACH,kBAAkB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAQ/C;IAED;;;;OAIG;IACH,mBAAmB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAShD;IAED,YAAY,CAAC,WAAW,EAAE,MAAM;QAQ5B,OAAO;QACP,OAAO;MAEV;IAED;;;;;;;OAOG;IACH,MAAM,CACJ,KAAK,EAAE,+BAA+B,EACtC,MAAM,EAAE,SAAS,EACjB,KAAK,CAAC,EAAE,gBAAgB,EACxB,GAAG,CAAC,EAAE,MAAM,OA0Bb;IAED;;;;;;;OAOG;IACH,OAAO,CACL,KAAK,EAAE,+BAA+B,EACtC,IAAI,EAAE,UAAU,EAAE,EAClB,GAAG,EAAE,QAAQ,EACb,KAAK,CAAC,EAAE,gBAAgB,EACxB,GAAG,CAAC,EAAE,MAAM,SAgDb;IAiBD;;;;OAIG;IACH,IAAI,KAAK,IAAI,OAAO,CAEnB;IAED,WAAW,IAAI,QAAQ,CAItB;CACF;AA2RD,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,YAAY,GAAG,EAAE,MAAM,EAEtB;CACF"}

package/out/zero-cache/src/services/view-syncer/snapshotter.js

@@ -1,5 +1,6 @@
 import { assert } from "../../../../shared/src/asserts.js";
 import { parse } from "../../../../shared/src/valita.js";
+import { getOrInsertComputed } from "../../../../shared/src/map.js";
 import { stringify } from "../../../../shared/src/bigint-json.js";
 import { Database } from "../../../../zqlite/src/db.js";
 import { fromSQLiteTypes } from "../../../../zqlite/src/table-source.js";
@@ -74,13 +75,21 @@ var Snapshotter = class {
 	#dbFile;
 	#appID;
 	#pageCacheSizeKib;
+	#rowCache;
 	#curr;
 	#prev;
-	constructor(lc, dbFile, { appID }, pageCacheSizeKib) {
+	/**
+	* @param rowCache An optional worker-wide {@link SnapshotRowCache} through
+	*        which the row reads performed when iterating over a
+	*        {@link SnapshotDiff} are shared with the other Snapshotters
+	*        (i.e. client groups) on the worker.
+	*/
+	constructor(lc, dbFile, { appID }, pageCacheSizeKib, rowCache) {
 		this.#lc = lc;
 		this.#dbFile = dbFile;
 		this.#appID = appID;
 		this.#pageCacheSizeKib = pageCacheSizeKib;
+		this.#rowCache = rowCache;
 	}
 	/**
 	* Initializes the snapshot to the current head of the database. This must be
@@ -137,10 +146,14 @@ var Snapshotter = class {
 	* change-applying iterations, the caller must (1) create a save point
 	* on `prev` before each iteration, and (2) rollback to the save point after
 	* the iteration.
+	*
+	* @param prevWrites How the caller writes to `prev` during the iteration,
+	*        which determines which `prev` reads can be shared through the
+	*        {@link SnapshotRowCache}. See {@link PrevWrites}.
 	*/
-	advance(syncableTables, allTableNames) {
+	advance(syncableTables, allTableNames, observedTables, prevWrites = "uniform") {
 		const { prev, curr } = this.advanceWithoutDiff();
-		return new Diff(this.#appID, syncableTables, allTableNames, prev, curr);
+		return new Diff(this.#appID, syncableTables, allTableNames, prev, curr, observedTables, this.#rowCache, prevWrites);
 	}
 	advanceWithoutDiff() {
 		assert(this.#curr !== void 0, "Snapshotter has not been initialized");
@@ -170,6 +183,32 @@ var ResetPipelinesSignal = class extends Error {
 		this.reason = reason;
 	}
 };
+var tableReads = /* @__PURE__ */ new WeakMap();
+function readsFor(table) {
+	return getOrInsertComputed(tableReads, table, () => ({
+		prefix: `SELECT ${Object.keys(table.columns).map((c) => id(c)).join(",")} FROM ${id(table.name)} WHERE `,
+		byKeyColumns: /* @__PURE__ */ new Map(),
+		byUniqueKeys: /* @__PURE__ */ new Map()
+	}));
+}
+var MAX_MEMOIZED_UNIQUE_KEYS = 30;
+/**
+* The index that SQLite creates for the `PRIMARY KEY("stateVersion", "pos")`
+* of `_zero.changeLog2` (the first constraint of its CREATE TABLE statement,
+* hence the `_1` suffix).
+*
+* For a `stateVersion > ?` query that reads a column outside of this index
+* (e.g. `op`) and has no ORDER BY, the query planner chooses between a range
+* search of the index and a full scan of the table based on `sqlite_stat1`.
+* Statistics recorded while the change log held only a few rows make the full
+* scan look cheaper, and they persist as the table grows to millions of rows:
+* the change log of a replication-manager's replica only receives entries
+* during column backfills, and view-syncers inherit its statistics when they
+* restore its backup. Such queries therefore pin this index with INDEXED BY,
+* which also makes them fail to prepare (rather than silently scan) if the
+* index is ever renamed.
+*/
+var CHANGE_LOG_STATE_VERSION_INDEX = "sqlite_autoindex__zero.changeLog2_1";
 var Snapshot = class Snapshot {
 	static create(lc, dbFile, appID, pageCacheSizeKib) {
 		const conn = new Database(lc, dbFile);
@@ -182,6 +221,7 @@ var Snapshot = class Snapshot {
 	db;
 	#appID;
 	version;
+	#reset = false;
 	constructor(db, appID) {
 		db.beginConcurrent();
 		const { stateVersion } = getReplicationState(db);
@@ -193,6 +233,24 @@ var Snapshot = class Snapshot {
 		const { count } = this.db.get("SELECT COUNT(*) AS count FROM \"_zero.changeLog2\" WHERE stateVersion > ?", prevVersion);
 		return count;
 	}
+	/**
+	* Whether a table-wide RESET op (i.e. a schema change) was logged after
+	* `prevVersion`. Table specs computed at or before `prevVersion` are
+	* stale with respect to this snapshot if so.
+	*/
+	schemaChangedSince(prevVersion) {
+		return this.db.get(`SELECT 1 FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_STATE_VERSION_INDEX}"
+         WHERE stateVersion > ? AND op = ? LIMIT 1`, prevVersion, "r") !== void 0;
+	}
+	/**
+	* Whether any table-wide op (i.e. a RESET or TRUNCATE) was logged after
+	* `prevVersion`. Iterating over a diff ending at this snapshot will
+	* abort with a {@link ResetPipelinesSignal} if so.
+	*/
+	hasTableWideOpSince(prevVersion) {
+		return this.db.get(`SELECT 1 FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_STATE_VERSION_INDEX}"
+         WHERE stateVersion > ? AND op IN (?, ?) LIMIT 1`, prevVersion, "r", "t") !== void 0;
+	}
 	changesSince(prevVersion) {
 		const cached = this.db.statementCache.get(`SELECT "stateVersion", "table", "rowKey", "op" FROM "_zero.changeLog2"
          WHERE "stateVersion" > ? ORDER BY "stateVersion" ASC, "pos" ASC`);
@@ -201,32 +259,83 @@ var Snapshot = class Snapshot {
 			cleanup: () => this.db.statementCache.return(cached)
 		};
 	}
-	getRow(table, rowKey) {
+	/**
+	* Reads the row with the given `rowKey`, or `undefined` if it does not
+	* exist.
+	*
+	* @param cache An optional {@link SnapshotRowCache} through which to
+	*        perform the read, along with the `tag` that identifies the
+	*        version(s) of the replica that the result depends on.
+	*/
+	getRow(table, rowKey, cache, tag) {
 		const key = normalizedKeyOrder(rowKey);
-		const conds = Object.keys(key).map((c) => `${id(c)}=?`);
-		const cols = Object.keys(table.columns);
-		const cached = this.db.statementCache.get(`SELECT ${cols.map((c) => id(c)).join(",")} FROM ${id(table.name)} WHERE ${conds.join(" AND ")}`);
-		cached.statement.safeIntegers(true);
-		try {
-			return cached.statement.get(Object.values(key));
-		} finally {
-			this.db.statementCache.return(cached);
-		}
+		const cols = Object.keys(key);
+		const reads = readsFor(table);
+		const sql = getOrInsertComputed(reads.byKeyColumns, cols.join("\0"), () => reads.prefix + cols.map((c) => `${id(c)}=?`).join(" AND "));
+		const args = Object.values(key);
+		const read = () => this.db.statementCache.use(sql, (cached) => {
+			cached.statement.safeIntegers(true);
+			return cached.statement.get(args);
+		});
+		return cache ? cache.getOrRead(tag ?? "", sql, "get", args, () => {
+			const row = read();
+			if (row !== void 0) this.#assertNotAdvanced(row);
+			return row;
+		}) : read();
 	}
-	getRows(table, keys, row) {
-		const validKeys = keys.filter((key) => key.every((column) => row[column] !== null && row[column] !== void 0));
-		if (validKeys.length === 0) return [];
-		const conds = validKeys.map((key) => key.map((c) => `${id(c)}=?`));
-		const cols = Object.keys(table.columns);
-		const cached = this.db.statementCache.get(`SELECT ${cols.map((c) => id(c)).join(",")} FROM ${id(table.name)} WHERE ${conds.map((cond) => cond.join(" AND ")).join(" OR ")}`);
-		cached.statement.safeIntegers(true);
-		try {
-			return cached.statement.all(validKeys.flatMap((key) => key.map((column) => row[column])));
-		} finally {
-			this.db.statementCache.return(cached);
+	/**
+	* Reads the rows that conflict with `row` on any of the given unique
+	* `keys` (which includes the row with the same primary key, if any).
+	*
+	* @param cache An optional {@link SnapshotRowCache} through which to
+	*        perform the read, along with the `tag` that identifies the
+	*        version(s) of the replica that the result depends on.
+	*/
+	getRows(table, keys, row, cache, tag) {
+		const validKeys = [];
+		let mask = 0;
+		for (let i = 0; i < keys.length; i++) {
+			const key = keys[i];
+			if (key.every((column) => row[column] !== null && row[column] !== void 0)) {
+				validKeys.push(key);
+				mask |= 1 << i;
+			}
 		}
+		if (validKeys.length === 0) return [];
+		const reads = readsFor(table);
+		const buildSQL = () => reads.prefix + validKeys.map((key) => key.map((c) => `${id(c)}=?`).join(" AND ")).join(" OR ");
+		const sql = keys === table.uniqueKeys && keys.length <= MAX_MEMOIZED_UNIQUE_KEYS ? getOrInsertComputed(reads.byUniqueKeys, mask, buildSQL) : buildSQL();
+		const args = validKeys.flatMap((key) => key.map((column) => row[column]));
+		const read = () => this.db.statementCache.use(sql, (cached) => {
+			cached.statement.safeIntegers(true);
+			return cached.statement.all(args);
+		});
+		return cache ? cache.getOrRead(tag ?? "", sql, "all", args, () => {
+			const rows = read();
+			for (const row of rows) this.#assertNotAdvanced(row);
+			return rows;
+		}) : read();
+	}
+	/**
+	* Rows are shared with other client groups via the SnapshotRowCache, so
+	* one read from a connection that has advanced past this snapshot (i.e. a
+	* {@link SnapshotDiff} used after its Snapshotter advanced) must fail
+	* before it is cached, rather than be served to the valid Diffs of other
+	* client groups (whose own validity checks would then fail).
+	*/
+	#assertNotAdvanced(row) {
+		if ((row["_0_version"] ?? "~") > this.version) throw new InvalidDiffError(`Diff is no longer valid. db has advanced past ${this.version}.`);
+	}
+	/**
+	* Whether the connection has been reset to a newer snapshot (i.e. via
+	* {@link resetToHead()}), after which reads no longer reflect
+	* {@link version}.
+	*/
+	get reset() {
+		return this.#reset;
 	}
 	resetToHead() {
+		this.#reset = true;
 		this.db.rollback();
 		return new Snapshot(this.db, this.#appID);
 	}
@@ -235,18 +344,30 @@ var Diff = class {
 	#permissionsTable;
 	#syncableTables;
 	#allTableNames;
+	#observedTables;
+	#rowCache;
+	#prevWrites;
+	#iterated = false;
 	prev;
 	curr;
 	changes;
-	constructor(appID, syncableTables, allTableNames, prev, curr) {
+	constructor(appID, syncableTables, allTableNames, prev, curr, observedTables, rowCache, prevWrites = "uniform") {
+		this.#prevWrites = prevWrites;
 		this.#permissionsTable = `${appID}.permissions`;
 		this.#syncableTables = syncableTables;
 		this.#allTableNames = allTableNames;
+		this.#observedTables = observedTables;
 		this.prev = prev;
 		this.curr = curr;
 		this.changes = curr.numChangesSince(prev.version);
+		this.#rowCache = rowCache && !curr.hasTableWideOpSince(prev.version) ? rowCache : void 0;
+	}
+	setPrevWrites(prevWrites) {
+		assert(!this.#iterated || prevWrites === "divergent", "prevWrites must be set before iterating the diff");
+		this.#prevWrites = prevWrites;
 	}
 	[Symbol.iterator]() {
+		this.#iterated = true;
 		const { changes, cleanup: done } = this.curr.changesSince(this.prev.version);
 		const cleanup = () => {
 			try {
@@ -270,6 +391,10 @@ var Diff = class {
 						const { table, rowKey, op, stateVersion } = parse(value, changeLogEntrySchema);
 						if (op === "r") throw new ResetPipelinesSignal(`schema for table ${table} has changed`, "schema-change");
 						if (op === "t") throw new ResetPipelinesSignal(`table ${table} has been truncated`, "truncation");
+						if (this.#observedTables && !this.#observedTables.has(table) && table !== this.#permissionsTable) {
+							if (this.#allTableNames.has(table)) continue;
+							throw new Error(`change for unknown table ${table}`);
+						}
 						const specs = this.#syncableTables.get(table);
 						if (!specs) {
 							if (this.#allTableNames.has(table)) continue;
@@ -278,11 +403,16 @@ var Diff = class {
 						const { tableSpec, zqlSpec } = specs;
 						assert((tableSpec.minRowVersion ?? "") < stateVersion, () => `unexpected change @${stateVersion} for table ${table} with minRowVersion ${tableSpec.minRowVersion}: ${op}(${rowKey})`);
 						assert(rowKey !== null, "rowKey must be present for row changes");
-						const nextValue = op === "s" ? this.curr.getRow(tableSpec, rowKey) : null;
+						if (this.prev.reset) throw new InvalidDiffError(`Diff is no longer valid. prev db has advanced past ${this.prev.version}.`);
+						const cache = this.#rowCache;
+						const prevDependsOnWrites = tableSpec.uniqueKeys.length > 1 && this.#prevWrites !== "none";
+						const prevCache = prevDependsOnWrites && this.#prevWrites === "divergent" ? void 0 : cache;
+						const prevTag = prevDependsOnWrites ? `p:${this.prev.version}:${this.curr.version}` : `p:${this.prev.version}`;
+						const nextValue = op === "s" ? this.curr.getRow(tableSpec, rowKey, cache, `n:${stateVersion}`) : null;
 						let prevValues;
-						if (nextValue) prevValues = this.prev.getRows(tableSpec, tableSpec.uniqueKeys, nextValue);
+						if (nextValue) prevValues = this.prev.getRows(tableSpec, tableSpec.uniqueKeys, nextValue, prevCache, prevTag);
 						else {
-							const prevValue = this.prev.getRow(tableSpec, rowKey);
+							const prevValue = this.prev.getRow(tableSpec, rowKey, prevCache, prevTag);
 							prevValues = prevValue ? [prevValue] : [];
 						}
 						if (nextValue === void 0) throw new Error(`Missing value for ${table} ${stringify(rowKey)}`);

package/out/zero-cache/src/services/view-syncer/snapshotter.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshotter.js","names":["#lc","#dbFile","#appID","#pageCacheSizeKib","#curr","#prev","#permissionsTable","#syncableTables","#allTableNames"],"sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {stringify, type JSONValue} from '../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Row} from '../../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../../zero-types/src/schema.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {fromSQLiteTypes} from '../../../../zqlite/src/table-source.ts';\nimport type {\n  LiteAndZqlSpec,\n  LiteTableSpecWithKeysAndVersion,\n} from '../../db/specs.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {\n  normalizedKeyOrder,\n  type RowKey,\n  type RowValue,\n} from '../../types/row-key.ts';\nimport type {AppID} from '../../types/shards.ts';\nimport {id} from '../../types/sql.ts';\nimport {\n  RESET_OP,\n  changeLogEntrySchema as schema,\n  SET_OP,\n  TRUNCATE_OP,\n} from '../replicator/schema/change-log.ts';\nimport {\n  getReplicationState,\n  ZERO_VERSION_COLUMN_NAME as ROW_VERSION,\n} from '../replicator/schema/replication-state.ts';\n\n/**\n * A `Snapshotter` manages the progression of database snapshots for a\n * ViewSyncer.\n *\n * The Replicator and ViewSyncers operate on the same SQLite file, with the\n * Replicator being the sole writer to the database. The IVM logic in\n * ViewSyncers, however, rely on incrementally applying changes to the DB to\n * update the state of its pipelines.\n *\n * To avoid coupling the progress of the Replicator and all IVM pipelines on\n * each other, ViewSyncers operate on ephemeral forks of the database by holding\n * [concurrent](https://sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md)\n * snapshots of the database and simulating (but ultimately rolling back)\n * mutations on these snapshots.\n *\n * Example:\n * 1. ViewSyncer takes `snapshot_a` at version `t1` of the database and\n *    hydrates its pipeline(s).\n * 2. Replicator applies a new transaction to the database and notifies\n *    subscribers.\n * 3. ViewSyncer takes `snapshot_b` at `t2`, and queries the `ChangeLog` at\n *    that snapshot for changes since `t1`.\n * 4. ViewSyncer applies those changes to `snapshot_a` for IVM, but does not\n *    commit them. (Recall that the Replicator is the sole writer to the db, so\n *    the ViewSyncer never commits any writes.)\n * 5. Replicator applies the next transaction and advances the database to `t3`.\n * 6. ViewSyncer rolls back `snapshot_a` and opens `snapshot_c` at `t3`, using\n *    `snapshot_b` to simulate changes from `t2` to `t3`.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ```\n *\n * Note that the Replicator (and ViewSyncers) do not wait on the progress of\n * other ViewSyncers. If a ViewSyncer is busy hydrating at `t1`, the Replicator\n * and other ViewSyncers can progress through `t2`, `t3`, etc. independently,\n * as the busy ViewSyncer simply takes its own snapshot when it is ready.\n *\n * ```\n * Replicator:   t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer1:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ViewSyncer2:       [.......... snapshot_a ..........] ----> [snapshot_b]\n * ```\n *\n * To minimize Database connections (and statement preparation, etc.), the\n * Snapshotter reuses the connection from the previous (rolled back)\n * snapshot when opening the new one.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n *                     (conn_1)           (conn_2)           (conn_1)\n * ```\n *\n * In this manner, each ViewSyncer uses two connections that continually\n * \"leapfrog\" each other to replay the timeline of changes in isolation from\n * the Replicator and other ViewSyncers.\n */\nexport class Snapshotter {\n  readonly #lc: LogContext;\n  readonly #dbFile: string;\n  readonly #appID: string;\n  readonly #pageCacheSizeKib: number | undefined;\n  #curr: Snapshot | undefined;\n  #prev: Snapshot | undefined;\n\n  constructor(\n    lc: LogContext,\n    dbFile: string,\n    {appID}: AppID,\n    pageCacheSizeKib?: number,\n  ) {\n    this.#lc = lc;\n    this.#dbFile = dbFile;\n    this.#appID = appID;\n    this.#pageCacheSizeKib = pageCacheSizeKib;\n  }\n\n  /**\n   * Initializes the snapshot to the current head of the database. This must be\n   * only be called once. The state of whether a Snapshotter has been initialized\n   * can be determined by calling {@link initialized()}.\n   */\n  init(): this {\n    assert(this.#curr === undefined, 'Already initialized');\n    this.#curr = Snapshot.create(\n      this.#lc,\n      this.#dbFile,\n      this.#appID,\n      this.#pageCacheSizeKib,\n    );\n    this.#lc.debug?.(`Initial snapshot at version ${this.#curr.version}`);\n    return this;\n  }\n\n  initialized(): boolean {\n    return this.#curr !== undefined;\n  }\n\n  /** Returns the current snapshot. Asserts if {@link initialized()} is false. */\n  current(): Snapshot {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    return this.#curr;\n  }\n\n  /**\n   * Advances to the head of the Database, returning a diff between the\n   * previously current Snapshot and a new Snapshot at head. This is called\n   * in response to a notification from a Replicator subscription. Subsequent\n   * calls to {@link current()} return the new Snapshot. Note that the Snapshotter\n   * must be initialized before advancing.\n   *\n   * The returned {@link SnapshotDiff} contains snapshots at the endpoints\n   * of the database timeline. Iterating over the diff generates a sequence\n   * of {@link Change}s between the two snapshots.\n   *\n   * Note that this sequence is not chronological; rather, the sequence is\n   * ordered by `<table, row-key>`, such that a row can appear at most once\n   * in the common case, or twice if its table is `TRUNCATE`'d and a new value\n   * is subsequently `INSERT`'ed. This results in dropping most intermediate\n   * changes to a row and bounds the amount of work needed to catch up;\n   * however, as a consequence, a consistent database state is only guaranteed\n   * when the sequence has been fully consumed.\n   *\n   * Note that Change generation relies on the state of the underlying\n   * database connections, and because the connection for the previous snapshot\n   * is reused to produce the next snapshot, the diff object is only valid\n   * until the next call to `advance()`.\n   *\n   * It is okay for the caller to apply `Change`s to the `prev` snapshot\n   * during the iteration (e.g. this is necessary for IVM); the remainder\n   * of the iteration is not affected because a given row can appear at most\n   * once in the sequence (with the exception being TRUNCATE, after which the\n   * deleted rows can be re-inserted, but this will also behave correctly if\n   * the changes are applied).\n   *\n   * Once the changes have been applied, however, a _subsequent_ iteration\n   * will not produce the correct results. In order to perform multiple\n   * change-applying iterations, the caller must (1) create a save point\n   * on `prev` before each iteration, and (2) rollback to the save point after\n   * the iteration.\n   */\n  advance(\n    syncableTables: Map<string, LiteAndZqlSpec>,\n    allTableNames: Set<string>,\n  ): SnapshotDiff {\n    const {prev, curr} = this.advanceWithoutDiff();\n    return new Diff(this.#appID, syncableTables, allTableNames, prev, curr);\n  }\n\n  advanceWithoutDiff() {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    const next = this.#prev\n      ? this.#prev.resetToHead()\n      : Snapshot.create(\n          this.#lc,\n          this.#curr.db.db.name,\n          this.#appID,\n          this.#pageCacheSizeKib,\n        );\n    this.#prev = this.#curr;\n    this.#curr = next;\n    return {prev: this.#prev, curr: this.#curr};\n  }\n\n  /**\n   * Call this to close the database connections when the Snapshotter is\n   * no longer needed.\n   */\n  destroy() {\n    this.#curr?.db.db.close();\n    this.#prev?.db.db.close();\n    this.#lc.debug?.('closed database connections');\n  }\n}\n\nexport type Change = {\n  readonly table: string;\n  /**\n   * If this change represents a remove the row to remove,\n   * if nextValue is not null then all rows that have a unique constraint\n   * violation with nextValue.\n   * In both cases these rows should be removed.\n   */\n  readonly prevValues: Readonly<Row>[];\n  readonly nextValue: Readonly<Row> | null;\n  readonly rowKey: RowKey;\n};\n\n/**\n * Represents the difference between two database Snapshots.\n * Iterating over the object will produce a sequence of {@link Change}s\n * between the two snapshots.\n *\n * See {@link Snapshotter.advance()} for semantics and usage.\n */\nexport interface SnapshotDiff extends Iterable<Change> {\n  readonly prev: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n  readonly curr: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n\n  /**\n   * The number of ChangeLog entries between the snapshots. Note that this\n   * may not necessarily equal the number of `Change` objects that the iteration\n   * will produce, as `TRUNCATE` entries are counted as a single log entry which\n   * may be expanded into many changes (i.e. row deletes).\n   *\n   * TODO: Determine if it is worth changing the definition to count the\n   *       truncated rows. This would make diff computation more expensive\n   *       (requiring the count to be aggregated by operation type), which\n   *       may not be worth it for a presumable rare operation.\n   */\n  readonly changes: number;\n}\n\n/**\n * Thrown during an iteration of a {@link SnapshotDiff} when a schema\n * change or truncate is encountered, which result in aborting the\n * advancement and resetting / rehydrating the pipelines.\n */\nexport type ResetPipelinesReason =\n  | 'advancement-timeout'\n  | 'scalar-subquery'\n  | 'schema-change'\n  | 'truncation'\n  | 'permissions-change';\n\nexport class ResetPipelinesSignal extends Error {\n  readonly name = 'ResetPipelinesSignal';\n  readonly reason: ResetPipelinesReason;\n\n  constructor(msg: string, reason: ResetPipelinesReason) {\n    super(msg);\n    this.reason = reason;\n  }\n}\n\nclass Snapshot {\n  static create(\n    lc: LogContext,\n    dbFile: string,\n    appID: string,\n    pageCacheSizeKib: number | undefined,\n  ) {\n    const conn = new Database(lc, dbFile);\n    conn.pragma('synchronous = OFF'); // Applied changes are ephemeral; COMMIT is never called.\n    if (pageCacheSizeKib !== undefined) {\n      conn.pragma(`cache_size = -${pageCacheSizeKib}`); // Negative = size in KiB\n    }\n    const [{journal_mode: mode}] = conn.pragma('journal_mode') as [\n      {journal_mode: string},\n    ];\n    // BEGIN CONCURRENT lets the Snapshotter simulate changes on a historic\n    // snapshot without taking the WAL writer lock. These private changes are\n    // always rolled back; the replicator is the only committer and can use\n    // BEGIN IMMEDIATE on the same WAL2 database.\n    assert(\n      mode === 'wal2',\n      `replica db must be in wal2 mode (current: ${mode})`,\n    );\n\n    const db = new StatementRunner(conn);\n    return new Snapshot(db, appID);\n  }\n\n  readonly db: StatementRunner;\n  readonly #appID: string;\n  readonly version: string;\n\n  constructor(db: StatementRunner, appID: string) {\n    db.beginConcurrent();\n    // Note: The subsequent read is necessary to acquire the read lock\n    // (which results in the logical creation of the snapshot). Calling\n    // `BEGIN CONCURRENT` alone does not result in acquiring the read lock.\n    const {stateVersion} = getReplicationState(db);\n\n    this.db = db;\n    this.#appID = appID;\n    this.version = stateVersion;\n  }\n\n  numChangesSince(prevVersion: string) {\n    const {count} = this.db.get(\n      'SELECT COUNT(*) AS count FROM \"_zero.changeLog2\" WHERE stateVersion > ?',\n      prevVersion,\n    );\n    return count;\n  }\n\n  changesSince(prevVersion: string) {\n    // Note: The queried fields are constrained to only those that are relevant\n    // to the snapshot diff, i.e. those defined in the changeLogEntrySchema.\n    const cached = this.db.statementCache.get(\n      `SELECT \"stateVersion\", \"table\", \"rowKey\", \"op\" FROM \"_zero.changeLog2\"\n         WHERE \"stateVersion\" > ? ORDER BY \"stateVersion\" ASC, \"pos\" ASC`,\n    );\n    return {\n      changes: cached.statement.iterate(prevVersion),\n      cleanup: () => this.db.statementCache.return(cached),\n    };\n  }\n\n  getRow(table: LiteTableSpecWithKeysAndVersion, rowKey: JSONValue) {\n    const key = normalizedKeyOrder(rowKey as RowKey);\n    const conds = Object.keys(key).map(c => `${id(c)}=?`);\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.join(' AND ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.get<any>(Object.values(key));\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  getRows(\n    table: LiteTableSpecWithKeysAndVersion,\n    keys: PrimaryKey[],\n    row: RowValue,\n  ) {\n    // Filter out keys where any column is NULL. This is both correct and\n    // critical for performance:\n    // 1. Correctness: NULL values can't violate uniqueness (NULL != NULL in SQL)\n    // 2. Performance: SQLite's MULTI-INDEX OR optimization completely fails when\n    //    any branch involves NULL, falling back to a full table scan. This was\n    //    causing slowdowns of hundreds of times on tables with nullable unique columns.\n    const validKeys = keys.filter(key =>\n      key.every(column => row[column] !== null && row[column] !== undefined),\n    );\n    if (validKeys.length === 0) {\n      return [];\n    }\n    const conds = validKeys.map(key => key.map(c => `${id(c)}=?`));\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.map(cond => cond.join(' AND ')).join(' OR ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.all<any>(\n        validKeys.flatMap(key => key.map(column => row[column])),\n      );\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  resetToHead(): Snapshot {\n    this.db.rollback();\n    return new Snapshot(this.db, this.#appID);\n  }\n}\n\nclass Diff implements SnapshotDiff {\n  readonly #permissionsTable: string;\n  readonly #syncableTables: Map<string, LiteAndZqlSpec>;\n  readonly #allTableNames: Set<string>;\n  readonly prev: Snapshot;\n  readonly curr: Snapshot;\n  readonly changes: number;\n\n  constructor(\n    appID: string,\n    syncableTables: Map<string, LiteAndZqlSpec>,\n    allTableNames: Set<string>,\n    prev: Snapshot,\n    curr: Snapshot,\n  ) {\n    this.#permissionsTable = `${appID}.permissions`;\n    this.#syncableTables = syncableTables;\n    this.#allTableNames = allTableNames;\n    this.prev = prev;\n    this.curr = curr;\n    this.changes = curr.numChangesSince(prev.version);\n  }\n\n  [Symbol.iterator](): Iterator<Change> {\n    const {changes, cleanup: done} = this.curr.changesSince(this.prev.version);\n\n    const cleanup = () => {\n      try {\n        // Allow open iterators to clean up their state.\n        changes.return?.(undefined);\n      } finally {\n        done();\n      }\n    };\n\n    return {\n      next: () => {\n        try {\n          for (;;) {\n            const {value, done} = changes.next();\n            if (done) {\n              cleanup();\n              return {value, done: true};\n            }\n\n            const {table, rowKey, op, stateVersion} = v.parse(value, schema);\n            if (op === RESET_OP) {\n              // The current map of `TableSpec`s may not have the correct or complete information.\n              throw new ResetPipelinesSignal(\n                `schema for table ${table} has changed`,\n                'schema-change',\n              );\n            }\n            if (op === TRUNCATE_OP) {\n              // Truncates are also processed by rehydrating pipelines at current.\n              throw new ResetPipelinesSignal(\n                `table ${table} has been truncated`,\n                'truncation',\n              );\n            }\n            const specs = this.#syncableTables.get(table);\n            if (!specs) {\n              if (this.#allTableNames.has(table)) {\n                continue; // skip change log entries for non-syncable tables.\n              }\n              throw new Error(`change for unknown table ${table}`);\n            }\n            const {tableSpec, zqlSpec} = specs;\n\n            // Sanity check: All change log ops should have a stateVersion\n            // greater than minRowVersion in the table metadata. This is a\n            // mini-proof that the overlay does not need to be applied to\n            // rows produced in incremental catchup, based on the invariant in\n            // change-processor's #bumpVersions(), whereby the setting of the\n            // minRowVersion is always followed by a RESET OP, meaning that\n            // subsequent change-log traversal happens at a later version.\n            assert(\n              (tableSpec.minRowVersion ?? '') < stateVersion,\n              () =>\n                `unexpected change @${stateVersion} for table ${table} with ` +\n                `minRowVersion ${tableSpec.minRowVersion}: ${op}(${rowKey})`,\n            );\n\n            assert(rowKey !== null, 'rowKey must be present for row changes');\n            const nextValue =\n              op === SET_OP ? this.curr.getRow(tableSpec, rowKey) : null;\n            let prevValues;\n            if (nextValue) {\n              prevValues = this.prev.getRows(\n                tableSpec,\n                tableSpec.uniqueKeys,\n                nextValue,\n              );\n            } else {\n              const prevValue = this.prev.getRow(tableSpec, rowKey);\n              prevValues = prevValue ? [prevValue] : [];\n            }\n            if (nextValue === undefined) {\n              throw new Error(\n                `Missing value for ${table} ${stringify(rowKey)}`,\n              );\n            }\n            // Sanity check detects if the diff is being accessed after the Snapshots have advanced.\n            this.checkThatDiffIsValid(stateVersion, op, prevValues, nextValue);\n\n            if (prevValues.length === 0 && nextValue === null) {\n              // Filter out no-op changes (e.g. a delete of a row that does not exist in prev).\n              // TODO: Consider doing this for deep-equal values.\n              continue;\n            }\n\n            if (\n              table === this.#permissionsTable &&\n              prevValues.some(\n                prevValue => prevValue.permissions !== nextValue.permissions,\n              )\n            ) {\n              throw new ResetPipelinesSignal(\n                `Permissions have changed ${\n                  prevValues.find(\n                    prevValue =>\n                      prevValue.permissions !== nextValue.permissions,\n                  ).hash\n                } => ${nextValue.hash}`,\n                'permissions-change',\n              );\n            }\n\n            // Modify the values in place when converting to ZQL rows\n            // This is safe since we're the first node in the iterator chain.\n            // TODO Can we get rid of these RowValue casts?\n            return {\n              value: {\n                table,\n                prevValues: prevValues.map(prevValue =>\n                  fromSQLiteTypes(zqlSpec, prevValue, table),\n                ),\n                nextValue: nextValue\n                  ? fromSQLiteTypes(zqlSpec, nextValue, table)\n                  : null,\n                rowKey,\n              } satisfies Change,\n            };\n          }\n        } catch (e) {\n          // This control flow path is not covered by the return() method (i.e. `break`).\n          cleanup();\n          throw e;\n        }\n      },\n\n      return: (value: unknown) => {\n        cleanup();\n        return {value, done: true};\n      },\n    };\n  }\n\n  checkThatDiffIsValid(\n    stateVersion: string,\n    op: string,\n    prevValues: RowValue[],\n    nextValue: RowValue,\n  ) {\n    // Sanity checks to detect that the diff is not being accessed after\n    // the Snapshots have advanced.\n    if (stateVersion > this.curr.version) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. curr db has advanced past ${this.curr.version}`,\n      );\n    }\n    if (\n      prevValues.findIndex(\n        prevValue => (prevValue[ROW_VERSION] ?? '~') > this.prev.version,\n      ) !== -1\n    ) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. prev db has advanced past ${this.prev.version}.`,\n      );\n    }\n    if (op === SET_OP && nextValue[ROW_VERSION] !== stateVersion) {\n      throw new InvalidDiffError(\n        'Diff is no longer valid. curr db has advanced.',\n      );\n    }\n  }\n}\n\nexport class InvalidDiffError extends Error {\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0FA,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,EAAC,SACD,kBACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,SAAS;EACd,KAAKC,oBAAoB;CAC3B;;;;;;CAOA,OAAa;EACX,OAAO,KAAKC,UAAU,KAAA,GAAW,qBAAqB;EACtD,KAAKA,QAAQ,SAAS,OACpB,KAAKJ,KACL,KAAKC,SACL,KAAKC,QACL,KAAKC,iBACP;EACA,KAAKH,IAAI,QAAQ,+BAA+B,KAAKI,MAAM,SAAS;EACpE,OAAO;CACT;CAEA,cAAuB;EACrB,OAAO,KAAKA,UAAU,KAAA;CACxB;;CAGA,UAAoB;EAClB,OAAO,KAAKA,UAAU,KAAA,GAAW,sCAAsC;EACvE,OAAO,KAAKA;CACd;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuCA,QACE,gBACA,eACc;EACd,MAAM,EAAC,MAAM,SAAQ,KAAK,mBAAmB;EAC7C,OAAO,IAAI,KAAK,KAAKF,QAAQ,gBAAgB,eAAe,MAAM,IAAI;CACxE;CAEA,qBAAqB;EACnB,OAAO,KAAKE,UAAU,KAAA,GAAW,sCAAsC;EACvE,MAAM,OAAO,KAAKC,QACd,KAAKA,MAAM,YAAY,IACvB,SAAS,OACP,KAAKL,KACL,KAAKI,MAAM,GAAG,GAAG,MACjB,KAAKF,QACL,KAAKC,iBACP;EACJ,KAAKE,QAAQ,KAAKD;EAClB,KAAKA,QAAQ;EACb,OAAO;GAAC,MAAM,KAAKC;GAAO,MAAM,KAAKD;EAAK;CAC5C;;;;;CAMA,UAAU;EACR,KAAKA,OAAO,GAAG,GAAG,MAAM;EACxB,KAAKC,OAAO,GAAG,GAAG,MAAM;EACxB,KAAKL,IAAI,QAAQ,6BAA6B;CAChD;AACF;AA0DA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAgB;CAChB;CAEA,YAAY,KAAa,QAA8B;EACrD,MAAM,GAAG;EACT,KAAK,SAAS;CAChB;AACF;AAEA,IAAM,WAAN,MAAM,SAAS;CACb,OAAO,OACL,IACA,QACA,OACA,kBACA;EACA,MAAM,OAAO,IAAI,SAAS,IAAI,MAAM;EACpC,KAAK,OAAO,mBAAmB;EAC/B,IAAI,qBAAqB,KAAA,GACvB,KAAK,OAAO,iBAAiB,kBAAkB;EAEjD,MAAM,CAAC,EAAC,cAAc,UAAS,KAAK,OAAO,cAAc;EAOzD,OACE,SAAS,QACT,6CAA6C,KAAK,EACpD;EAGA,OAAO,IAAI,SAAS,IADL,gBAAgB,IACX,GAAI,KAAK;CAC/B;CAEA;CACA;CACA;CAEA,YAAY,IAAqB,OAAe;EAC9C,GAAG,gBAAgB;EAInB,MAAM,EAAC,iBAAgB,oBAAoB,EAAE;EAE7C,KAAK,KAAK;EACV,KAAKE,SAAS;EACd,KAAK,UAAU;CACjB;CAEA,gBAAgB,aAAqB;EACnC,MAAM,EAAC,UAAS,KAAK,GAAG,IACtB,6EACA,WACF;EACA,OAAO;CACT;CAEA,aAAa,aAAqB;EAGhC,MAAM,SAAS,KAAK,GAAG,eAAe,IACpC;yEAEF;EACA,OAAO;GACL,SAAS,OAAO,UAAU,QAAQ,WAAW;GAC7C,eAAe,KAAK,GAAG,eAAe,OAAO,MAAM;EACrD;CACF;CAEA,OAAO,OAAwC,QAAmB;EAChE,MAAM,MAAM,mBAAmB,MAAgB;EAC/C,MAAM,QAAQ,OAAO,KAAK,GAAG,EAAE,KAAI,MAAK,GAAG,GAAG,CAAC,EAAE,GAAG;EACpD,MAAM,OAAO,OAAO,KAAK,MAAM,OAAO;EACtC,MAAM,SAAS,KAAK,GAAG,eAAe,IACpC,UAAU,KAAK,KAAI,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,EAAE,QAAQ,GAC/C,MAAM,IACR,EAAE,SAAS,MAAM,KAAK,OAAO,GAC/B;EACA,OAAO,UAAU,aAAa,IAAI;EAClC,IAAI;GAEF,OAAO,OAAO,UAAU,IAAS,OAAO,OAAO,GAAG,CAAC;EACrD,UAAU;GACR,KAAK,GAAG,eAAe,OAAO,MAAM;EACtC;CACF;CAEA,QACE,OACA,MACA,KACA;EAOA,MAAM,YAAY,KAAK,QAAO,QAC5B,IAAI,OAAM,WAAU,IAAI,YAAY,QAAQ,IAAI,YAAY,KAAA,CAAS,CACvE;EACA,IAAI,UAAU,WAAW,GACvB,OAAO,CAAC;EAEV,MAAM,QAAQ,UAAU,KAAI,QAAO,IAAI,KAAI,MAAK,GAAG,GAAG,CAAC,EAAE,GAAG,CAAC;EAC7D,MAAM,OAAO,OAAO,KAAK,MAAM,OAAO;EACtC,MAAM,SAAS,KAAK,GAAG,eAAe,IACpC,UAAU,KAAK,KAAI,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,EAAE,QAAQ,GAC/C,MAAM,IACR,EAAE,SAAS,MAAM,KAAI,SAAQ,KAAK,KAAK,OAAO,CAAC,EAAE,KAAK,MAAM,GAC9D;EACA,OAAO,UAAU,aAAa,IAAI;EAClC,IAAI;GAEF,OAAO,OAAO,UAAU,IACtB,UAAU,SAAQ,QAAO,IAAI,KAAI,WAAU,IAAI,OAAO,CAAC,CACzD;EACF,UAAU;GACR,KAAK,GAAG,eAAe,OAAO,MAAM;EACtC;CACF;CAEA,cAAwB;EACtB,KAAK,GAAG,SAAS;EACjB,OAAO,IAAI,SAAS,KAAK,IAAI,KAAKA,MAAM;CAC1C;AACF;AAEA,IAAM,OAAN,MAAmC;CACjC;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,OACA,gBACA,eACA,MACA,MACA;EACA,KAAKI,oBAAoB,GAAG,MAAM;EAClC,KAAKC,kBAAkB;EACvB,KAAKC,iBAAiB;EACtB,KAAK,OAAO;EACZ,KAAK,OAAO;EACZ,KAAK,UAAU,KAAK,gBAAgB,KAAK,OAAO;CAClD;CAEA,CAAC,OAAO,YAA8B;EACpC,MAAM,EAAC,SAAS,SAAS,SAAQ,KAAK,KAAK,aAAa,KAAK,KAAK,OAAO;EAEzE,MAAM,gBAAgB;GACpB,IAAI;IAEF,QAAQ,SAAS,KAAA,CAAS;GAC5B,UAAU;IACR,KAAK;GACP;EACF;EAEA,OAAO;GACL,YAAY;IACV,IAAI;KACF,SAAS;MACP,MAAM,EAAC,OAAO,SAAQ,QAAQ,KAAK;MACnC,IAAI,MAAM;OACR,QAAQ;OACR,OAAO;QAAC;QAAO,MAAM;OAAI;MAC3B;MAEA,MAAM,EAAC,OAAO,QAAQ,IAAI,iBAAgB,MAAQ,OAAO,oBAAM;MAC/D,IAAI,OAAA,KAEF,MAAM,IAAI,qBACR,oBAAoB,MAAM,eAC1B,eACF;MAEF,IAAI,OAAA,KAEF,MAAM,IAAI,qBACR,SAAS,MAAM,sBACf,YACF;MAEF,MAAM,QAAQ,KAAKD,gBAAgB,IAAI,KAAK;MAC5C,IAAI,CAAC,OAAO;OACV,IAAI,KAAKC,eAAe,IAAI,KAAK,GAC/B;OAEF,MAAM,IAAI,MAAM,4BAA4B,OAAO;MACrD;MACA,MAAM,EAAC,WAAW,YAAW;MAS7B,QACG,UAAU,iBAAiB,MAAM,oBAEhC,sBAAsB,aAAa,aAAa,MAAM,sBACrC,UAAU,cAAc,IAAI,GAAG,GAAG,OAAO,EAC9D;MAEA,OAAO,WAAW,MAAM,wCAAwC;MAChE,MAAM,YACJ,OAAA,MAAgB,KAAK,KAAK,OAAO,WAAW,MAAM,IAAI;MACxD,IAAI;MACJ,IAAI,WACF,aAAa,KAAK,KAAK,QACrB,WACA,UAAU,YACV,SACF;WACK;OACL,MAAM,YAAY,KAAK,KAAK,OAAO,WAAW,MAAM;OACpD,aAAa,YAAY,CAAC,SAAS,IAAI,CAAC;MAC1C;MACA,IAAI,cAAc,KAAA,GAChB,MAAM,IAAI,MACR,qBAAqB,MAAM,GAAG,UAAU,MAAM,GAChD;MAGF,KAAK,qBAAqB,cAAc,IAAI,YAAY,SAAS;MAEjE,IAAI,WAAW,WAAW,KAAK,cAAc,MAG3C;MAGF,IACE,UAAU,KAAKF,qBACf,WAAW,MACT,cAAa,UAAU,gBAAgB,UAAU,WACnD,GAEA,MAAM,IAAI,qBACR,4BACE,WAAW,MACT,cACE,UAAU,gBAAgB,UAAU,WACxC,EAAE,KACH,MAAM,UAAU,QACjB,oBACF;MAMF,OAAO,EACL,OAAO;OACL;OACA,YAAY,WAAW,KAAI,cACzB,gBAAgB,SAAS,WAAW,KAAK,CAC3C;OACA,WAAW,YACP,gBAAgB,SAAS,WAAW,KAAK,IACzC;OACJ;MACF,EACF;KACF;IACF,SAAS,GAAG;KAEV,QAAQ;KACR,MAAM;IACR;GACF;GAEA,SAAS,UAAmB;IAC1B,QAAQ;IACR,OAAO;KAAC;KAAO,MAAM;IAAI;GAC3B;EACF;CACF;CAEA,qBACE,cACA,IACA,YACA,WACA;EAGA,IAAI,eAAe,KAAK,KAAK,SAC3B,MAAM,IAAI,iBACR,sDAAsD,KAAK,KAAK,SAClE;EAEF,IACE,WAAW,WACT,eAAc,UAAA,iBAA0B,OAAO,KAAK,KAAK,OAC3D,MAAM,IAEN,MAAM,IAAI,iBACR,sDAAsD,KAAK,KAAK,QAAQ,EAC1E;EAEF,IAAI,OAAA,OAAiB,UAAA,kBAA2B,cAC9C,MAAM,IAAI,iBACR,gDACF;CAEJ;AACF;AAEA,IAAa,mBAAb,cAAsC,MAAM;CAC1C,YAAY,KAAa;EACvB,MAAM,GAAG;CACX;AACF"}
+{"version":3,"file":"snapshotter.js","names":["#lc","#dbFile","#appID","#pageCacheSizeKib","#rowCache","#curr","#prev","#assertNotAdvanced","#reset","#permissionsTable","#syncableTables","#allTableNames","#observedTables","#prevWrites","#iterated"],"sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {stringify, type JSONValue} from '../../../../shared/src/bigint-json.ts';\nimport {getOrInsertComputed} from '../../../../shared/src/map.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Row} from '../../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../../zero-types/src/schema.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {fromSQLiteTypes} from '../../../../zqlite/src/table-source.ts';\nimport type {\n  LiteAndZqlSpec,\n  LiteTableSpecWithKeysAndVersion,\n} from '../../db/specs.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {\n  normalizedKeyOrder,\n  type RowKey,\n  type RowValue,\n} from '../../types/row-key.ts';\nimport type {AppID} from '../../types/shards.ts';\nimport {id} from '../../types/sql.ts';\nimport {\n  RESET_OP,\n  changeLogEntrySchema as schema,\n  SET_OP,\n  TRUNCATE_OP,\n} from '../replicator/schema/change-log.ts';\nimport {\n  getReplicationState,\n  ZERO_VERSION_COLUMN_NAME as ROW_VERSION,\n} from '../replicator/schema/replication-state.ts';\nimport type {SnapshotRowCache} from './snapshot-row-cache.ts';\n\n/**\n * A `Snapshotter` manages the progression of database snapshots for a\n * ViewSyncer.\n *\n * The Replicator and ViewSyncers operate on the same SQLite file, with the\n * Replicator being the sole writer to the database. The IVM logic in\n * ViewSyncers, however, rely on incrementally applying changes to the DB to\n * update the state of its pipelines.\n *\n * To avoid coupling the progress of the Replicator and all IVM pipelines on\n * each other, ViewSyncers operate on ephemeral forks of the database by holding\n * [concurrent](https://sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md)\n * snapshots of the database and simulating (but ultimately rolling back)\n * mutations on these snapshots.\n *\n * Example:\n * 1. ViewSyncer takes `snapshot_a` at version `t1` of the database and\n *    hydrates its pipeline(s).\n * 2. Replicator applies a new transaction to the database and notifies\n *    subscribers.\n * 3. ViewSyncer takes `snapshot_b` at `t2`, and queries the `ChangeLog` at\n *    that snapshot for changes since `t1`.\n * 4. ViewSyncer applies those changes to `snapshot_a` for IVM, but does not\n *    commit them. (Recall that the Replicator is the sole writer to the db, so\n *    the ViewSyncer never commits any writes.)\n * 5. Replicator applies the next transaction and advances the database to `t3`.\n * 6. ViewSyncer rolls back `snapshot_a` and opens `snapshot_c` at `t3`, using\n *    `snapshot_b` to simulate changes from `t2` to `t3`.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ```\n *\n * Note that the Replicator (and ViewSyncers) do not wait on the progress of\n * other ViewSyncers. If a ViewSyncer is busy hydrating at `t1`, the Replicator\n * and other ViewSyncers can progress through `t2`, `t3`, etc. independently,\n * as the busy ViewSyncer simply takes its own snapshot when it is ready.\n *\n * ```\n * Replicator:   t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer1:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ViewSyncer2:       [.......... snapshot_a ..........] ----> [snapshot_b]\n * ```\n *\n * To minimize Database connections (and statement preparation, etc.), the\n * Snapshotter reuses the connection from the previous (rolled back)\n * snapshot when opening the new one.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n *                     (conn_1)           (conn_2)           (conn_1)\n * ```\n *\n * In this manner, each ViewSyncer uses two connections that continually\n * \"leapfrog\" each other to replay the timeline of changes in isolation from\n * the Replicator and other ViewSyncers.\n */\nexport class Snapshotter {\n  readonly #lc: LogContext;\n  readonly #dbFile: string;\n  readonly #appID: string;\n  readonly #pageCacheSizeKib: number | undefined;\n  readonly #rowCache: SnapshotRowCache | undefined;\n  #curr: Snapshot | undefined;\n  #prev: Snapshot | undefined;\n\n  /**\n   * @param rowCache An optional worker-wide {@link SnapshotRowCache} through\n   *        which the row reads performed when iterating over a\n   *        {@link SnapshotDiff} are shared with the other Snapshotters\n   *        (i.e. client groups) on the worker.\n   */\n  constructor(\n    lc: LogContext,\n    dbFile: string,\n    {appID}: AppID,\n    pageCacheSizeKib?: number,\n    rowCache?: SnapshotRowCache,\n  ) {\n    this.#lc = lc;\n    this.#dbFile = dbFile;\n    this.#appID = appID;\n    this.#pageCacheSizeKib = pageCacheSizeKib;\n    this.#rowCache = rowCache;\n  }\n\n  /**\n   * Initializes the snapshot to the current head of the database. This must be\n   * only be called once. The state of whether a Snapshotter has been initialized\n   * can be determined by calling {@link initialized()}.\n   */\n  init(): this {\n    assert(this.#curr === undefined, 'Already initialized');\n    this.#curr = Snapshot.create(\n      this.#lc,\n      this.#dbFile,\n      this.#appID,\n      this.#pageCacheSizeKib,\n    );\n    this.#lc.debug?.(`Initial snapshot at version ${this.#curr.version}`);\n    return this;\n  }\n\n  initialized(): boolean {\n    return this.#curr !== undefined;\n  }\n\n  /** Returns the current snapshot. Asserts if {@link initialized()} is false. */\n  current(): Snapshot {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    return this.#curr;\n  }\n\n  /**\n   * Advances to the head of the Database, returning a diff between the\n   * previously current Snapshot and a new Snapshot at head. This is called\n   * in response to a notification from a Replicator subscription. Subsequent\n   * calls to {@link current()} return the new Snapshot. Note that the Snapshotter\n   * must be initialized before advancing.\n   *\n   * The returned {@link SnapshotDiff} contains snapshots at the endpoints\n   * of the database timeline. Iterating over the diff generates a sequence\n   * of {@link Change}s between the two snapshots.\n   *\n   * Note that this sequence is not chronological; rather, the sequence is\n   * ordered by `<table, row-key>`, such that a row can appear at most once\n   * in the common case, or twice if its table is `TRUNCATE`'d and a new value\n   * is subsequently `INSERT`'ed. This results in dropping most intermediate\n   * changes to a row and bounds the amount of work needed to catch up;\n   * however, as a consequence, a consistent database state is only guaranteed\n   * when the sequence has been fully consumed.\n   *\n   * Note that Change generation relies on the state of the underlying\n   * database connections, and because the connection for the previous snapshot\n   * is reused to produce the next snapshot, the diff object is only valid\n   * until the next call to `advance()`.\n   *\n   * It is okay for the caller to apply `Change`s to the `prev` snapshot\n   * during the iteration (e.g. this is necessary for IVM); the remainder\n   * of the iteration is not affected because a given row can appear at most\n   * once in the sequence (with the exception being TRUNCATE, after which the\n   * deleted rows can be re-inserted, but this will also behave correctly if\n   * the changes are applied).\n   *\n   * Once the changes have been applied, however, a _subsequent_ iteration\n   * will not produce the correct results. In order to perform multiple\n   * change-applying iterations, the caller must (1) create a save point\n   * on `prev` before each iteration, and (2) rollback to the save point after\n   * the iteration.\n   *\n   * @param prevWrites How the caller writes to `prev` during the iteration,\n   *        which determines which `prev` reads can be shared through the\n   *        {@link SnapshotRowCache}. See {@link PrevWrites}.\n   */\n  advance(\n    syncableTables: Map<string, LiteAndZqlSpec>,\n    allTableNames: Set<string>,\n    observedTables?: TableFilter | undefined,\n    prevWrites: PrevWrites = 'uniform',\n  ): SnapshotDiff {\n    const {prev, curr} = this.advanceWithoutDiff();\n    return new Diff(\n      this.#appID,\n      syncableTables,\n      allTableNames,\n      prev,\n      curr,\n      observedTables,\n      this.#rowCache,\n      prevWrites,\n    );\n  }\n\n  advanceWithoutDiff() {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    const next = this.#prev\n      ? this.#prev.resetToHead()\n      : Snapshot.create(\n          this.#lc,\n          this.#curr.db.db.name,\n          this.#appID,\n          this.#pageCacheSizeKib,\n        );\n    this.#prev = this.#curr;\n    this.#curr = next;\n    return {prev: this.#prev, curr: this.#curr};\n  }\n\n  /**\n   * Call this to close the database connections when the Snapshotter is\n   * no longer needed.\n   */\n  destroy() {\n    this.#curr?.db.db.close();\n    this.#prev?.db.db.close();\n    this.#lc.debug?.('closed database connections');\n  }\n}\n\nexport type Change = {\n  readonly table: string;\n  /**\n   * If this change represents a remove the row to remove,\n   * if nextValue is not null then all rows that have a unique constraint\n   * violation with nextValue.\n   * In both cases these rows should be removed.\n   */\n  readonly prevValues: Readonly<Row>[];\n  readonly nextValue: Readonly<Row> | null;\n  readonly rowKey: RowKey;\n};\n\nexport interface TableFilter {\n  has(table: string): boolean;\n}\n\n/**\n * How the caller of {@link Snapshotter.advance()} writes the changes of a\n * {@link SnapshotDiff} to its `prev` snapshot while iterating over it.\n *\n * A `prev` read for a table with a single unique key (its primary key)\n * cannot observe earlier changes of the iteration, since a row key appears\n * once in the diff. A read for a table with other unique keys can, because\n * the rows it returns may have been displaced or edited by earlier changes.\n *\n * * `uniform`: every caller applies every change, so such a read depends on\n *   the `<prev, curr>` versions of the diff, and is shared on that basis.\n * * `divergent`: callers apply different subsets of the changes (e.g. the\n *   pipeline driver skips changes that no pipeline can observe), so such a\n *   read depends on the caller and is not shared.\n * * `none`: the caller does not write to `prev` (e.g. IVM derivation is\n *   held in memory), so every read depends solely on `prev.version`.\n */\nexport type PrevWrites = 'uniform' | 'divergent' | 'none';\n\n/**\n * Represents the difference between two database Snapshots.\n * Iterating over the object will produce a sequence of {@link Change}s\n * between the two snapshots.\n *\n * See {@link Snapshotter.advance()} for semantics and usage.\n */\nexport interface SnapshotDiff extends Iterable<Change> {\n  readonly prev: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n  readonly curr: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n\n  /**\n   * The number of ChangeLog entries between the snapshots. Note that this\n   * may not necessarily equal the number of `Change` objects that the iteration\n   * will produce, as `TRUNCATE` entries are counted as a single log entry which\n   * may be expanded into many changes (i.e. row deletes).\n   *\n   * TODO: Determine if it is worth changing the definition to count the\n   *       truncated rows. This would make diff computation more expensive\n   *       (requiring the count to be aggregated by operation type), which\n   *       may not be worth it for a presumable rare operation.\n   */\n  readonly changes: number;\n\n  /**\n   * Overrides the `prevWrites` passed to {@link Snapshotter.advance()}, for a\n   * caller that decides how to write to `prev` once it knows the number of\n   * {@link changes}. Must be called before the diff is iterated, except to\n   * switch to `divergent`, which can be done between any two changes: it\n   * shares only the reads that no write to `prev` can affect.\n   */\n  setPrevWrites(prevWrites: PrevWrites): void;\n}\n\n/**\n * Thrown during an iteration of a {@link SnapshotDiff} when a schema\n * change or truncate is encountered, which result in aborting the\n * advancement and resetting / rehydrating the pipelines.\n */\nexport type ResetPipelinesReason =\n  | 'advancement-timeout'\n  | 'scalar-subquery'\n  | 'schema-change'\n  | 'truncation'\n  | 'permissions-change';\n\nexport class ResetPipelinesSignal extends Error {\n  readonly name = 'ResetPipelinesSignal';\n  readonly reason: ResetPipelinesReason;\n\n  constructor(msg: string, reason: ResetPipelinesReason) {\n    super(msg);\n    this.reason = reason;\n  }\n}\n\n// Memoizes, per table spec, the SQL of the row lookups performed for each\n// change, which would otherwise be rebuilt from the table spec for every\n// read. Handing out the same string for every read of a given shape also\n// lets the SnapshotRowCache and the statement cache look it up without\n// rehashing it. Specs are recreated whenever they are recomputed, so stale\n// SQL is never served.\ntype TableReads = {\n  /** `SELECT <columns> FROM <table> WHERE ` */\n  readonly prefix: string;\n  /** {@link Snapshot.getRow} SQL, by the NUL-joined key columns. */\n  readonly byKeyColumns: Map<string, string>;\n  /**\n   * {@link Snapshot.getRows} SQL, by the bitmask of the table's `uniqueKeys`\n   * that are queried.\n   */\n  readonly byUniqueKeys: Map<number, string>;\n};\n\nconst tableReads = new WeakMap<LiteTableSpecWithKeysAndVersion, TableReads>();\n\nfunction readsFor(table: LiteTableSpecWithKeysAndVersion): TableReads {\n  return getOrInsertComputed(tableReads, table, () => ({\n    prefix: `SELECT ${Object.keys(table.columns)\n      .map(c => id(c))\n      .join(',')} FROM ${id(table.name)} WHERE `,\n    byKeyColumns: new Map(),\n    byUniqueKeys: new Map(),\n  }));\n}\n\n// The byUniqueKeys bitmask must fit in a (positive) 31-bit integer.\nconst MAX_MEMOIZED_UNIQUE_KEYS = 30;\n\n/**\n * The index that SQLite creates for the `PRIMARY KEY(\"stateVersion\", \"pos\")`\n * of `_zero.changeLog2` (the first constraint of its CREATE TABLE statement,\n * hence the `_1` suffix).\n *\n * For a `stateVersion > ?` query that reads a column outside of this index\n * (e.g. `op`) and has no ORDER BY, the query planner chooses between a range\n * search of the index and a full scan of the table based on `sqlite_stat1`.\n * Statistics recorded while the change log held only a few rows make the full\n * scan look cheaper, and they persist as the table grows to millions of rows:\n * the change log of a replication-manager's replica only receives entries\n * during column backfills, and view-syncers inherit its statistics when they\n * restore its backup. Such queries therefore pin this index with INDEXED BY,\n * which also makes them fail to prepare (rather than silently scan) if the\n * index is ever renamed.\n */\nconst CHANGE_LOG_STATE_VERSION_INDEX = 'sqlite_autoindex__zero.changeLog2_1';\n\nclass Snapshot {\n  static create(\n    lc: LogContext,\n    dbFile: string,\n    appID: string,\n    pageCacheSizeKib: number | undefined,\n  ) {\n    const conn = new Database(lc, dbFile);\n    conn.pragma('synchronous = OFF'); // Applied changes are ephemeral; COMMIT is never called.\n    if (pageCacheSizeKib !== undefined) {\n      conn.pragma(`cache_size = -${pageCacheSizeKib}`); // Negative = size in KiB\n    }\n    const [{journal_mode: mode}] = conn.pragma('journal_mode') as [\n      {journal_mode: string},\n    ];\n    // BEGIN CONCURRENT lets the Snapshotter simulate changes on a historic\n    // snapshot without taking the WAL writer lock. These private changes are\n    // always rolled back; the replicator is the only committer and can use\n    // BEGIN IMMEDIATE on the same WAL2 database.\n    assert(\n      mode === 'wal2',\n      `replica db must be in wal2 mode (current: ${mode})`,\n    );\n\n    const db = new StatementRunner(conn);\n    return new Snapshot(db, appID);\n  }\n\n  readonly db: StatementRunner;\n  readonly #appID: string;\n  readonly version: string;\n  #reset = false;\n\n  constructor(db: StatementRunner, appID: string) {\n    db.beginConcurrent();\n    // Note: The subsequent read is necessary to acquire the read lock\n    // (which results in the logical creation of the snapshot). Calling\n    // `BEGIN CONCURRENT` alone does not result in acquiring the read lock.\n    const {stateVersion} = getReplicationState(db);\n\n    this.db = db;\n    this.#appID = appID;\n    this.version = stateVersion;\n  }\n\n  numChangesSince(prevVersion: string) {\n    const {count} = this.db.get(\n      'SELECT COUNT(*) AS count FROM \"_zero.changeLog2\" WHERE stateVersion > ?',\n      prevVersion,\n    );\n    return count;\n  }\n\n  /**\n   * Whether a table-wide RESET op (i.e. a schema change) was logged after\n   * `prevVersion`. Table specs computed at or before `prevVersion` are\n   * stale with respect to this snapshot if so.\n   */\n  schemaChangedSince(prevVersion: string): boolean {\n    const row = this.db.get(\n      `SELECT 1 FROM \"_zero.changeLog2\" INDEXED BY \"${CHANGE_LOG_STATE_VERSION_INDEX}\"\n         WHERE stateVersion > ? AND op = ? LIMIT 1`,\n      prevVersion,\n      RESET_OP,\n    );\n    return row !== undefined;\n  }\n\n  /**\n   * Whether any table-wide op (i.e. a RESET or TRUNCATE) was logged after\n   * `prevVersion`. Iterating over a diff ending at this snapshot will\n   * abort with a {@link ResetPipelinesSignal} if so.\n   */\n  hasTableWideOpSince(prevVersion: string): boolean {\n    const row = this.db.get(\n      `SELECT 1 FROM \"_zero.changeLog2\" INDEXED BY \"${CHANGE_LOG_STATE_VERSION_INDEX}\"\n         WHERE stateVersion > ? AND op IN (?, ?) LIMIT 1`,\n      prevVersion,\n      RESET_OP,\n      TRUNCATE_OP,\n    );\n    return row !== undefined;\n  }\n\n  changesSince(prevVersion: string) {\n    // Note: The queried fields are constrained to only those that are relevant\n    // to the snapshot diff, i.e. those defined in the changeLogEntrySchema.\n    const cached = this.db.statementCache.get(\n      `SELECT \"stateVersion\", \"table\", \"rowKey\", \"op\" FROM \"_zero.changeLog2\"\n         WHERE \"stateVersion\" > ? ORDER BY \"stateVersion\" ASC, \"pos\" ASC`,\n    );\n    return {\n      changes: cached.statement.iterate(prevVersion),\n      cleanup: () => this.db.statementCache.return(cached),\n    };\n  }\n\n  /**\n   * Reads the row with the given `rowKey`, or `undefined` if it does not\n   * exist.\n   *\n   * @param cache An optional {@link SnapshotRowCache} through which to\n   *        perform the read, along with the `tag` that identifies the\n   *        version(s) of the replica that the result depends on.\n   */\n  getRow(\n    table: LiteTableSpecWithKeysAndVersion,\n    rowKey: JSONValue,\n    cache?: SnapshotRowCache,\n    tag?: string,\n  ) {\n    const key = normalizedKeyOrder(rowKey as RowKey);\n    const cols = Object.keys(key);\n    const reads = readsFor(table);\n    const sql = getOrInsertComputed(\n      reads.byKeyColumns,\n      cols.join('\\0'),\n      () => reads.prefix + cols.map(c => `${id(c)}=?`).join(' AND '),\n    );\n    const args = Object.values(key);\n    const read = () =>\n      this.db.statementCache.use(sql, cached => {\n        cached.statement.safeIntegers(true);\n        // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n        return cached.statement.get<any>(args);\n      });\n    return cache\n      ? cache.getOrRead(tag ?? '', sql, 'get', args, () => {\n          const row = read();\n          if (row !== undefined) {\n            this.#assertNotAdvanced(row);\n          }\n          return row;\n        })\n      : read();\n  }\n\n  /**\n   * Reads the rows that conflict with `row` on any of the given unique\n   * `keys` (which includes the row with the same primary key, if any).\n   *\n   * @param cache An optional {@link SnapshotRowCache} through which to\n   *        perform the read, along with the `tag` that identifies the\n   *        version(s) of the replica that the result depends on.\n   */\n  getRows(\n    table: LiteTableSpecWithKeysAndVersion,\n    keys: PrimaryKey[],\n    row: RowValue,\n    cache?: SnapshotRowCache,\n    tag?: string,\n  ) {\n    // Filter out keys where any column is NULL. This is both correct and\n    // critical for performance:\n    // 1. Correctness: NULL values can't violate uniqueness (NULL != NULL in SQL)\n    // 2. Performance: SQLite's MULTI-INDEX OR optimization completely fails when\n    //    any branch involves NULL, falling back to a full table scan. This was\n    //    causing slowdowns of hundreds of times on tables with nullable unique columns.\n    const validKeys: PrimaryKey[] = [];\n    let mask = 0;\n    for (let i = 0; i < keys.length; i++) {\n      const key = keys[i];\n      if (\n        key.every(column => row[column] !== null && row[column] !== undefined)\n      ) {\n        validKeys.push(key);\n        mask |= 1 << i;\n      }\n    }\n    if (validKeys.length === 0) {\n      return [];\n    }\n    const reads = readsFor(table);\n    const buildSQL = () =>\n      reads.prefix +\n      validKeys\n        .map(key => key.map(c => `${id(c)}=?`).join(' AND '))\n        .join(' OR ');\n    const sql =\n      keys === table.uniqueKeys && keys.length <= MAX_MEMOIZED_UNIQUE_KEYS\n        ? getOrInsertComputed(reads.byUniqueKeys, mask, buildSQL)\n        : buildSQL();\n    const args = validKeys.flatMap(key => key.map(column => row[column]));\n    const read = () =>\n      this.db.statementCache.use(sql, cached => {\n        cached.statement.safeIntegers(true);\n        // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n        return cached.statement.all<any>(args);\n      });\n    return cache\n      ? cache.getOrRead(tag ?? '', sql, 'all', args, () => {\n          const rows = read();\n          for (const row of rows) {\n            this.#assertNotAdvanced(row);\n          }\n          return rows;\n        })\n      : read();\n  }\n\n  /**\n   * Rows are shared with other client groups via the SnapshotRowCache, so\n   * one read from a connection that has advanced past this snapshot (i.e. a\n   * {@link SnapshotDiff} used after its Snapshotter advanced) must fail\n   * before it is cached, rather than be served to the valid Diffs of other\n   * client groups (whose own validity checks would then fail).\n   */\n  #assertNotAdvanced(row: RowValue) {\n    if ((row[ROW_VERSION] ?? '~') > this.version) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. db has advanced past ${this.version}.`,\n      );\n    }\n  }\n\n  /**\n   * Whether the connection has been reset to a newer snapshot (i.e. via\n   * {@link resetToHead()}), after which reads no longer reflect\n   * {@link version}.\n   */\n  get reset(): boolean {\n    return this.#reset;\n  }\n\n  resetToHead(): Snapshot {\n    this.#reset = true;\n    this.db.rollback();\n    return new Snapshot(this.db, this.#appID);\n  }\n}\n\nclass Diff implements SnapshotDiff {\n  readonly #permissionsTable: string;\n  readonly #syncableTables: Map<string, LiteAndZqlSpec>;\n  readonly #allTableNames: Set<string>;\n  readonly #observedTables: TableFilter | undefined;\n  readonly #rowCache: SnapshotRowCache | undefined;\n  #prevWrites: PrevWrites;\n  #iterated = false;\n  readonly prev: Snapshot;\n  readonly curr: Snapshot;\n  readonly changes: number;\n\n  constructor(\n    appID: string,\n    syncableTables: Map<string, LiteAndZqlSpec>,\n    allTableNames: Set<string>,\n    prev: Snapshot,\n    curr: Snapshot,\n    observedTables?: TableFilter | undefined,\n    rowCache?: SnapshotRowCache | undefined,\n    prevWrites: PrevWrites = 'uniform',\n  ) {\n    this.#prevWrites = prevWrites;\n    this.#permissionsTable = `${appID}.permissions`;\n    this.#syncableTables = syncableTables;\n    this.#allTableNames = allTableNames;\n    this.#observedTables = observedTables;\n    this.prev = prev;\n    this.curr = curr;\n    this.changes = curr.numChangesSince(prev.version);\n    // Row values are only shared (via the cache) between Diffs that do not\n    // straddle a table-wide op. This guarantees that any two Diffs that\n    // encounter the same change log entry read the row from tables with\n    // the same schema (and contents), since a RESET or TRUNCATE between\n    // their `curr` versions would necessarily fall within the range of the\n    // later Diff. (Such a Diff aborts with a ResetPipelinesSignal anyway.)\n    this.#rowCache =\n      rowCache && !curr.hasTableWideOpSince(prev.version)\n        ? rowCache\n        : undefined;\n  }\n\n  setPrevWrites(prevWrites: PrevWrites): void {\n    assert(\n      !this.#iterated || prevWrites === 'divergent',\n      'prevWrites must be set before iterating the diff',\n    );\n    this.#prevWrites = prevWrites;\n  }\n\n  [Symbol.iterator](): Iterator<Change> {\n    this.#iterated = true;\n    const {changes, cleanup: done} = this.curr.changesSince(this.prev.version);\n\n    const cleanup = () => {\n      try {\n        // Allow open iterators to clean up their state.\n        changes.return?.(undefined);\n      } finally {\n        done();\n      }\n    };\n\n    return {\n      next: () => {\n        try {\n          for (;;) {\n            const {value, done} = changes.next();\n            if (done) {\n              cleanup();\n              return {value, done: true};\n            }\n\n            const {table, rowKey, op, stateVersion} = v.parse(value, schema);\n            if (op === RESET_OP) {\n              // The current map of `TableSpec`s may not have the correct or complete information.\n              throw new ResetPipelinesSignal(\n                `schema for table ${table} has changed`,\n                'schema-change',\n              );\n            }\n            if (op === TRUNCATE_OP) {\n              // Truncates are also processed by rehydrating pipelines at current.\n              throw new ResetPipelinesSignal(\n                `table ${table} has been truncated`,\n                'truncation',\n              );\n            }\n            if (\n              this.#observedTables &&\n              !this.#observedTables.has(table) &&\n              table !== this.#permissionsTable\n            ) {\n              if (this.#allTableNames.has(table)) {\n                continue; // skip change log entries for unobserved tables.\n              }\n              throw new Error(`change for unknown table ${table}`);\n            }\n            const specs = this.#syncableTables.get(table);\n            if (!specs) {\n              if (this.#allTableNames.has(table)) {\n                continue; // skip change log entries for non-syncable tables.\n              }\n              throw new Error(`change for unknown table ${table}`);\n            }\n            const {tableSpec, zqlSpec} = specs;\n\n            // Sanity check: All change log ops should have a stateVersion\n            // greater than minRowVersion in the table metadata. This is a\n            // mini-proof that the overlay does not need to be applied to\n            // rows produced in incremental catchup, based on the invariant in\n            // change-processor's #bumpVersions(), whereby the setting of the\n            // minRowVersion is always followed by a RESET OP, meaning that\n            // subsequent change-log traversal happens at a later version.\n            assert(\n              (tableSpec.minRowVersion ?? '') < stateVersion,\n              () =>\n                `unexpected change @${stateVersion} for table ${table} with ` +\n                `minRowVersion ${tableSpec.minRowVersion}: ${op}(${rowKey})`,\n            );\n\n            assert(rowKey !== null, 'rowKey must be present for row changes');\n\n            // Row reads are shared with the other client groups on the\n            // worker via the SnapshotRowCache (if configured):\n            //\n            // * The new value of the row is read from `curr`, which is\n            //   never written to during the iteration, and is the version\n            //   of the row at `stateVersion` (as verified by\n            //   checkThatDiffIsValid()). It is thus identical for every\n            //   Diff that encounters the change log entry.\n            //\n            // * Previous values are read from `prev`, at `prev.version`,\n            //   which the caller (i.e. IVM) writes to as the iteration\n            //   progresses. For a table whose only unique key is its\n            //   primary key, no change applied before this one can affect\n            //   this row, so the result depends solely on `prev.version`.\n            //   Otherwise, the result can include rows that conflict on\n            //   other unique keys, which may have been added or removed by\n            //   preceding changes; see PrevWrites for when that can be\n            //   shared.\n            // The first advance() after this Diff was created resets the\n            // `prev` connection to head. Reads would then no longer reflect\n            // `prev.version`, and cache hits would not reveal it (the cached\n            // rows were read by valid Diffs), so this must be checked\n            // explicitly rather than inferred from the rows' versions.\n            if (this.prev.reset) {\n              throw new InvalidDiffError(\n                `Diff is no longer valid. prev db has advanced past ${this.prev.version}.`,\n              );\n            }\n\n            const cache = this.#rowCache;\n            const prevDependsOnWrites =\n              tableSpec.uniqueKeys.length > 1 && this.#prevWrites !== 'none';\n            const prevCache =\n              prevDependsOnWrites && this.#prevWrites === 'divergent'\n                ? undefined\n                : cache;\n            const prevTag = prevDependsOnWrites\n              ? `p:${this.prev.version}:${this.curr.version}`\n              : `p:${this.prev.version}`;\n            const nextValue =\n              op === SET_OP\n                ? this.curr.getRow(\n                    tableSpec,\n                    rowKey,\n                    cache,\n                    `n:${stateVersion}`,\n                  )\n                : null;\n            let prevValues;\n            if (nextValue) {\n              prevValues = this.prev.getRows(\n                tableSpec,\n                tableSpec.uniqueKeys,\n                nextValue,\n                prevCache,\n                prevTag,\n              );\n            } else {\n              const prevValue = this.prev.getRow(\n                tableSpec,\n                rowKey,\n                prevCache,\n                prevTag,\n              );\n              prevValues = prevValue ? [prevValue] : [];\n            }\n            if (nextValue === undefined) {\n              throw new Error(\n                `Missing value for ${table} ${stringify(rowKey)}`,\n              );\n            }\n            // Sanity check detects if the diff is being accessed after the Snapshots have advanced.\n            this.checkThatDiffIsValid(stateVersion, op, prevValues, nextValue);\n\n            if (prevValues.length === 0 && nextValue === null) {\n              // Filter out no-op changes (e.g. a delete of a row that does not exist in prev).\n              // TODO: Consider doing this for deep-equal values.\n              continue;\n            }\n\n            if (\n              table === this.#permissionsTable &&\n              prevValues.some(\n                prevValue => prevValue.permissions !== nextValue.permissions,\n              )\n            ) {\n              throw new ResetPipelinesSignal(\n                `Permissions have changed ${\n                  prevValues.find(\n                    prevValue =>\n                      prevValue.permissions !== nextValue.permissions,\n                  ).hash\n                } => ${nextValue.hash}`,\n                'permissions-change',\n              );\n            }\n\n            // Note: The raw SQLite rows may be shared with other client\n            // groups via the SnapshotRowCache, so they must not be modified.\n            // fromSQLiteTypes() produces new row objects.\n            // TODO Can we get rid of these RowValue casts?\n            return {\n              value: {\n                table,\n                prevValues: prevValues.map(prevValue =>\n                  fromSQLiteTypes(zqlSpec, prevValue, table),\n                ),\n                nextValue: nextValue\n                  ? fromSQLiteTypes(zqlSpec, nextValue, table)\n                  : null,\n                rowKey,\n              } satisfies Change,\n            };\n          }\n        } catch (e) {\n          // This control flow path is not covered by the return() method (i.e. `break`).\n          cleanup();\n          throw e;\n        }\n      },\n\n      return: (value: unknown) => {\n        cleanup();\n        return {value, done: true};\n      },\n    };\n  }\n\n  checkThatDiffIsValid(\n    stateVersion: string,\n    op: string,\n    prevValues: RowValue[],\n    nextValue: RowValue,\n  ) {\n    // Sanity checks to detect that the diff is not being accessed after\n    // the Snapshots have advanced.\n    if (stateVersion > this.curr.version) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. curr db has advanced past ${this.curr.version}`,\n      );\n    }\n    if (\n      prevValues.findIndex(\n        prevValue => (prevValue[ROW_VERSION] ?? '~') > this.prev.version,\n      ) !== -1\n    ) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. prev db has advanced past ${this.prev.version}.`,\n      );\n    }\n    if (op === SET_OP && nextValue[ROW_VERSION] !== stateVersion) {\n      throw new InvalidDiffError(\n        'Diff is no longer valid. curr db has advanced.',\n      );\n    }\n  }\n}\n\nexport class InvalidDiffError extends Error {\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4FA,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CACA;CACA;CACA;CACA;;;;;;;CAQA,YACE,IACA,QACA,EAAC,SACD,kBACA,UACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,SAAS;EACd,KAAKC,oBAAoB;EACzB,KAAKC,YAAY;CACnB;;;;;;CAOA,OAAa;EACX,OAAO,KAAKC,UAAU,KAAA,GAAW,qBAAqB;EACtD,KAAKA,QAAQ,SAAS,OACpB,KAAKL,KACL,KAAKC,SACL,KAAKC,QACL,KAAKC,iBACP;EACA,KAAKH,IAAI,QAAQ,+BAA+B,KAAKK,MAAM,SAAS;EACpE,OAAO;CACT;CAEA,cAAuB;EACrB,OAAO,KAAKA,UAAU,KAAA;CACxB;;CAGA,UAAoB;EAClB,OAAO,KAAKA,UAAU,KAAA,GAAW,sCAAsC;EACvE,OAAO,KAAKA;CACd;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2CA,QACE,gBACA,eACA,gBACA,aAAyB,WACX;EACd,MAAM,EAAC,MAAM,SAAQ,KAAK,mBAAmB;EAC7C,OAAO,IAAI,KACT,KAAKH,QACL,gBACA,eACA,MACA,MACA,gBACA,KAAKE,WACL,UACF;CACF;CAEA,qBAAqB;EACnB,OAAO,KAAKC,UAAU,KAAA,GAAW,sCAAsC;EACvE,MAAM,OAAO,KAAKC,QACd,KAAKA,MAAM,YAAY,IACvB,SAAS,OACP,KAAKN,KACL,KAAKK,MAAM,GAAG,GAAG,MACjB,KAAKH,QACL,KAAKC,iBACP;EACJ,KAAKG,QAAQ,KAAKD;EAClB,KAAKA,QAAQ;EACb,OAAO;GAAC,MAAM,KAAKC;GAAO,MAAM,KAAKD;EAAK;CAC5C;;;;;CAMA,UAAU;EACR,KAAKA,OAAO,GAAG,GAAG,MAAM;EACxB,KAAKC,OAAO,GAAG,GAAG,MAAM;EACxB,KAAKN,IAAI,QAAQ,6BAA6B;CAChD;AACF;AA0FA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAgB;CAChB;CAEA,YAAY,KAAa,QAA8B;EACrD,MAAM,GAAG;EACT,KAAK,SAAS;CAChB;AACF;AAoBA,IAAM,6BAAa,IAAI,QAAqD;AAE5E,SAAS,SAAS,OAAoD;CACpE,OAAO,oBAAoB,YAAY,cAAc;EACnD,QAAQ,UAAU,OAAO,KAAK,MAAM,OAAO,EACxC,KAAI,MAAK,GAAG,CAAC,CAAC,EACd,KAAK,GAAG,EAAE,QAAQ,GAAG,MAAM,IAAI,EAAE;EACpC,8BAAc,IAAI,IAAI;EACtB,8BAAc,IAAI,IAAI;CACxB,EAAE;AACJ;AAGA,IAAM,2BAA2B;;;;;;;;;;;;;;;;;AAkBjC,IAAM,iCAAiC;AAEvC,IAAM,WAAN,MAAM,SAAS;CACb,OAAO,OACL,IACA,QACA,OACA,kBACA;EACA,MAAM,OAAO,IAAI,SAAS,IAAI,MAAM;EACpC,KAAK,OAAO,mBAAmB;EAC/B,IAAI,qBAAqB,KAAA,GACvB,KAAK,OAAO,iBAAiB,kBAAkB;EAEjD,MAAM,CAAC,EAAC,cAAc,UAAS,KAAK,OAAO,cAAc;EAOzD,OACE,SAAS,QACT,6CAA6C,KAAK,EACpD;EAGA,OAAO,IAAI,SAAS,IADL,gBAAgB,IACX,GAAI,KAAK;CAC/B;CAEA;CACA;CACA;CACA,SAAS;CAET,YAAY,IAAqB,OAAe;EAC9C,GAAG,gBAAgB;EAInB,MAAM,EAAC,iBAAgB,oBAAoB,EAAE;EAE7C,KAAK,KAAK;EACV,KAAKE,SAAS;EACd,KAAK,UAAU;CACjB;CAEA,gBAAgB,aAAqB;EACnC,MAAM,EAAC,UAAS,KAAK,GAAG,IACtB,6EACA,WACF;EACA,OAAO;CACT;;;;;;CAOA,mBAAmB,aAA8B;EAO/C,OANY,KAAK,GAAG,IAClB,gDAAgD,+BAA+B;qDAE/E,aAAA,GAGK,MAAQ,KAAA;CACjB;;;;;;CAOA,oBAAoB,aAA8B;EAQhD,OAPY,KAAK,GAAG,IAClB,gDAAgD,+BAA+B;2DAE/E,aAAA,KAAA,GAIK,MAAQ,KAAA;CACjB;CAEA,aAAa,aAAqB;EAGhC,MAAM,SAAS,KAAK,GAAG,eAAe,IACpC;yEAEF;EACA,OAAO;GACL,SAAS,OAAO,UAAU,QAAQ,WAAW;GAC7C,eAAe,KAAK,GAAG,eAAe,OAAO,MAAM;EACrD;CACF;;;;;;;;;CAUA,OACE,OACA,QACA,OACA,KACA;EACA,MAAM,MAAM,mBAAmB,MAAgB;EAC/C,MAAM,OAAO,OAAO,KAAK,GAAG;EAC5B,MAAM,QAAQ,SAAS,KAAK;EAC5B,MAAM,MAAM,oBACV,MAAM,cACN,KAAK,KAAK,IAAI,SACR,MAAM,SAAS,KAAK,KAAI,MAAK,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAC/D;EACA,MAAM,OAAO,OAAO,OAAO,GAAG;EAC9B,MAAM,aACJ,KAAK,GAAG,eAAe,IAAI,MAAK,WAAU;GACxC,OAAO,UAAU,aAAa,IAAI;GAElC,OAAO,OAAO,UAAU,IAAS,IAAI;EACvC,CAAC;EACH,OAAO,QACH,MAAM,UAAU,OAAO,IAAI,KAAK,OAAO,YAAY;GACjD,MAAM,MAAM,KAAK;GACjB,IAAI,QAAQ,KAAA,GACV,KAAKK,mBAAmB,GAAG;GAE7B,OAAO;EACT,CAAC,IACD,KAAK;CACX;;;;;;;;;CAUA,QACE,OACA,MACA,KACA,OACA,KACA;EAOA,MAAM,YAA0B,CAAC;EACjC,IAAI,OAAO;EACX,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;GACpC,MAAM,MAAM,KAAK;GACjB,IACE,IAAI,OAAM,WAAU,IAAI,YAAY,QAAQ,IAAI,YAAY,KAAA,CAAS,GACrE;IACA,UAAU,KAAK,GAAG;IAClB,QAAQ,KAAK;GACf;EACF;EACA,IAAI,UAAU,WAAW,GACvB,OAAO,CAAC;EAEV,MAAM,QAAQ,SAAS,KAAK;EAC5B,MAAM,iBACJ,MAAM,SACN,UACG,KAAI,QAAO,IAAI,KAAI,MAAK,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,EACnD,KAAK,MAAM;EAChB,MAAM,MACJ,SAAS,MAAM,cAAc,KAAK,UAAU,2BACxC,oBAAoB,MAAM,cAAc,MAAM,QAAQ,IACtD,SAAS;EACf,MAAM,OAAO,UAAU,SAAQ,QAAO,IAAI,KAAI,WAAU,IAAI,OAAO,CAAC;EACpE,MAAM,aACJ,KAAK,GAAG,eAAe,IAAI,MAAK,WAAU;GACxC,OAAO,UAAU,aAAa,IAAI;GAElC,OAAO,OAAO,UAAU,IAAS,IAAI;EACvC,CAAC;EACH,OAAO,QACH,MAAM,UAAU,OAAO,IAAI,KAAK,OAAO,YAAY;GACjD,MAAM,OAAO,KAAK;GAClB,KAAK,MAAM,OAAO,MAChB,KAAKA,mBAAmB,GAAG;GAE7B,OAAO;EACT,CAAC,IACD,KAAK;CACX;;;;;;;;CASA,mBAAmB,KAAe;EAChC,KAAK,IAAA,iBAAoB,OAAO,KAAK,SACnC,MAAM,IAAI,iBACR,iDAAiD,KAAK,QAAQ,EAChE;CAEJ;;;;;;CAOA,IAAI,QAAiB;EACnB,OAAO,KAAKC;CACd;CAEA,cAAwB;EACtB,KAAKA,SAAS;EACd,KAAK,GAAG,SAAS;EACjB,OAAO,IAAI,SAAS,KAAK,IAAI,KAAKN,MAAM;CAC1C;AACF;AAEA,IAAM,OAAN,MAAmC;CACjC;CACA;CACA;CACA;CACA;CACA;CACA,YAAY;CACZ;CACA;CACA;CAEA,YACE,OACA,gBACA,eACA,MACA,MACA,gBACA,UACA,aAAyB,WACzB;EACA,KAAKW,cAAc;EACnB,KAAKJ,oBAAoB,GAAG,MAAM;EAClC,KAAKC,kBAAkB;EACvB,KAAKC,iBAAiB;EACtB,KAAKC,kBAAkB;EACvB,KAAK,OAAO;EACZ,KAAK,OAAO;EACZ,KAAK,UAAU,KAAK,gBAAgB,KAAK,OAAO;EAOhD,KAAKR,YACH,YAAY,CAAC,KAAK,oBAAoB,KAAK,OAAO,IAC9C,WACA,KAAA;CACR;CAEA,cAAc,YAA8B;EAC1C,OACE,CAAC,KAAKU,aAAa,eAAe,aAClC,kDACF;EACA,KAAKD,cAAc;CACrB;CAEA,CAAC,OAAO,YAA8B;EACpC,KAAKC,YAAY;EACjB,MAAM,EAAC,SAAS,SAAS,SAAQ,KAAK,KAAK,aAAa,KAAK,KAAK,OAAO;EAEzE,MAAM,gBAAgB;GACpB,IAAI;IAEF,QAAQ,SAAS,KAAA,CAAS;GAC5B,UAAU;IACR,KAAK;GACP;EACF;EAEA,OAAO;GACL,YAAY;IACV,IAAI;KACF,SAAS;MACP,MAAM,EAAC,OAAO,SAAQ,QAAQ,KAAK;MACnC,IAAI,MAAM;OACR,QAAQ;OACR,OAAO;QAAC;QAAO,MAAM;OAAI;MAC3B;MAEA,MAAM,EAAC,OAAO,QAAQ,IAAI,iBAAgB,MAAQ,OAAO,oBAAM;MAC/D,IAAI,OAAA,KAEF,MAAM,IAAI,qBACR,oBAAoB,MAAM,eAC1B,eACF;MAEF,IAAI,OAAA,KAEF,MAAM,IAAI,qBACR,SAAS,MAAM,sBACf,YACF;MAEF,IACE,KAAKF,mBACL,CAAC,KAAKA,gBAAgB,IAAI,KAAK,KAC/B,UAAU,KAAKH,mBACf;OACA,IAAI,KAAKE,eAAe,IAAI,KAAK,GAC/B;OAEF,MAAM,IAAI,MAAM,4BAA4B,OAAO;MACrD;MACA,MAAM,QAAQ,KAAKD,gBAAgB,IAAI,KAAK;MAC5C,IAAI,CAAC,OAAO;OACV,IAAI,KAAKC,eAAe,IAAI,KAAK,GAC/B;OAEF,MAAM,IAAI,MAAM,4BAA4B,OAAO;MACrD;MACA,MAAM,EAAC,WAAW,YAAW;MAS7B,QACG,UAAU,iBAAiB,MAAM,oBAEhC,sBAAsB,aAAa,aAAa,MAAM,sBACrC,UAAU,cAAc,IAAI,GAAG,GAAG,OAAO,EAC9D;MAEA,OAAO,WAAW,MAAM,wCAAwC;MAyBhE,IAAI,KAAK,KAAK,OACZ,MAAM,IAAI,iBACR,sDAAsD,KAAK,KAAK,QAAQ,EAC1E;MAGF,MAAM,QAAQ,KAAKP;MACnB,MAAM,sBACJ,UAAU,WAAW,SAAS,KAAK,KAAKS,gBAAgB;MAC1D,MAAM,YACJ,uBAAuB,KAAKA,gBAAgB,cACxC,KAAA,IACA;MACN,MAAM,UAAU,sBACZ,KAAK,KAAK,KAAK,QAAQ,GAAG,KAAK,KAAK,YACpC,KAAK,KAAK,KAAK;MACnB,MAAM,YACJ,OAAA,MACI,KAAK,KAAK,OACR,WACA,QACA,OACA,KAAK,cACP,IACA;MACN,IAAI;MACJ,IAAI,WACF,aAAa,KAAK,KAAK,QACrB,WACA,UAAU,YACV,WACA,WACA,OACF;WACK;OACL,MAAM,YAAY,KAAK,KAAK,OAC1B,WACA,QACA,WACA,OACF;OACA,aAAa,YAAY,CAAC,SAAS,IAAI,CAAC;MAC1C;MACA,IAAI,cAAc,KAAA,GAChB,MAAM,IAAI,MACR,qBAAqB,MAAM,GAAG,UAAU,MAAM,GAChD;MAGF,KAAK,qBAAqB,cAAc,IAAI,YAAY,SAAS;MAEjE,IAAI,WAAW,WAAW,KAAK,cAAc,MAG3C;MAGF,IACE,UAAU,KAAKJ,qBACf,WAAW,MACT,cAAa,UAAU,gBAAgB,UAAU,WACnD,GAEA,MAAM,IAAI,qBACR,4BACE,WAAW,MACT,cACE,UAAU,gBAAgB,UAAU,WACxC,EAAE,KACH,MAAM,UAAU,QACjB,oBACF;MAOF,OAAO,EACL,OAAO;OACL;OACA,YAAY,WAAW,KAAI,cACzB,gBAAgB,SAAS,WAAW,KAAK,CAC3C;OACA,WAAW,YACP,gBAAgB,SAAS,WAAW,KAAK,IACzC;OACJ;MACF,EACF;KACF;IACF,SAAS,GAAG;KAEV,QAAQ;KACR,MAAM;IACR;GACF;GAEA,SAAS,UAAmB;IAC1B,QAAQ;IACR,OAAO;KAAC;KAAO,MAAM;IAAI;GAC3B;EACF;CACF;CAEA,qBACE,cACA,IACA,YACA,WACA;EAGA,IAAI,eAAe,KAAK,KAAK,SAC3B,MAAM,IAAI,iBACR,sDAAsD,KAAK,KAAK,SAClE;EAEF,IACE,WAAW,WACT,eAAc,UAAA,iBAA0B,OAAO,KAAK,KAAK,OAC3D,MAAM,IAEN,MAAM,IAAI,iBACR,sDAAsD,KAAK,KAAK,QAAQ,EAC1E;EAEF,IAAI,OAAA,OAAiB,UAAA,kBAA2B,cAC9C,MAAM,IAAI,iBACR,gDACF;CAEJ;AACF;AAEA,IAAa,mBAAb,cAAsC,MAAM;CAC1C,YAAY,KAAa;EACvB,MAAM,GAAG;CACX;AACF"}

package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts

@@ -18,7 +18,10 @@ import type { ReplicaState } from '../replicator/replicator.ts';
 import type { ActivityBasedService } from '../service.ts';
 import type { ConnectionContextManager, ConnectionSelector } from './connection-context-manager.ts';
 import type { DrainCoordinator } from './drain-coordinator.ts';
+import { type MonotonicClock } from './hydration-budget.ts';
 import type { PipelineDriver } from './pipeline-driver.ts';
+import { type ClientQueryRecord, type CustomQueryRecord, type InternalQueryRecord } from './schema/types.ts';
+import { type TTLClock } from './ttl-clock.ts';
 export interface ViewSyncer {
     initConnection(selector: ConnectionSelector, initConnectionMessage: InitConnectionMessage): Source<ViewSyncerDownstream>;
     changeDesiredQueries(selector: ConnectionSelector, msg: ChangeDesiredQueriesMessage): Promise<void>;
@@ -67,7 +70,7 @@ export declare class ViewSyncerService implements ViewSyncer, ActivityBasedServi
     readonly id: string;
     readonly createdAtMs: number;
     readonly connContextManager: ConnectionContextManager;
-    constructor(config: NormalizedZeroConfig, lc: LogContext, shard: ShardID, taskID: string, clientGroupID: string, cvrDb: PostgresDB, pipelineDriver: PipelineDriver, versionChanges: Subscription<ReplicaState>, drainCoordinator: DrainCoordinator, slowHydrateThreshold: number, inspectorDelegate: InspectorDelegate, connContextManager: ConnectionContextManager, customQueryTransformer: CustomQueryTransformer | undefined, runPriorityOp: <T>(lc: LogContext, description: string, op: () => Promise<T>) => Promise<T>, keepaliveMs?: number, setTimeoutFn?: SetTimeout);
+    constructor(config: NormalizedZeroConfig, lc: LogContext, shard: ShardID, taskID: string, clientGroupID: string, cvrDb: PostgresDB, pipelineDriver: PipelineDriver, versionChanges: Subscription<ReplicaState>, drainCoordinator: DrainCoordinator, slowHydrateThreshold: number, inspectorDelegate: InspectorDelegate, connContextManager: ConnectionContextManager, customQueryTransformer: CustomQueryTransformer | undefined, runPriorityOp: <T>(lc: LogContext, description: string, op: () => Promise<T>) => Promise<T>, keepaliveMs?: number, setTimeoutFn?: SetTimeout, now?: MonotonicClock);
     readyState(): Promise<'initialized' | 'draining'>;
     run(): Promise<void>;
     get queryCount(): number;
@@ -98,6 +101,13 @@ export declare class ViewSyncerService implements ViewSyncer, ActivityBasedServi
      */
     markInitialized(): void;
 }
+/**
+ * Whether a query's TTL has elapsed. A query must be expired for all clients
+ * in order to be considered expired.
+ *
+ * Exported for testing.
+ */
+export declare function expired(ttlClock: TTLClock, q: InternalQueryRecord | ClientQueryRecord | CustomQueryRecord): boolean;
 export declare class TimeSliceTimer {
     #private;
     constructor(lc: LogContext);