@rocicorp/zero 0.26.1-canary.9 → 0.26.2-canary.1

package/out/zero-cache/src/services/view-syncer/snapshotter.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshotter.js","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {stringify, type JSONValue} from '../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Row} from '../../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../../zero-types/src/schema.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {fromSQLiteTypes} from '../../../../zqlite/src/table-source.ts';\nimport type {LiteAndZqlSpec, LiteTableSpecWithKeys} from '../../db/specs.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {\n  normalizedKeyOrder,\n  type RowKey,\n  type RowValue,\n} from '../../types/row-key.ts';\nimport type {AppID} from '../../types/shards.ts';\nimport {id} from '../../types/sql.ts';\nimport {\n  RESET_OP,\n  changeLogEntrySchema as schema,\n  SET_OP,\n  TRUNCATE_OP,\n} from '../replicator/schema/change-log.ts';\nimport {\n  getReplicationState,\n  ZERO_VERSION_COLUMN_NAME as ROW_VERSION,\n} from '../replicator/schema/replication-state.ts';\n\n/**\n * A `Snapshotter` manages the progression of database snapshots for a\n * ViewSyncer.\n *\n * The Replicator and ViewSyncers operate on the same SQLite file, with the\n * Replicator being the sole writer to the database. The IVM logic in\n * ViewSyncers, however, rely on incrementally applying changes to the DB to\n * update the state of its pipelines.\n *\n * To avoid coupling the progress of the Replicator and all IVM pipelines on\n * each other, ViewSyncers operate on ephemeral forks of the database by holding\n * [concurrent](https://sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md)\n * snapshots of the database and simulating (but ultimately rolling back)\n * mutations on these snapshots.\n *\n * Example:\n * 1. ViewSyncer takes `snapshot_a` at version `t1` of the database and\n *    hydrates its pipeline(s).\n * 2. Replicator applies a new transaction to the database and notifies\n *    subscribers.\n * 3. ViewSyncer takes `snapshot_b` at `t2`, and queries the `ChangeLog` at\n *    that snapshot for changes since `t1`.\n * 4. ViewSyncer applies those changes to `snapshot_a` for IVM, but does not\n *    commit them. (Recall that the Replicator is the sole writer to the db, so\n *    the ViewSyncer never commits any writes.)\n * 5. Replicator applies the next transaction and advances the database to `t3`.\n * 6. ViewSyncer rolls back `snapshot_a` and opens `snapshot_c` at `t3`, using\n *    `snapshot_b` to simulate changes from `t2` to `t3`.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ```\n *\n * Note that the Replicator (and ViewSyncers) do not wait on the progress of\n * other ViewSyncers. If a ViewSyncer is busy hydrating at `t1`, the Replicator\n * and other ViewSyncers can progress through `t2`, `t3`, etc. independently,\n * as the busy ViewSyncer simply takes its own snapshot when it is ready.\n *\n * ```\n * Replicator:   t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer1:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ViewSyncer2:       [.......... snapshot_a ..........] ----> [snapshot_b]\n * ```\n *\n * To minimize Database connections (and statement preparation, etc.), the\n * Snapshotter reuses the connection from the previous (rolled back)\n * snapshot when opening the new one.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n *                     (conn_1)           (conn_2)           (conn_1)\n * ```\n *\n * In this manner, each ViewSyncer uses two connections that continually\n * \"leapfrog\" each other to replay the timeline of changes in isolation from\n * the Replicator and other ViewSyncers.\n */\nexport class Snapshotter {\n  readonly #lc: LogContext;\n  readonly #dbFile: string;\n  readonly #appID: string;\n  readonly #pageCacheSizeKib: number | undefined;\n  #curr: Snapshot | undefined;\n  #prev: Snapshot | undefined;\n\n  constructor(\n    lc: LogContext,\n    dbFile: string,\n    {appID}: AppID,\n    pageCacheSizeKib?: number | undefined,\n  ) {\n    this.#lc = lc;\n    this.#dbFile = dbFile;\n    this.#appID = appID;\n    this.#pageCacheSizeKib = pageCacheSizeKib;\n  }\n\n  /**\n   * Initializes the snapshot to the current head of the database. This must be\n   * only be called once. The state of whether a Snapshotter has been initialized\n   * can be determined by calling {@link initialized()}.\n   */\n  init(): this {\n    assert(this.#curr === undefined, 'Already initialized');\n    this.#curr = Snapshot.create(\n      this.#lc,\n      this.#dbFile,\n      this.#appID,\n      this.#pageCacheSizeKib,\n    );\n    this.#lc.debug?.(`Initial snapshot at version ${this.#curr.version}`);\n    return this;\n  }\n\n  initialized(): boolean {\n    return this.#curr !== undefined;\n  }\n\n  /** Returns the current snapshot. Asserts if {@link initialized()} is false. */\n  current(): Snapshot {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    return this.#curr;\n  }\n\n  /**\n   * Advances to the head of the Database, returning a diff between the\n   * previously current Snapshot and a new Snapshot at head. This is called\n   * in response to a notification from a Replicator subscription. Subsequent\n   * calls to {@link current()} return the new Snapshot. Note that the Snapshotter\n   * must be initialized before advancing.\n   *\n   * The returned {@link SnapshotDiff} contains snapshots at the endpoints\n   * of the database timeline. Iterating over the diff generates a sequence\n   * of {@link Change}s between the two snapshots.\n   *\n   * Note that this sequence is not chronological; rather, the sequence is\n   * ordered by `<table, row-key>`, such that a row can appear at most once\n   * in the common case, or twice if its table is `TRUNCATE`'d and a new value\n   * is subsequently `INSERT`'ed. This results in dropping most intermediate\n   * changes to a row and bounds the amount of work needed to catch up;\n   * however, as a consequence, a consistent database state is only guaranteed\n   * when the sequence has been fully consumed.\n   *\n   * Note that Change generation relies on the state of the underlying\n   * database connections, and because the connection for the previous snapshot\n   * is reused to produce the next snapshot, the diff object is only valid\n   * until the next call to `advance()`.\n   *\n   * It is okay for the caller to apply `Change`s to the `prev` snapshot\n   * during the iteration (e.g. this is necessary for IVM); the remainder\n   * of the iteration is not affected because a given row can appear at most\n   * once in the sequence (with the exception being TRUNCATE, after which the\n   * deleted rows can be re-inserted, but this will also behave correctly if\n   * the changes are applied).\n   *\n   * Once the changes have been applied, however, a _subsequent_ iteration\n   * will not produce the correct results. In order to perform multiple\n   * change-applying iterations, the caller must (1) create a save point\n   * on `prev` before each iteration, and (2) rollback to the save point after\n   * the iteration.\n   */\n  advance(\n    syncableTables: Map<string, LiteAndZqlSpec>,\n    allTableNames: Set<string>,\n  ): SnapshotDiff {\n    const {prev, curr} = this.advanceWithoutDiff();\n    return new Diff(this.#appID, syncableTables, allTableNames, prev, curr);\n  }\n\n  advanceWithoutDiff() {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    const next = this.#prev\n      ? this.#prev.resetToHead()\n      : Snapshot.create(\n          this.#lc,\n          this.#curr.db.db.name,\n          this.#appID,\n          this.#pageCacheSizeKib,\n        );\n    this.#prev = this.#curr;\n    this.#curr = next;\n    return {prev: this.#prev, curr: this.#curr};\n  }\n\n  /**\n   * Call this to close the database connections when the Snapshotter is\n   * no longer needed.\n   */\n  destroy() {\n    this.#curr?.db.db.close();\n    this.#prev?.db.db.close();\n    this.#lc.debug?.('closed database connections');\n  }\n}\n\nexport type Change = {\n  readonly table: string;\n  /**\n   * If this change represents a remove the row to remove,\n   * if nextValue is not null then all rows that have a unique constraint\n   * violation with nextValue.\n   * In both cases these rows should be removed.\n   */\n  readonly prevValues: Readonly<Row>[];\n  readonly nextValue: Readonly<Row> | null;\n  readonly rowKey: RowKey;\n};\n\n/**\n * Represents the difference between two database Snapshots.\n * Iterating over the object will produce a sequence of {@link Change}s\n * between the two snapshots.\n *\n * See {@link Snapshotter.advance()} for semantics and usage.\n */\nexport interface SnapshotDiff extends Iterable<Change> {\n  readonly prev: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n  readonly curr: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n\n  /**\n   * The number of ChangeLog entries between the snapshots. Note that this\n   * may not necessarily equal the number of `Change` objects that the iteration\n   * will produce, as `TRUNCATE` entries are counted as a single log entry which\n   * may be expanded into many changes (i.e. row deletes).\n   *\n   * TODO: Determine if it is worth changing the definition to count the\n   *       truncated rows. This would make diff computation more expensive\n   *       (requiring the count to be aggregated by operation type), which\n   *       may not be worth it for a presumable rare operation.\n   */\n  readonly changes: number;\n}\n\n/**\n * Thrown during an iteration of a {@link SnapshotDiff} when a schema\n * change or truncate is encountered, which result in aborting the\n * advancement and resetting / rehydrating the pipelines.\n */\nexport class ResetPipelinesSignal extends Error {\n  readonly name = 'ResetPipelinesSignal';\n\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n\nclass Snapshot {\n  static create(\n    lc: LogContext,\n    dbFile: string,\n    appID: string,\n    pageCacheSizeKib: number | undefined,\n  ) {\n    const conn = new Database(lc, dbFile);\n    conn.pragma('synchronous = OFF'); // Applied changes are ephemeral; COMMIT is never called.\n    if (pageCacheSizeKib !== undefined) {\n      conn.pragma(`cache_size = -${pageCacheSizeKib}`); // Negative = size in KiB\n    }\n    const [{journal_mode: mode}] = conn.pragma('journal_mode') as [\n      {journal_mode: string},\n    ];\n    // The Snapshotter operates on the replica file with BEGIN CONCURRENT,\n    // which must be used in concert with the replicator using BEGIN CONCURRENT\n    // on a db in the wal2 journal_mode.\n    assert(\n      mode === 'wal2',\n      `replica db must be in wal2 mode (current: ${mode})`,\n    );\n\n    const db = new StatementRunner(conn);\n    return new Snapshot(db, appID);\n  }\n\n  readonly db: StatementRunner;\n  readonly #appID: string;\n  readonly version: string;\n\n  constructor(db: StatementRunner, appID: string) {\n    db.beginConcurrent();\n    // Note: The subsequent read is necessary to acquire the read lock\n    // (which results in the logical creation of the snapshot). Calling\n    // `BEGIN CONCURRENT` alone does not result in acquiring the read lock.\n    const {stateVersion} = getReplicationState(db);\n\n    this.db = db;\n    this.#appID = appID;\n    this.version = stateVersion;\n  }\n\n  numChangesSince(prevVersion: string) {\n    const {count} = this.db.get(\n      'SELECT COUNT(*) AS count FROM \"_zero.changeLog2\" WHERE stateVersion > ?',\n      prevVersion,\n    );\n    return count;\n  }\n\n  changesSince(prevVersion: string) {\n    // Note: The queried fields are constrained to only those that are relevant\n    // to the snapshot diff, i.e. those defined in the changeLogEntrySchema.\n    const cached = this.db.statementCache.get(\n      `SELECT \"stateVersion\", \"table\", \"rowKey\", \"op\" FROM \"_zero.changeLog2\"\n         WHERE \"stateVersion\" > ? ORDER BY \"stateVersion\" ASC, \"pos\" ASC`,\n    );\n    return {\n      changes: cached.statement.iterate(prevVersion),\n      cleanup: () => this.db.statementCache.return(cached),\n    };\n  }\n\n  getRow(table: LiteTableSpecWithKeys, rowKey: JSONValue) {\n    const key = normalizedKeyOrder(rowKey as RowKey);\n    const conds = Object.keys(key).map(c => `${id(c)}=?`);\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.join(' AND ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.get<any>(Object.values(key));\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  getRows(table: LiteTableSpecWithKeys, keys: PrimaryKey[], row: RowValue) {\n    // Filter out keys where any column is NULL. This is both correct and\n    // critical for performance:\n    // 1. Correctness: NULL values can't violate uniqueness (NULL != NULL in SQL)\n    // 2. Performance: SQLite's MULTI-INDEX OR optimization completely fails when\n    //    any branch involves NULL, falling back to a full table scan. This was\n    //    causing slowdowns of hundreds of times on tables with nullable unique columns.\n    const validKeys = keys.filter(key =>\n      key.every(column => row[column] !== null && row[column] !== undefined),\n    );\n    if (validKeys.length === 0) {\n      return [];\n    }\n    const conds = validKeys.map(key => key.map(c => `${id(c)}=?`));\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.map(cond => cond.join(' AND ')).join(' OR ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.all<any>(\n        validKeys.flatMap(key => key.map(column => row[column])),\n      );\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  resetToHead(): Snapshot {\n    this.db.rollback();\n    return new Snapshot(this.db, this.#appID);\n  }\n}\n\nclass Diff implements SnapshotDiff {\n  readonly #permissionsTable: string;\n  readonly #syncableTables: Map<string, LiteAndZqlSpec>;\n  readonly #allTableNames: Set<string>;\n  readonly prev: Snapshot;\n  readonly curr: Snapshot;\n  readonly changes: number;\n\n  constructor(\n    appID: string,\n    syncableTables: Map<string, LiteAndZqlSpec>,\n    allTableNames: Set<string>,\n    prev: Snapshot,\n    curr: Snapshot,\n  ) {\n    this.#permissionsTable = `${appID}.permissions`;\n    this.#syncableTables = syncableTables;\n    this.#allTableNames = allTableNames;\n    this.prev = prev;\n    this.curr = curr;\n    this.changes = curr.numChangesSince(prev.version);\n  }\n\n  [Symbol.iterator](): Iterator<Change> {\n    const {changes, cleanup: done} = this.curr.changesSince(this.prev.version);\n\n    const cleanup = () => {\n      try {\n        // Allow open iterators to clean up their state.\n        changes.return?.(undefined);\n      } finally {\n        done();\n      }\n    };\n\n    return {\n      next: () => {\n        try {\n          for (;;) {\n            const {value, done} = changes.next();\n            if (done) {\n              cleanup();\n              return {value, done: true};\n            }\n\n            const {table, rowKey, op, stateVersion} = v.parse(value, schema);\n            if (op === RESET_OP) {\n              // The current map of `TableSpec`s may not have the correct or complete information.\n              throw new ResetPipelinesSignal(\n                `schema for table ${table} has changed`,\n              );\n            }\n            if (op === TRUNCATE_OP) {\n              // Truncates are also processed by rehydrating pipelines at current.\n              throw new ResetPipelinesSignal(\n                `table ${table} has been truncated`,\n              );\n            }\n            const specs = this.#syncableTables.get(table);\n            if (!specs) {\n              if (this.#allTableNames.has(table)) {\n                continue; // skip change log entries for non-syncable tables.\n              }\n              throw new Error(`change for unknown table ${table}`);\n            }\n            const {tableSpec, zqlSpec} = specs;\n\n            assert(rowKey !== null, 'rowKey must be present for row changes');\n            const nextValue =\n              op === SET_OP ? this.curr.getRow(tableSpec, rowKey) : null;\n            let prevValues;\n            if (nextValue) {\n              prevValues = this.prev.getRows(\n                tableSpec,\n                tableSpec.uniqueKeys,\n                nextValue,\n              );\n            } else {\n              const prevValue = this.prev.getRow(tableSpec, rowKey);\n              prevValues = prevValue ? [prevValue] : [];\n            }\n            if (nextValue === undefined) {\n              throw new Error(\n                `Missing value for ${table} ${stringify(rowKey)}`,\n              );\n            }\n            // Sanity check detects if the diff is being accessed after the Snapshots have advanced.\n            this.checkThatDiffIsValid(stateVersion, op, prevValues, nextValue);\n\n            if (prevValues.length === 0 && nextValue === null) {\n              // Filter out no-op changes (e.g. a delete of a row that does not exist in prev).\n              // TODO: Consider doing this for deep-equal values.\n              continue;\n            }\n\n            if (\n              table === this.#permissionsTable &&\n              prevValues.find(\n                prevValue => prevValue.permissions !== nextValue.permissions,\n              )\n            ) {\n              throw new ResetPipelinesSignal(\n                `Permissions have changed ${\n                  prevValues.find(\n                    prevValue =>\n                      prevValue.permissions !== nextValue.permissions,\n                  ).hash\n                } => ${nextValue.hash}`,\n              );\n            }\n\n            // Modify the values in place when converting to ZQL rows\n            // This is safe since we're the first node in the iterator chain.\n            // TODO Can we get rid of these RowValue casts?\n            return {\n              value: {\n                table,\n                prevValues: prevValues.map(prevValue =>\n                  fromSQLiteTypes(zqlSpec, prevValue, table),\n                ),\n                nextValue: nextValue\n                  ? fromSQLiteTypes(zqlSpec, nextValue, table)\n                  : null,\n                rowKey,\n              } satisfies Change,\n            };\n          }\n        } catch (e) {\n          // This control flow path is not covered by the return() method (i.e. `break`).\n          cleanup();\n          throw e;\n        }\n      },\n\n      return: (value: unknown) => {\n        cleanup();\n        return {value, done: true};\n      },\n    };\n  }\n\n  checkThatDiffIsValid(\n    stateVersion: string,\n    op: string,\n    prevValues: RowValue[],\n    nextValue: RowValue,\n  ) {\n    // Sanity checks to detect that the diff is not being accessed after\n    // the Snapshots have advanced.\n    if (stateVersion > this.curr.version) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. curr db has advanced past ${this.curr.version}`,\n      );\n    }\n    if (\n      prevValues.findIndex(\n        prevValue => (prevValue[ROW_VERSION] ?? '~') > this.prev.version,\n      ) !== -1\n    ) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. prev db has advanced past ${this.prev.version}.`,\n      );\n    }\n    if (op === SET_OP && nextValue[ROW_VERSION] !== stateVersion) {\n      throw new InvalidDiffError(\n        'Diff is no longer valid. curr db has advanced.',\n      );\n    }\n  }\n}\n\nexport class InvalidDiffError extends Error {\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n"],"names":["done","v.parse","schema","ROW_VERSION"],"mappings":";;;;;;;;;;;AAuFO,MAAM,YAAY;AAAA,EACd;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACT;AAAA,EACA;AAAA,EAEA,YACE,IACA,QACA,EAAC,MAAA,GACD,kBACA;AACA,SAAK,MAAM;AACX,SAAK,UAAU;AACf,SAAK,SAAS;AACd,SAAK,oBAAoB;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,OAAa;AACX,WAAO,KAAK,UAAU,QAAW,qBAAqB;AACtD,SAAK,QAAQ,SAAS;AAAA,MACpB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IAAA;AAEP,SAAK,IAAI,QAAQ,+BAA+B,KAAK,MAAM,OAAO,EAAE;AACpE,WAAO;AAAA,EACT;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK,UAAU;AAAA,EACxB;AAAA;AAAA,EAGA,UAAoB;AAClB,WAAO,KAAK,UAAU,QAAW,sCAAsC;AACvE,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAuCA,QACE,gBACA,eACc;AACd,UAAM,EAAC,MAAM,SAAQ,KAAK,mBAAA;AAC1B,WAAO,IAAI,KAAK,KAAK,QAAQ,gBAAgB,eAAe,MAAM,IAAI;AAAA,EACxE;AAAA,EAEA,qBAAqB;AACnB,WAAO,KAAK,UAAU,QAAW,sCAAsC;AACvE,UAAM,OAAO,KAAK,QACd,KAAK,MAAM,YAAA,IACX,SAAS;AAAA,MACP,KAAK;AAAA,MACL,KAAK,MAAM,GAAG,GAAG;AAAA,MACjB,KAAK;AAAA,MACL,KAAK;AAAA,IAAA;AAEX,SAAK,QAAQ,KAAK;AAClB,SAAK,QAAQ;AACb,WAAO,EAAC,MAAM,KAAK,OAAO,MAAM,KAAK,MAAA;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAU;AACR,SAAK,OAAO,GAAG,GAAG,MAAA;AAClB,SAAK,OAAO,GAAG,GAAG,MAAA;AAClB,SAAK,IAAI,QAAQ,6BAA6B;AAAA,EAChD;AACF;AAmDO,MAAM,6BAA6B,MAAM;AAAA,EACrC,OAAO;AAAA,EAEhB,YAAY,KAAa;AACvB,UAAM,GAAG;AAAA,EACX;AACF;AAEA,MAAM,SAAS;AAAA,EACb,OAAO,OACL,IACA,QACA,OACA,kBACA;AACA,UAAM,OAAO,IAAI,SAAS,IAAI,MAAM;AACpC,SAAK,OAAO,mBAAmB;AAC/B,QAAI,qBAAqB,QAAW;AAClC,WAAK,OAAO,iBAAiB,gBAAgB,EAAE;AAAA,IACjD;AACA,UAAM,CAAC,EAAC,cAAc,KAAA,CAAK,IAAI,KAAK,OAAO,cAAc;AAMzD;AAAA,MACE,SAAS;AAAA,MACT,6CAA6C,IAAI;AAAA,IAAA;AAGnD,UAAM,KAAK,IAAI,gBAAgB,IAAI;AACnC,WAAO,IAAI,SAAS,IAAI,KAAK;AAAA,EAC/B;AAAA,EAES;AAAA,EACA;AAAA,EACA;AAAA,EAET,YAAY,IAAqB,OAAe;AAC9C,OAAG,gBAAA;AAIH,UAAM,EAAC,aAAA,IAAgB,oBAAoB,EAAE;AAE7C,SAAK,KAAK;AACV,SAAK,SAAS;AACd,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,gBAAgB,aAAqB;AACnC,UAAM,EAAC,MAAA,IAAS,KAAK,GAAG;AAAA,MACtB;AAAA,MACA;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,aAAqB;AAGhC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC;AAAA;AAAA,IAAA;AAGF,WAAO;AAAA,MACL,SAAS,OAAO,UAAU,QAAQ,WAAW;AAAA,MAC7C,SAAS,MAAM,KAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IAAA;AAAA,EAEvD;AAAA,EAEA,OAAO,OAA8B,QAAmB;AACtD,UAAM,MAAM,mBAAmB,MAAgB;AAC/C,UAAM,QAAQ,OAAO,KAAK,GAAG,EAAE,IAAI,CAAA,MAAK,GAAG,GAAG,CAAC,CAAC,IAAI;AACpD,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO;AACtC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC,UAAU,KAAK,IAAI,CAAA,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,SAAS;AAAA,QAC/C,MAAM;AAAA,MAAA,CACP,UAAU,MAAM,KAAK,OAAO,CAAC;AAAA,IAAA;AAEhC,WAAO,UAAU,aAAa,IAAI;AAClC,QAAI;AAEF,aAAO,OAAO,UAAU,IAAS,OAAO,OAAO,GAAG,CAAC;AAAA,IACrD,UAAA;AACE,WAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,QAAQ,OAA8B,MAAoB,KAAe;AAOvE,UAAM,YAAY,KAAK;AAAA,MAAO,CAAA,QAC5B,IAAI,MAAM,CAAA,WAAU,IAAI,MAAM,MAAM,QAAQ,IAAI,MAAM,MAAM,MAAS;AAAA,IAAA;AAEvE,QAAI,UAAU,WAAW,GAAG;AAC1B,aAAO,CAAA;AAAA,IACT;AACA,UAAM,QAAQ,UAAU,IAAI,CAAA,QAAO,IAAI,IAAI,CAAA,MAAK,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC;AAC7D,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO;AACtC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC,UAAU,KAAK,IAAI,CAAA,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,SAAS;AAAA,QAC/C,MAAM;AAAA,MAAA,CACP,UAAU,MAAM,IAAI,CAAA,SAAQ,KAAK,KAAK,OAAO,CAAC,EAAE,KAAK,MAAM,CAAC;AAAA,IAAA;AAE/D,WAAO,UAAU,aAAa,IAAI;AAClC,QAAI;AAEF,aAAO,OAAO,UAAU;AAAA,QACtB,UAAU,QAAQ,CAAA,QAAO,IAAI,IAAI,CAAA,WAAU,IAAI,MAAM,CAAC,CAAC;AAAA,MAAA;AAAA,IAE3D,UAAA;AACE,WAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,cAAwB;AACtB,SAAK,GAAG,SAAA;AACR,WAAO,IAAI,SAAS,KAAK,IAAI,KAAK,MAAM;AAAA,EAC1C;AACF;AAEA,MAAM,KAA6B;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YACE,OACA,gBACA,eACA,MACA,MACA;AACA,SAAK,oBAAoB,GAAG,KAAK;AACjC,SAAK,kBAAkB;AACvB,SAAK,iBAAiB;AACtB,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,UAAU,KAAK,gBAAgB,KAAK,OAAO;AAAA,EAClD;AAAA,EAEA,CAAC,OAAO,QAAQ,IAAsB;AACpC,UAAM,EAAC,SAAS,SAAS,SAAQ,KAAK,KAAK,aAAa,KAAK,KAAK,OAAO;AAEzE,UAAM,UAAU,MAAM;AACpB,UAAI;AAEF,gBAAQ,SAAS,MAAS;AAAA,MAC5B,UAAA;AACE,aAAA;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL,MAAM,MAAM;AACV,YAAI;AACF,qBAAS;AACP,kBAAM,EAAC,OAAO,MAAAA,MAAAA,IAAQ,QAAQ,KAAA;AAC9B,gBAAIA,OAAM;AACR,sBAAA;AACA,qBAAO,EAAC,OAAO,MAAM,KAAA;AAAA,YACvB;AAEA,kBAAM,EAAC,OAAO,QAAQ,IAAI,iBAAgBC,MAAQ,OAAOC,oBAAM;AAC/D,gBAAI,OAAO,UAAU;AAEnB,oBAAM,IAAI;AAAA,gBACR,oBAAoB,KAAK;AAAA,cAAA;AAAA,YAE7B;AACA,gBAAI,OAAO,aAAa;AAEtB,oBAAM,IAAI;AAAA,gBACR,SAAS,KAAK;AAAA,cAAA;AAAA,YAElB;AACA,kBAAM,QAAQ,KAAK,gBAAgB,IAAI,KAAK;AAC5C,gBAAI,CAAC,OAAO;AACV,kBAAI,KAAK,eAAe,IAAI,KAAK,GAAG;AAClC;AAAA,cACF;AACA,oBAAM,IAAI,MAAM,4BAA4B,KAAK,EAAE;AAAA,YACrD;AACA,kBAAM,EAAC,WAAW,QAAA,IAAW;AAE7B,mBAAO,WAAW,MAAM,wCAAwC;AAChE,kBAAM,YACJ,OAAO,SAAS,KAAK,KAAK,OAAO,WAAW,MAAM,IAAI;AACxD,gBAAI;AACJ,gBAAI,WAAW;AACb,2BAAa,KAAK,KAAK;AAAA,gBACrB;AAAA,gBACA,UAAU;AAAA,gBACV;AAAA,cAAA;AAAA,YAEJ,OAAO;AACL,oBAAM,YAAY,KAAK,KAAK,OAAO,WAAW,MAAM;AACpD,2BAAa,YAAY,CAAC,SAAS,IAAI,CAAA;AAAA,YACzC;AACA,gBAAI,cAAc,QAAW;AAC3B,oBAAM,IAAI;AAAA,gBACR,qBAAqB,KAAK,IAAI,UAAU,MAAM,CAAC;AAAA,cAAA;AAAA,YAEnD;AAEA,iBAAK,qBAAqB,cAAc,IAAI,YAAY,SAAS;AAEjE,gBAAI,WAAW,WAAW,KAAK,cAAc,MAAM;AAGjD;AAAA,YACF;AAEA,gBACE,UAAU,KAAK,qBACf,WAAW;AAAA,cACT,CAAA,cAAa,UAAU,gBAAgB,UAAU;AAAA,YAAA,GAEnD;AACA,oBAAM,IAAI;AAAA,gBACR,4BACE,WAAW;AAAA,kBACT,CAAA,cACE,UAAU,gBAAgB,UAAU;AAAA,gBAAA,EACtC,IACJ,OAAO,UAAU,IAAI;AAAA,cAAA;AAAA,YAEzB;AAKA,mBAAO;AAAA,cACL,OAAO;AAAA,gBACL;AAAA,gBACA,YAAY,WAAW;AAAA,kBAAI,CAAA,cACzB,gBAAgB,SAAS,WAAW,KAAK;AAAA,gBAAA;AAAA,gBAE3C,WAAW,YACP,gBAAgB,SAAS,WAAW,KAAK,IACzC;AAAA,gBACJ;AAAA,cAAA;AAAA,YACF;AAAA,UAEJ;AAAA,QACF,SAAS,GAAG;AAEV,kBAAA;AACA,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,MAEA,QAAQ,CAAC,UAAmB;AAC1B,gBAAA;AACA,eAAO,EAAC,OAAO,MAAM,KAAA;AAAA,MACvB;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEA,qBACE,cACA,IACA,YACA,WACA;AAGA,QAAI,eAAe,KAAK,KAAK,SAAS;AACpC,YAAM,IAAI;AAAA,QACR,sDAAsD,KAAK,KAAK,OAAO;AAAA,MAAA;AAAA,IAE3E;AACA,QACE,WAAW;AAAA,MACT,gBAAc,UAAUC,wBAAW,KAAK,OAAO,KAAK,KAAK;AAAA,IAAA,MACrD,IACN;AACA,YAAM,IAAI;AAAA,QACR,sDAAsD,KAAK,KAAK,OAAO;AAAA,MAAA;AAAA,IAE3E;AACA,QAAI,OAAO,UAAU,UAAUA,wBAAW,MAAM,cAAc;AAC5D,YAAM,IAAI;AAAA,QACR;AAAA,MAAA;AAAA,IAEJ;AAAA,EACF;AACF;AAEO,MAAM,yBAAyB,MAAM;AAAA,EAC1C,YAAY,KAAa;AACvB,UAAM,GAAG;AAAA,EACX;AACF;"}
+{"version":3,"file":"snapshotter.js","names":["#lc","#dbFile","#appID","#pageCacheSizeKib","#curr","#prev","#permissionsTable","#syncableTables","#allTableNames"],"sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {stringify, type JSONValue} from '../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Row} from '../../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../../zero-types/src/schema.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {fromSQLiteTypes} from '../../../../zqlite/src/table-source.ts';\nimport type {\n  LiteAndZqlSpec,\n  LiteTableSpecWithKeysAndVersion,\n} from '../../db/specs.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {\n  normalizedKeyOrder,\n  type RowKey,\n  type RowValue,\n} from '../../types/row-key.ts';\nimport type {AppID} from '../../types/shards.ts';\nimport {id} from '../../types/sql.ts';\nimport {\n  RESET_OP,\n  changeLogEntrySchema as schema,\n  SET_OP,\n  TRUNCATE_OP,\n} from '../replicator/schema/change-log.ts';\nimport {\n  getReplicationState,\n  ZERO_VERSION_COLUMN_NAME as ROW_VERSION,\n} from '../replicator/schema/replication-state.ts';\n\n/**\n * A `Snapshotter` manages the progression of database snapshots for a\n * ViewSyncer.\n *\n * The Replicator and ViewSyncers operate on the same SQLite file, with the\n * Replicator being the sole writer to the database. The IVM logic in\n * ViewSyncers, however, rely on incrementally applying changes to the DB to\n * update the state of its pipelines.\n *\n * To avoid coupling the progress of the Replicator and all IVM pipelines on\n * each other, ViewSyncers operate on ephemeral forks of the database by holding\n * [concurrent](https://sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md)\n * snapshots of the database and simulating (but ultimately rolling back)\n * mutations on these snapshots.\n *\n * Example:\n * 1. ViewSyncer takes `snapshot_a` at version `t1` of the database and\n *    hydrates its pipeline(s).\n * 2. Replicator applies a new transaction to the database and notifies\n *    subscribers.\n * 3. ViewSyncer takes `snapshot_b` at `t2`, and queries the `ChangeLog` at\n *    that snapshot for changes since `t1`.\n * 4. ViewSyncer applies those changes to `snapshot_a` for IVM, but does not\n *    commit them. (Recall that the Replicator is the sole writer to the db, so\n *    the ViewSyncer never commits any writes.)\n * 5. Replicator applies the next transaction and advances the database to `t3`.\n * 6. ViewSyncer rolls back `snapshot_a` and opens `snapshot_c` at `t3`, using\n *    `snapshot_b` to simulate changes from `t2` to `t3`.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ```\n *\n * Note that the Replicator (and ViewSyncers) do not wait on the progress of\n * other ViewSyncers. If a ViewSyncer is busy hydrating at `t1`, the Replicator\n * and other ViewSyncers can progress through `t2`, `t3`, etc. independently,\n * as the busy ViewSyncer simply takes its own snapshot when it is ready.\n *\n * ```\n * Replicator:   t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer1:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ViewSyncer2:       [.......... snapshot_a ..........] ----> [snapshot_b]\n * ```\n *\n * To minimize Database connections (and statement preparation, etc.), the\n * Snapshotter reuses the connection from the previous (rolled back)\n * snapshot when opening the new one.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n *                     (conn_1)           (conn_2)           (conn_1)\n * ```\n *\n * In this manner, each ViewSyncer uses two connections that continually\n * \"leapfrog\" each other to replay the timeline of changes in isolation from\n * the Replicator and other ViewSyncers.\n */\nexport class Snapshotter {\n  readonly #lc: LogContext;\n  readonly #dbFile: string;\n  readonly #appID: string;\n  readonly #pageCacheSizeKib: number | undefined;\n  #curr: Snapshot | undefined;\n  #prev: Snapshot | undefined;\n\n  constructor(\n    lc: LogContext,\n    dbFile: string,\n    {appID}: AppID,\n    pageCacheSizeKib?: number | undefined,\n  ) {\n    this.#lc = lc;\n    this.#dbFile = dbFile;\n    this.#appID = appID;\n    this.#pageCacheSizeKib = pageCacheSizeKib;\n  }\n\n  /**\n   * Initializes the snapshot to the current head of the database. This must be\n   * only be called once. The state of whether a Snapshotter has been initialized\n   * can be determined by calling {@link initialized()}.\n   */\n  init(): this {\n    assert(this.#curr === undefined, 'Already initialized');\n    this.#curr = Snapshot.create(\n      this.#lc,\n      this.#dbFile,\n      this.#appID,\n      this.#pageCacheSizeKib,\n    );\n    this.#lc.debug?.(`Initial snapshot at version ${this.#curr.version}`);\n    return this;\n  }\n\n  initialized(): boolean {\n    return this.#curr !== undefined;\n  }\n\n  /** Returns the current snapshot. Asserts if {@link initialized()} is false. */\n  current(): Snapshot {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    return this.#curr;\n  }\n\n  /**\n   * Advances to the head of the Database, returning a diff between the\n   * previously current Snapshot and a new Snapshot at head. This is called\n   * in response to a notification from a Replicator subscription. Subsequent\n   * calls to {@link current()} return the new Snapshot. Note that the Snapshotter\n   * must be initialized before advancing.\n   *\n   * The returned {@link SnapshotDiff} contains snapshots at the endpoints\n   * of the database timeline. Iterating over the diff generates a sequence\n   * of {@link Change}s between the two snapshots.\n   *\n   * Note that this sequence is not chronological; rather, the sequence is\n   * ordered by `<table, row-key>`, such that a row can appear at most once\n   * in the common case, or twice if its table is `TRUNCATE`'d and a new value\n   * is subsequently `INSERT`'ed. This results in dropping most intermediate\n   * changes to a row and bounds the amount of work needed to catch up;\n   * however, as a consequence, a consistent database state is only guaranteed\n   * when the sequence has been fully consumed.\n   *\n   * Note that Change generation relies on the state of the underlying\n   * database connections, and because the connection for the previous snapshot\n   * is reused to produce the next snapshot, the diff object is only valid\n   * until the next call to `advance()`.\n   *\n   * It is okay for the caller to apply `Change`s to the `prev` snapshot\n   * during the iteration (e.g. this is necessary for IVM); the remainder\n   * of the iteration is not affected because a given row can appear at most\n   * once in the sequence (with the exception being TRUNCATE, after which the\n   * deleted rows can be re-inserted, but this will also behave correctly if\n   * the changes are applied).\n   *\n   * Once the changes have been applied, however, a _subsequent_ iteration\n   * will not produce the correct results. In order to perform multiple\n   * change-applying iterations, the caller must (1) create a save point\n   * on `prev` before each iteration, and (2) rollback to the save point after\n   * the iteration.\n   */\n  advance(\n    syncableTables: Map<string, LiteAndZqlSpec>,\n    allTableNames: Set<string>,\n  ): SnapshotDiff {\n    const {prev, curr} = this.advanceWithoutDiff();\n    return new Diff(this.#appID, syncableTables, allTableNames, prev, curr);\n  }\n\n  advanceWithoutDiff() {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    const next = this.#prev\n      ? this.#prev.resetToHead()\n      : Snapshot.create(\n          this.#lc,\n          this.#curr.db.db.name,\n          this.#appID,\n          this.#pageCacheSizeKib,\n        );\n    this.#prev = this.#curr;\n    this.#curr = next;\n    return {prev: this.#prev, curr: this.#curr};\n  }\n\n  /**\n   * Call this to close the database connections when the Snapshotter is\n   * no longer needed.\n   */\n  destroy() {\n    this.#curr?.db.db.close();\n    this.#prev?.db.db.close();\n    this.#lc.debug?.('closed database connections');\n  }\n}\n\nexport type Change = {\n  readonly table: string;\n  /**\n   * If this change represents a remove the row to remove,\n   * if nextValue is not null then all rows that have a unique constraint\n   * violation with nextValue.\n   * In both cases these rows should be removed.\n   */\n  readonly prevValues: Readonly<Row>[];\n  readonly nextValue: Readonly<Row> | null;\n  readonly rowKey: RowKey;\n};\n\n/**\n * Represents the difference between two database Snapshots.\n * Iterating over the object will produce a sequence of {@link Change}s\n * between the two snapshots.\n *\n * See {@link Snapshotter.advance()} for semantics and usage.\n */\nexport interface SnapshotDiff extends Iterable<Change> {\n  readonly prev: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n  readonly curr: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n\n  /**\n   * The number of ChangeLog entries between the snapshots. Note that this\n   * may not necessarily equal the number of `Change` objects that the iteration\n   * will produce, as `TRUNCATE` entries are counted as a single log entry which\n   * may be expanded into many changes (i.e. row deletes).\n   *\n   * TODO: Determine if it is worth changing the definition to count the\n   *       truncated rows. This would make diff computation more expensive\n   *       (requiring the count to be aggregated by operation type), which\n   *       may not be worth it for a presumable rare operation.\n   */\n  readonly changes: number;\n}\n\n/**\n * Thrown during an iteration of a {@link SnapshotDiff} when a schema\n * change or truncate is encountered, which result in aborting the\n * advancement and resetting / rehydrating the pipelines.\n */\nexport class ResetPipelinesSignal extends Error {\n  readonly name = 'ResetPipelinesSignal';\n\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n\nclass Snapshot {\n  static create(\n    lc: LogContext,\n    dbFile: string,\n    appID: string,\n    pageCacheSizeKib: number | undefined,\n  ) {\n    const conn = new Database(lc, dbFile);\n    conn.pragma('synchronous = OFF'); // Applied changes are ephemeral; COMMIT is never called.\n    if (pageCacheSizeKib !== undefined) {\n      conn.pragma(`cache_size = -${pageCacheSizeKib}`); // Negative = size in KiB\n    }\n    const [{journal_mode: mode}] = conn.pragma('journal_mode') as [\n      {journal_mode: string},\n    ];\n    // The Snapshotter operates on the replica file with BEGIN CONCURRENT,\n    // which must be used in concert with the replicator using BEGIN CONCURRENT\n    // on a db in the wal2 journal_mode.\n    assert(\n      mode === 'wal2',\n      `replica db must be in wal2 mode (current: ${mode})`,\n    );\n\n    const db = new StatementRunner(conn);\n    return new Snapshot(db, appID);\n  }\n\n  readonly db: StatementRunner;\n  readonly #appID: string;\n  readonly version: string;\n\n  constructor(db: StatementRunner, appID: string) {\n    db.beginConcurrent();\n    // Note: The subsequent read is necessary to acquire the read lock\n    // (which results in the logical creation of the snapshot). Calling\n    // `BEGIN CONCURRENT` alone does not result in acquiring the read lock.\n    const {stateVersion} = getReplicationState(db);\n\n    this.db = db;\n    this.#appID = appID;\n    this.version = stateVersion;\n  }\n\n  numChangesSince(prevVersion: string) {\n    const {count} = this.db.get(\n      'SELECT COUNT(*) AS count FROM \"_zero.changeLog2\" WHERE stateVersion > ?',\n      prevVersion,\n    );\n    return count;\n  }\n\n  changesSince(prevVersion: string) {\n    // Note: The queried fields are constrained to only those that are relevant\n    // to the snapshot diff, i.e. those defined in the changeLogEntrySchema.\n    const cached = this.db.statementCache.get(\n      `SELECT \"stateVersion\", \"table\", \"rowKey\", \"op\" FROM \"_zero.changeLog2\"\n         WHERE \"stateVersion\" > ? ORDER BY \"stateVersion\" ASC, \"pos\" ASC`,\n    );\n    return {\n      changes: cached.statement.iterate(prevVersion),\n      cleanup: () => this.db.statementCache.return(cached),\n    };\n  }\n\n  getRow(table: LiteTableSpecWithKeysAndVersion, rowKey: JSONValue) {\n    const key = normalizedKeyOrder(rowKey as RowKey);\n    const conds = Object.keys(key).map(c => `${id(c)}=?`);\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.join(' AND ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.get<any>(Object.values(key));\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  getRows(\n    table: LiteTableSpecWithKeysAndVersion,\n    keys: PrimaryKey[],\n    row: RowValue,\n  ) {\n    // Filter out keys where any column is NULL. This is both correct and\n    // critical for performance:\n    // 1. Correctness: NULL values can't violate uniqueness (NULL != NULL in SQL)\n    // 2. Performance: SQLite's MULTI-INDEX OR optimization completely fails when\n    //    any branch involves NULL, falling back to a full table scan. This was\n    //    causing slowdowns of hundreds of times on tables with nullable unique columns.\n    const validKeys = keys.filter(key =>\n      key.every(column => row[column] !== null && row[column] !== undefined),\n    );\n    if (validKeys.length === 0) {\n      return [];\n    }\n    const conds = validKeys.map(key => key.map(c => `${id(c)}=?`));\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.map(cond => cond.join(' AND ')).join(' OR ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.all<any>(\n        validKeys.flatMap(key => key.map(column => row[column])),\n      );\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  resetToHead(): Snapshot {\n    this.db.rollback();\n    return new Snapshot(this.db, this.#appID);\n  }\n}\n\nclass Diff implements SnapshotDiff {\n  readonly #permissionsTable: string;\n  readonly #syncableTables: Map<string, LiteAndZqlSpec>;\n  readonly #allTableNames: Set<string>;\n  readonly prev: Snapshot;\n  readonly curr: Snapshot;\n  readonly changes: number;\n\n  constructor(\n    appID: string,\n    syncableTables: Map<string, LiteAndZqlSpec>,\n    allTableNames: Set<string>,\n    prev: Snapshot,\n    curr: Snapshot,\n  ) {\n    this.#permissionsTable = `${appID}.permissions`;\n    this.#syncableTables = syncableTables;\n    this.#allTableNames = allTableNames;\n    this.prev = prev;\n    this.curr = curr;\n    this.changes = curr.numChangesSince(prev.version);\n  }\n\n  [Symbol.iterator](): Iterator<Change> {\n    const {changes, cleanup: done} = this.curr.changesSince(this.prev.version);\n\n    const cleanup = () => {\n      try {\n        // Allow open iterators to clean up their state.\n        changes.return?.(undefined);\n      } finally {\n        done();\n      }\n    };\n\n    return {\n      next: () => {\n        try {\n          for (;;) {\n            const {value, done} = changes.next();\n            if (done) {\n              cleanup();\n              return {value, done: true};\n            }\n\n            const {table, rowKey, op, stateVersion} = v.parse(value, schema);\n            if (op === RESET_OP) {\n              // The current map of `TableSpec`s may not have the correct or complete information.\n              throw new ResetPipelinesSignal(\n                `schema for table ${table} has changed`,\n              );\n            }\n            if (op === TRUNCATE_OP) {\n              // Truncates are also processed by rehydrating pipelines at current.\n              throw new ResetPipelinesSignal(\n                `table ${table} has been truncated`,\n              );\n            }\n            const specs = this.#syncableTables.get(table);\n            if (!specs) {\n              if (this.#allTableNames.has(table)) {\n                continue; // skip change log entries for non-syncable tables.\n              }\n              throw new Error(`change for unknown table ${table}`);\n            }\n            const {tableSpec, zqlSpec} = specs;\n\n            // Sanity check: All change log ops should have a stateVersion\n            // greater than minRowVersion in the table metadata. This is a\n            // mini-proof that the overlay does not need to be applied to\n            // rows produced in incremental catchup, based on the invariant in\n            // change-processor's #bumpVersions(), whereby the setting of the\n            // minRowVersion is always followed by a RESET OP, meaning that\n            // subsequent change-log traversal happens at a later version.\n            assert(\n              (tableSpec.minRowVersion ?? '') < stateVersion,\n              () =>\n                `unexpected change @${stateVersion} for table ${table} with ` +\n                `minRowVersion ${tableSpec.minRowVersion}: ${op}(${rowKey})`,\n            );\n\n            assert(rowKey !== null, 'rowKey must be present for row changes');\n            const nextValue =\n              op === SET_OP ? this.curr.getRow(tableSpec, rowKey) : null;\n            let prevValues;\n            if (nextValue) {\n              prevValues = this.prev.getRows(\n                tableSpec,\n                tableSpec.uniqueKeys,\n                nextValue,\n              );\n            } else {\n              const prevValue = this.prev.getRow(tableSpec, rowKey);\n              prevValues = prevValue ? [prevValue] : [];\n            }\n            if (nextValue === undefined) {\n              throw new Error(\n                `Missing value for ${table} ${stringify(rowKey)}`,\n              );\n            }\n            // Sanity check detects if the diff is being accessed after the Snapshots have advanced.\n            this.checkThatDiffIsValid(stateVersion, op, prevValues, nextValue);\n\n            if (prevValues.length === 0 && nextValue === null) {\n              // Filter out no-op changes (e.g. a delete of a row that does not exist in prev).\n              // TODO: Consider doing this for deep-equal values.\n              continue;\n            }\n\n            if (\n              table === this.#permissionsTable &&\n              prevValues.find(\n                prevValue => prevValue.permissions !== nextValue.permissions,\n              )\n            ) {\n              throw new ResetPipelinesSignal(\n                `Permissions have changed ${\n                  prevValues.find(\n                    prevValue =>\n                      prevValue.permissions !== nextValue.permissions,\n                  ).hash\n                } => ${nextValue.hash}`,\n              );\n            }\n\n            // Modify the values in place when converting to ZQL rows\n            // This is safe since we're the first node in the iterator chain.\n            // TODO Can we get rid of these RowValue casts?\n            return {\n              value: {\n                table,\n                prevValues: prevValues.map(prevValue =>\n                  fromSQLiteTypes(zqlSpec, prevValue, table),\n                ),\n                nextValue: nextValue\n                  ? fromSQLiteTypes(zqlSpec, nextValue, table)\n                  : null,\n                rowKey,\n              } satisfies Change,\n            };\n          }\n        } catch (e) {\n          // This control flow path is not covered by the return() method (i.e. `break`).\n          cleanup();\n          throw e;\n        }\n      },\n\n      return: (value: unknown) => {\n        cleanup();\n        return {value, done: true};\n      },\n    };\n  }\n\n  checkThatDiffIsValid(\n    stateVersion: string,\n    op: string,\n    prevValues: RowValue[],\n    nextValue: RowValue,\n  ) {\n    // Sanity checks to detect that the diff is not being accessed after\n    // the Snapshots have advanced.\n    if (stateVersion > this.curr.version) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. curr db has advanced past ${this.curr.version}`,\n      );\n    }\n    if (\n      prevValues.findIndex(\n        prevValue => (prevValue[ROW_VERSION] ?? '~') > this.prev.version,\n      ) !== -1\n    ) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. prev db has advanced past ${this.prev.version}.`,\n      );\n    }\n    if (op === SET_OP && nextValue[ROW_VERSION] !== stateVersion) {\n      throw new InvalidDiffError(\n        'Diff is no longer valid. curr db has advanced.',\n      );\n    }\n  }\n}\n\nexport class InvalidDiffError extends Error {\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0FA,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,EAAC,SACD,kBACA;AACA,QAAA,KAAW;AACX,QAAA,SAAe;AACf,QAAA,QAAc;AACd,QAAA,mBAAyB;;;;;;;CAQ3B,OAAa;AACX,SAAO,MAAA,SAAe,KAAA,GAAW,sBAAsB;AACvD,QAAA,OAAa,SAAS,OACpB,MAAA,IACA,MAAA,QACA,MAAA,OACA,MAAA,iBACD;AACD,QAAA,GAAS,QAAQ,+BAA+B,MAAA,KAAW,UAAU;AACrE,SAAO;;CAGT,cAAuB;AACrB,SAAO,MAAA,SAAe,KAAA;;;CAIxB,UAAoB;AAClB,SAAO,MAAA,SAAe,KAAA,GAAW,uCAAuC;AACxE,SAAO,MAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwCT,QACE,gBACA,eACc;EACd,MAAM,EAAC,MAAM,SAAQ,KAAK,oBAAoB;AAC9C,SAAO,IAAI,KAAK,MAAA,OAAa,gBAAgB,eAAe,MAAM,KAAK;;CAGzE,qBAAqB;AACnB,SAAO,MAAA,SAAe,KAAA,GAAW,uCAAuC;EACxE,MAAM,OAAO,MAAA,OACT,MAAA,KAAW,aAAa,GACxB,SAAS,OACP,MAAA,IACA,MAAA,KAAW,GAAG,GAAG,MACjB,MAAA,OACA,MAAA,iBACD;AACL,QAAA,OAAa,MAAA;AACb,QAAA,OAAa;AACb,SAAO;GAAC,MAAM,MAAA;GAAY,MAAM,MAAA;GAAW;;;;;;CAO7C,UAAU;AACR,QAAA,MAAY,GAAG,GAAG,OAAO;AACzB,QAAA,MAAY,GAAG,GAAG,OAAO;AACzB,QAAA,GAAS,QAAQ,8BAA8B;;;;;;;;AAqDnD,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAgB;CAEhB,YAAY,KAAa;AACvB,QAAM,IAAI;;;AAId,IAAM,WAAN,MAAM,SAAS;CACb,OAAO,OACL,IACA,QACA,OACA,kBACA;EACA,MAAM,OAAO,IAAI,SAAS,IAAI,OAAO;AACrC,OAAK,OAAO,oBAAoB;AAChC,MAAI,qBAAqB,KAAA,EACvB,MAAK,OAAO,iBAAiB,mBAAmB;EAElD,MAAM,CAAC,EAAC,cAAc,UAAS,KAAK,OAAO,eAAe;AAM1D,SACE,SAAS,QACT,6CAA6C,KAAK,GACnD;AAGD,SAAO,IAAI,SADA,IAAI,gBAAgB,KAAK,EACZ,MAAM;;CAGhC;CACA;CACA;CAEA,YAAY,IAAqB,OAAe;AAC9C,KAAG,iBAAiB;EAIpB,MAAM,EAAC,iBAAgB,oBAAoB,GAAG;AAE9C,OAAK,KAAK;AACV,QAAA,QAAc;AACd,OAAK,UAAU;;CAGjB,gBAAgB,aAAqB;EACnC,MAAM,EAAC,UAAS,KAAK,GAAG,IACtB,6EACA,YACD;AACD,SAAO;;CAGT,aAAa,aAAqB;EAGhC,MAAM,SAAS,KAAK,GAAG,eAAe,IACpC;0EAED;AACD,SAAO;GACL,SAAS,OAAO,UAAU,QAAQ,YAAY;GAC9C,eAAe,KAAK,GAAG,eAAe,OAAO,OAAO;GACrD;;CAGH,OAAO,OAAwC,QAAmB;EAChE,MAAM,MAAM,mBAAmB,OAAiB;EAChD,MAAM,QAAQ,OAAO,KAAK,IAAI,CAAC,KAAI,MAAK,GAAG,GAAG,EAAE,CAAC,IAAI;EACrD,MAAM,OAAO,OAAO,KAAK,MAAM,QAAQ;EACvC,MAAM,SAAS,KAAK,GAAG,eAAe,IACpC,UAAU,KAAK,KAAI,MAAK,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,QAAQ,GAC/C,MAAM,KACP,CAAC,SAAS,MAAM,KAAK,QAAQ,GAC/B;AACD,SAAO,UAAU,aAAa,KAAK;AACnC,MAAI;AAEF,UAAO,OAAO,UAAU,IAAS,OAAO,OAAO,IAAI,CAAC;YAC5C;AACR,QAAK,GAAG,eAAe,OAAO,OAAO;;;CAIzC,QACE,OACA,MACA,KACA;EAOA,MAAM,YAAY,KAAK,QAAO,QAC5B,IAAI,OAAM,WAAU,IAAI,YAAY,QAAQ,IAAI,YAAY,KAAA,EAAU,CACvE;AACD,MAAI,UAAU,WAAW,EACvB,QAAO,EAAE;EAEX,MAAM,QAAQ,UAAU,KAAI,QAAO,IAAI,KAAI,MAAK,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC;EAC9D,MAAM,OAAO,OAAO,KAAK,MAAM,QAAQ;EACvC,MAAM,SAAS,KAAK,GAAG,eAAe,IACpC,UAAU,KAAK,KAAI,MAAK,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,QAAQ,GAC/C,MAAM,KACP,CAAC,SAAS,MAAM,KAAI,SAAQ,KAAK,KAAK,QAAQ,CAAC,CAAC,KAAK,OAAO,GAC9D;AACD,SAAO,UAAU,aAAa,KAAK;AACnC,MAAI;AAEF,UAAO,OAAO,UAAU,IACtB,UAAU,SAAQ,QAAO,IAAI,KAAI,WAAU,IAAI,QAAQ,CAAC,CACzD;YACO;AACR,QAAK,GAAG,eAAe,OAAO,OAAO;;;CAIzC,cAAwB;AACtB,OAAK,GAAG,UAAU;AAClB,SAAO,IAAI,SAAS,KAAK,IAAI,MAAA,MAAY;;;AAI7C,IAAM,OAAN,MAAmC;CACjC;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,OACA,gBACA,eACA,MACA,MACA;AACA,QAAA,mBAAyB,GAAG,MAAM;AAClC,QAAA,iBAAuB;AACvB,QAAA,gBAAsB;AACtB,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,UAAU,KAAK,gBAAgB,KAAK,QAAQ;;CAGnD,CAAC,OAAO,YAA8B;EACpC,MAAM,EAAC,SAAS,SAAS,SAAQ,KAAK,KAAK,aAAa,KAAK,KAAK,QAAQ;EAE1E,MAAM,gBAAgB;AACpB,OAAI;AAEF,YAAQ,SAAS,KAAA,EAAU;aACnB;AACR,UAAM;;;AAIV,SAAO;GACL,YAAY;AACV,QAAI;AACF,cAAS;MACP,MAAM,EAAC,OAAO,SAAQ,QAAQ,MAAM;AACpC,UAAI,MAAM;AACR,gBAAS;AACT,cAAO;QAAC;QAAO,MAAM;QAAK;;MAG5B,MAAM,EAAC,OAAO,QAAQ,IAAI,iBAAgB,MAAQ,OAAO,qBAAO;AAChE,UAAI,OAAA,IAEF,OAAM,IAAI,qBACR,oBAAoB,MAAM,cAC3B;AAEH,UAAI,OAAA,IAEF,OAAM,IAAI,qBACR,SAAS,MAAM,qBAChB;MAEH,MAAM,QAAQ,MAAA,eAAqB,IAAI,MAAM;AAC7C,UAAI,CAAC,OAAO;AACV,WAAI,MAAA,cAAoB,IAAI,MAAM,CAChC;AAEF,aAAM,IAAI,MAAM,4BAA4B,QAAQ;;MAEtD,MAAM,EAAC,WAAW,YAAW;AAS7B,cACG,UAAU,iBAAiB,MAAM,oBAEhC,sBAAsB,aAAa,aAAa,MAAM,sBACrC,UAAU,cAAc,IAAI,GAAG,GAAG,OAAO,GAC7D;AAED,aAAO,WAAW,MAAM,yCAAyC;MACjE,MAAM,YACJ,OAAA,MAAgB,KAAK,KAAK,OAAO,WAAW,OAAO,GAAG;MACxD,IAAI;AACJ,UAAI,UACF,cAAa,KAAK,KAAK,QACrB,WACA,UAAU,YACV,UACD;WACI;OACL,MAAM,YAAY,KAAK,KAAK,OAAO,WAAW,OAAO;AACrD,oBAAa,YAAY,CAAC,UAAU,GAAG,EAAE;;AAE3C,UAAI,cAAc,KAAA,EAChB,OAAM,IAAI,MACR,qBAAqB,MAAM,GAAG,UAAU,OAAO,GAChD;AAGH,WAAK,qBAAqB,cAAc,IAAI,YAAY,UAAU;AAElE,UAAI,WAAW,WAAW,KAAK,cAAc,KAG3C;AAGF,UACE,UAAU,MAAA,oBACV,WAAW,MACT,cAAa,UAAU,gBAAgB,UAAU,YAClD,CAED,OAAM,IAAI,qBACR,4BACE,WAAW,MACT,cACE,UAAU,gBAAgB,UAAU,YACvC,CAAC,KACH,MAAM,UAAU,OAClB;AAMH,aAAO,EACL,OAAO;OACL;OACA,YAAY,WAAW,KAAI,cACzB,gBAAgB,SAAS,WAAW,MAAM,CAC3C;OACD,WAAW,YACP,gBAAgB,SAAS,WAAW,MAAM,GAC1C;OACJ;OACD,EACF;;aAEI,GAAG;AAEV,cAAS;AACT,WAAM;;;GAIV,SAAS,UAAmB;AAC1B,aAAS;AACT,WAAO;KAAC;KAAO,MAAM;KAAK;;GAE7B;;CAGH,qBACE,cACA,IACA,YACA,WACA;AAGA,MAAI,eAAe,KAAK,KAAK,QAC3B,OAAM,IAAI,iBACR,sDAAsD,KAAK,KAAK,UACjE;AAEH,MACE,WAAW,WACT,eAAc,UAAA,iBAA0B,OAAO,KAAK,KAAK,QAC1D,KAAK,GAEN,OAAM,IAAI,iBACR,sDAAsD,KAAK,KAAK,QAAQ,GACzE;AAEH,MAAI,OAAA,OAAiB,UAAA,kBAA2B,aAC9C,OAAM,IAAI,iBACR,iDACD;;;AAKP,IAAa,mBAAb,cAAsC,MAAM;CAC1C,YAAY,KAAa;AACvB,QAAM,IAAI"}

package/out/zero-cache/src/services/view-syncer/tracer.js

@@ -1,7 +1,8 @@
-import { trace } from "@opentelemetry/api";
 import { version } from "../../../../otel/src/version.js";
-const tracer = trace.getTracer("view-syncer", version);
-export {
-  tracer
-};
-//# sourceMappingURL=tracer.js.map
+import { trace } from "@opentelemetry/api";
+//#region ../zero-cache/src/services/view-syncer/tracer.ts
+var tracer = trace.getTracer("view-syncer", version);
+//#endregion
+export { tracer };
+
+//# sourceMappingURL=tracer.js.map

package/out/zero-cache/src/services/view-syncer/tracer.js.map

@@ -1 +1 @@
-{"version":3,"file":"tracer.js","sources":["../../../../../../zero-cache/src/services/view-syncer/tracer.ts"],"sourcesContent":["import {trace} from '@opentelemetry/api';\nimport {version} from '../../../../otel/src/version.ts';\n\nexport const tracer = trace.getTracer('view-syncer', version);\n"],"names":[],"mappings":";;AAGO,MAAM,SAAS,MAAM,UAAU,eAAe,OAAO;"}
+{"version":3,"file":"tracer.js","names":[],"sources":["../../../../../../zero-cache/src/services/view-syncer/tracer.ts"],"sourcesContent":["import {trace} from '@opentelemetry/api';\nimport {version} from '../../../../otel/src/version.ts';\n\nexport const tracer = trace.getTracer('view-syncer', version);\n"],"mappings":";;;AAGA,IAAa,SAAS,MAAM,UAAU,eAAe,QAAQ"}

package/out/zero-cache/src/services/view-syncer/ttl-clock.js

@@ -1,15 +1,13 @@
-import "../../../../shared/src/valita.js";
-import { number } from "@badrap/valita";
-const ttlClockSchema = number();
+import { valita_exports } from "../../../../shared/src/valita.js";
+//#region ../zero-cache/src/services/view-syncer/ttl-clock.ts
+var ttlClockSchema = valita_exports.number();
 function ttlClockAsNumber(ttlClock) {
-  return ttlClock;
+	return ttlClock;
 }
 function ttlClockFromNumber(ttlClock) {
-  return ttlClock;
+	return ttlClock;
 }
-export {
-  ttlClockAsNumber,
-  ttlClockFromNumber,
-  ttlClockSchema
-};
-//# sourceMappingURL=ttl-clock.js.map
+//#endregion
+export { ttlClockAsNumber, ttlClockFromNumber, ttlClockSchema };
+
+//# sourceMappingURL=ttl-clock.js.map

package/out/zero-cache/src/services/view-syncer/ttl-clock.js.map

@@ -1 +1 @@
-{"version":3,"file":"ttl-clock.js","sources":["../../../../../../zero-cache/src/services/view-syncer/ttl-clock.ts"],"sourcesContent":["import * as v from '../../../../shared/src/valita.ts';\n\ndeclare const ttlClockTag: unique symbol;\n\nexport type TTLClock = {[ttlClockTag]: true};\n\nexport const ttlClockSchema = v.number() as v.Type<unknown> as v.Type<TTLClock>;\n\nexport function ttlClockAsNumber(ttlClock: TTLClock): number {\n  return ttlClock as unknown as number;\n}\n\nexport function ttlClockFromNumber(ttlClock: number): TTLClock {\n  return ttlClock as unknown as TTLClock;\n}\n"],"names":["v.number"],"mappings":";;AAMO,MAAM,iBAAiBA,OAAE;AAEzB,SAAS,iBAAiB,UAA4B;AAC3D,SAAO;AACT;AAEO,SAAS,mBAAmB,UAA4B;AAC7D,SAAO;AACT;"}
+{"version":3,"file":"ttl-clock.js","names":[],"sources":["../../../../../../zero-cache/src/services/view-syncer/ttl-clock.ts"],"sourcesContent":["import * as v from '../../../../shared/src/valita.ts';\n\ndeclare const ttlClockTag: unique symbol;\n\nexport type TTLClock = {[ttlClockTag]: true};\n\nexport const ttlClockSchema = v.number() as v.Type<unknown> as v.Type<TTLClock>;\n\nexport function ttlClockAsNumber(ttlClock: TTLClock): number {\n  return ttlClock as unknown as number;\n}\n\nexport function ttlClockFromNumber(ttlClock: number): TTLClock {\n  return ttlClock as unknown as TTLClock;\n}\n"],"mappings":";;AAMA,IAAa,iBAAiB,eAAE,QAAQ;AAExC,SAAgB,iBAAiB,UAA4B;AAC3D,QAAO;;AAGT,SAAgB,mBAAmB,UAA4B;AAC7D,QAAO"}