@rocicorp/zero 1.9.0-canary.0 → 1.9.0-canary.10

package/out/zero-cache/src/services/replicator/reporter/recorder.js.map

@@ -1 +1 @@
-{"version":3,"file":"recorder.js","names":["#lc","#last"],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../../observability/metrics.ts';\nimport type {ReplicationReport} from './report-schema.ts';\n\n// Hook for sanity checking lag reports in development.\nconst LOG_ALL_REPLICATION_REPORTS_AT_DEBUG =\n  process.env.ZERO_LOG_ALL_REPLICATION_REPORTS_AT_DEBUG === '1';\n\nexport class ReplicationReportRecorder {\n  readonly #lc: LogContext;\n  #last: ReplicationReport | null = null;\n\n  constructor(lc: LogContext) {\n    this.#lc = lc;\n  }\n\n  record(report: ReplicationReport) {\n    const first = this.#last === null;\n    this.#last = report;\n\n    const {lastTimings} = report;\n    if (lastTimings) {\n      const total = lastTimings.replicateTimeMs - lastTimings.sendTimeMs;\n      if (total > 10_000) {\n        this.#lc.warn?.(`high replication lag: ${total} ms`, report);\n      } else if (total > 1_000) {\n        this.#lc.info?.(`replication lag: ${total} ms`, report);\n      }\n      if (LOG_ALL_REPLICATION_REPORTS_AT_DEBUG) {\n        this.#lc.debug?.(`replication lag ${total} ms`, report);\n      }\n    }\n\n    if (first) {\n      getOrCreateGauge('replication', 'upstream_lag', {\n        description:\n          'Latency from sending an upstream replication report ' +\n          'to receiving it in the replication stream',\n        unit: 'millisecond',\n      }).addCallback(this.reportUpstreamLag);\n\n      getOrCreateGauge('replication', 'replica_lag', {\n        description:\n          'Latency from receiving an upstream replication report ' +\n          'to its reaching the replica',\n        unit: 'millisecond',\n      }).addCallback(this.reportReplicaLag);\n\n      getOrCreateGauge('replication', 'total_lag', {\n        description:\n          'Latency from sending an upstream replication report to its ' +\n          'reaching the replica. This reflects the actual measured ' +\n          'round-trip of the most recently received report and does not ' +\n          'grow when reports stop arriving.',\n        unit: 'millisecond',\n      }).addCallback(this.reportTotalLag);\n\n      getOrCreateGauge('replication', 'last_total_lag', {\n        description:\n          'Latency from sending the most recently received upstream ' +\n          'replication report to its reaching the replica. This is an alias ' +\n          'of replication.total_lag retained for dashboards that explicitly ' +\n          'want the non-extrapolated value.',\n        unit: 'millisecond',\n      }).addCallback(this.reportLastTotalLag);\n    }\n  }\n\n  readonly reportUpstreamLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.receiveTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportReplicaLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.receiveTimeMs);\n    }\n  };\n\n  readonly reportTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportLastTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n}\n"],"mappings":";;AAMA,IAAM,uCACJ,QAAQ,IAAI,8CAA8C;AAE5D,IAAa,4BAAb,MAAuC;CACrC;CACA,QAAkC;CAElC,YAAY,IAAgB;EAC1B,KAAKA,MAAM;CACb;CAEA,OAAO,QAA2B;EAChC,MAAM,QAAQ,KAAKC,UAAU;EAC7B,KAAKA,QAAQ;EAEb,MAAM,EAAC,gBAAe;EACtB,IAAI,aAAa;GACf,MAAM,QAAQ,YAAY,kBAAkB,YAAY;GACxD,IAAI,QAAQ,KACV,KAAKD,IAAI,OAAO,yBAAyB,MAAM,MAAM,MAAM;QACtD,IAAI,QAAQ,KACjB,KAAKA,IAAI,OAAO,oBAAoB,MAAM,MAAM,MAAM;GAExD,IAAI,sCACF,KAAKA,IAAI,QAAQ,mBAAmB,MAAM,MAAM,MAAM;EAE1D;EAEA,IAAI,OAAO;GACT,iBAAiB,eAAe,gBAAgB;IAC9C,aACE;IAEF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,iBAAiB;GAErC,iBAAiB,eAAe,eAAe;IAC7C,aACE;IAEF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,gBAAgB;GAEpC,iBAAiB,eAAe,aAAa;IAC3C,aACE;IAIF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,cAAc;GAElC,iBAAiB,eAAe,kBAAkB;IAChD,aACE;IAIF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,kBAAkB;EACxC;CACF;CAEA,qBAA8B,MAAwB;EACpD,MAAM,OAAO,KAAKC,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,gBAAgB,KAAK,UAAU;CAElD;CAEA,oBAA6B,MAAwB;EACnD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,aAAa;CAEvD;CAEA,kBAA2B,MAAwB;EACjD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,sBAA+B,MAAwB;EACrD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;AACF"}
+{"version":3,"file":"recorder.js","names":["#lc","#last"],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../../observability/metrics.ts';\nimport type {ReplicationReport} from './report-schema.ts';\n\n// Hook for sanity checking lag reports in development.\nconst LOG_ALL_REPLICATION_REPORTS_AT_DEBUG =\n  process.env.ZERO_LOG_ALL_REPLICATION_REPORTS_AT_DEBUG === '1';\n\nexport class ReplicationReportRecorder {\n  readonly #lc: LogContext;\n  #last: ReplicationReport | null = null;\n\n  constructor(lc: LogContext) {\n    this.#lc = lc;\n  }\n\n  record(report: ReplicationReport) {\n    const first = this.#last === null;\n    this.#last = report;\n\n    const {lastTimings} = report;\n    if (lastTimings) {\n      const total = lastTimings.replicateTimeMs - lastTimings.sendTimeMs;\n      if (total > 10_000) {\n        this.#lc.warn?.(`high replication lag: ${total} ms`, report);\n      } else if (total > 1_000) {\n        this.#lc.info?.(`replication lag: ${total} ms`, report);\n      }\n      if (LOG_ALL_REPLICATION_REPORTS_AT_DEBUG) {\n        this.#lc.debug?.(`replication lag ${total} ms`, report);\n      }\n    }\n\n    if (first) {\n      getOrCreateGauge('replication', 'upstream_lag', {\n        description:\n          'Latency from sending an upstream replication report ' +\n          'to receiving it in the replication stream',\n        unit: 'millisecond',\n      }).addCallback(this.reportUpstreamLag);\n\n      getOrCreateGauge('replication', 'replica_lag', {\n        description:\n          'Latency from receiving an upstream replication report ' +\n          'to its reaching the replica',\n        unit: 'millisecond',\n      }).addCallback(this.reportReplicaLag);\n\n      getOrCreateGauge('replication', 'total_lag', {\n        description:\n          'Latency from sending an upstream replication report to its ' +\n          'reaching the replica. This reflects the actual measured ' +\n          'round-trip of the most recently received report and does not ' +\n          'grow when reports stop arriving.',\n        unit: 'millisecond',\n      }).addCallback(this.reportTotalLag);\n\n      getOrCreateGauge('replication', 'last_total_lag', {\n        description:\n          'Latency from sending the most recently received upstream ' +\n          'replication report to its reaching the replica. This is an alias ' +\n          'of replication.total_lag retained for dashboards that explicitly ' +\n          'want the non-extrapolated value.',\n        unit: 'millisecond',\n      }).addCallback(this.reportLastTotalLag);\n\n      getOrCreateGauge('replication', 'upstream_clock_skew', {\n        description:\n          'Estimated offset of the upstream database clock relative to ' +\n          \"zero-cache's. Positive means upstream is running ahead. \" +\n          'Derived from the lag report round-trip, whose send and receive ' +\n          'times are stamped on our clock either side of the upstream ' +\n          'commit timestamp. This matters because ' +\n          'sync.e2e_serving_lag subtracts an upstream commit timestamp from ' +\n          'a local clock, so a skewed upstream clock biases it directly -- ' +\n          'and an upstream clock running ahead biases it *low*, which looks ' +\n          'like healthy serving rather than a broken measurement.',\n        unit: 'millisecond',\n      }).addCallback(this.reportUpstreamClockSkew);\n    }\n  }\n\n  readonly reportUpstreamLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.receiveTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportReplicaLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.receiveTimeMs);\n    }\n  };\n\n  readonly reportTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportLastTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportUpstreamClockSkew = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(estimateUpstreamClockSkewMs(last));\n    }\n  };\n}\n\n/**\n * Estimates how far the upstream database's clock is from ours, in\n * milliseconds, positive meaning upstream is ahead.\n *\n * `sendTimeMs` and `receiveTimeMs` are both stamped in the\n * replication-manager, either side of `commitTimeMs`, which upstream stamps.\n * Assuming the upstream commit fell midway through that round trip, the\n * difference between where it actually landed and that midpoint is the clock\n * offset -- the same round-trip estimate NTP uses.\n *\n * The estimate is biased by any asymmetry between the two legs: the outbound\n * leg is a SQL round trip, while the return leg additionally waits on a WAL\n * flush and the walsender (which the pre-PG-17 path notes can add 50~100ms on\n * an idle database). So treat this as a detector of gross skew -- the kind\n * that invalidates sync.e2e_serving_lag -- and not as a sub-100ms correction.\n *\n * The rigorous bound, if a tighter reading is ever needed: the commit\n * genuinely happened between send and receive, so a `commitTimeMs` outside\n * that window proves skew of at least the amount by which it falls outside,\n * independent of any latency assumption.\n */\nexport function estimateUpstreamClockSkewMs({\n  sendTimeMs,\n  commitTimeMs,\n  receiveTimeMs,\n}: {\n  sendTimeMs: number;\n  commitTimeMs: number;\n  receiveTimeMs: number;\n}): number {\n  return commitTimeMs - (sendTimeMs + receiveTimeMs) / 2;\n}\n"],"mappings":";;AAMA,IAAM,uCACJ,QAAQ,IAAI,8CAA8C;AAE5D,IAAa,4BAAb,MAAuC;CACrC;CACA,QAAkC;CAElC,YAAY,IAAgB;EAC1B,KAAKA,MAAM;CACb;CAEA,OAAO,QAA2B;EAChC,MAAM,QAAQ,KAAKC,UAAU;EAC7B,KAAKA,QAAQ;EAEb,MAAM,EAAC,gBAAe;EACtB,IAAI,aAAa;GACf,MAAM,QAAQ,YAAY,kBAAkB,YAAY;GACxD,IAAI,QAAQ,KACV,KAAKD,IAAI,OAAO,yBAAyB,MAAM,MAAM,MAAM;QACtD,IAAI,QAAQ,KACjB,KAAKA,IAAI,OAAO,oBAAoB,MAAM,MAAM,MAAM;GAExD,IAAI,sCACF,KAAKA,IAAI,QAAQ,mBAAmB,MAAM,MAAM,MAAM;EAE1D;EAEA,IAAI,OAAO;GACT,iBAAiB,eAAe,gBAAgB;IAC9C,aACE;IAEF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,iBAAiB;GAErC,iBAAiB,eAAe,eAAe;IAC7C,aACE;IAEF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,gBAAgB;GAEpC,iBAAiB,eAAe,aAAa;IAC3C,aACE;IAIF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,cAAc;GAElC,iBAAiB,eAAe,kBAAkB;IAChD,aACE;IAIF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,kBAAkB;GAEtC,iBAAiB,eAAe,uBAAuB;IACrD,aACE;IASF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,uBAAuB;EAC7C;CACF;CAEA,qBAA8B,MAAwB;EACpD,MAAM,OAAO,KAAKC,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,gBAAgB,KAAK,UAAU;CAElD;CAEA,oBAA6B,MAAwB;EACnD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,aAAa;CAEvD;CAEA,kBAA2B,MAAwB;EACjD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,sBAA+B,MAAwB;EACrD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,2BAAoC,MAAwB;EAC1D,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,4BAA4B,IAAI,CAAC;CAE/C;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,SAAgB,4BAA4B,EAC1C,YACA,cACA,iBAKS;CACT,OAAO,gBAAgB,aAAa,iBAAiB;AACvD"}

package/out/zero-cache/src/services/replicator/reporter/report-schema.d.ts

@@ -21,8 +21,8 @@ export declare const replicationTimingsSchema: v.ObjectType<Omit<{
 }, undefined>;
 export declare const replicationReportSchema: v.ObjectType<{
     lastTimings: v.Optional<{
-        sendTimeMs: number;
         commitTimeMs: number;
+        sendTimeMs: number;
         receiveTimeMs: number;
         replicateTimeMs: number;
     }>;

package/out/zero-cache/src/services/replicator/write-worker.js

@@ -4,6 +4,7 @@ import { getSubscriptionState } from "./schema/replication-state.js";
 import { StatementRunner } from "../../db/statements.js";
 import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics, registerSQLiteCorruptionDiagnosticTarget } from "../../db/sqlite-corruption.js";
 import { createLogContext } from "../../server/logging.js";
+import { deleteLiteDB } from "../../db/delete-lite-db.js";
 import { ChangeProcessor } from "./change-processor.js";
 import { applyPragmas, serializeError } from "./write-worker-client.js";
 import { parentPort } from "node:worker_threads";
@@ -18,12 +19,19 @@ function createAPI() {
 	let lc;
 	let replicaDbPath;
 	let unregisterCorruptionDiagnosticTarget;
-	function logCorruptionDiagnostics(err) {
-		if (lc && replicaDbPath && isSQLiteCorruption(err)) logSQLiteCorruptionDiagnostics(lc, "write-worker", replicaDbPath, err);
+	function handleCorruptedDb(err) {
+		if (!lc || !replicaDbPath || !isSQLiteCorruption(err)) return;
+		logSQLiteCorruptionDiagnostics(lc, "write-worker", replicaDbPath, err);
+		try {
+			lc.warn?.(`deleting corrupted db at ${replicaDbPath}`);
+			deleteLiteDB(replicaDbPath);
+		} catch (e) {
+			lc.warn?.(`error deleting corrupted db at ${replicaDbPath}`, e);
+		}
 	}
 	function createProcessor() {
 		processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {
-			logCorruptionDiagnostics(err);
+			handleCorruptedDb(err);
 			port.postMessage({ writeError: serializeError(err) });
 		});
 	}
@@ -43,7 +51,7 @@ function createAPI() {
 				mode = cpMode;
 				createProcessor();
 			} catch (e) {
-				logCorruptionDiagnostics(e);
+				handleCorruptedDb(e);
 				throw e;
 			}
 		},
@@ -51,7 +59,7 @@ function createAPI() {
 			try {
 				return getSubscriptionState(must(runner));
 			} catch (e) {
-				logCorruptionDiagnostics(e);
+				handleCorruptedDb(e);
 				throw e;
 			}
 		},
@@ -59,7 +67,7 @@ function createAPI() {
 			try {
 				return must(processor).processMessage(must(lc), downstream);
 			} catch (e) {
-				logCorruptionDiagnostics(e);
+				handleCorruptedDb(e);
 				throw e;
 			}
 		},

package/out/zero-cache/src/services/replicator/write-worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"write-worker.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker.ts"],"sourcesContent":["import {parentPort} from 'node:worker_threads';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n  registerSQLiteCorruptionDiagnosticTarget,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {createLogContext} from '../../server/logging.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {ChangeProcessor, type ChangeProcessorMode} from './change-processor.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport {\n  applyPragmas,\n  serializeError,\n  type ArgsMap,\n  type Method,\n  type PragmaConfig,\n  type Request,\n  type Response,\n  type ResultMap,\n  type WriteError,\n} from './write-worker-client.ts';\n\nif (!parentPort) {\n  throw new Error('write-worker must be run as a worker thread');\n}\n\nconst port = parentPort;\n\ntype API = {[M in Method]: (...args: ArgsMap[M]) => ResultMap[M]};\n\nfunction createAPI(): API {\n  let db: Database | undefined;\n  let runner: StatementRunner | undefined;\n  let processor: ChangeProcessor | undefined;\n  let mode: ChangeProcessorMode | undefined;\n  let lc: LogContext | undefined;\n  let replicaDbPath: string | undefined;\n  let unregisterCorruptionDiagnosticTarget: (() => void) | undefined;\n\n  function logCorruptionDiagnostics(err: unknown) {\n    if (lc && replicaDbPath && isSQLiteCorruption(err)) {\n      logSQLiteCorruptionDiagnostics(lc, 'write-worker', replicaDbPath, err);\n    }\n  }\n\n  function createProcessor() {\n    processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {\n      logCorruptionDiagnostics(err);\n      port.postMessage({\n        writeError: serializeError(err),\n      } satisfies WriteError);\n    });\n  }\n\n  return {\n    init(\n      dbPath: string,\n      cpMode: ChangeProcessorMode,\n      pragmas: PragmaConfig,\n      logConfig: LogConfig,\n    ) {\n      replicaDbPath = dbPath;\n      lc = createLogContext({log: logConfig}, 'write-worker');\n      unregisterCorruptionDiagnosticTarget?.();\n      unregisterCorruptionDiagnosticTarget =\n        registerSQLiteCorruptionDiagnosticTarget({\n          debugName: 'write-worker',\n          dbPath,\n        });\n      try {\n        db = new Database(lc, dbPath);\n        applyPragmas(db, pragmas);\n        runner = new StatementRunner(db);\n        mode = cpMode;\n        createProcessor();\n      } catch (e) {\n        logCorruptionDiagnostics(e);\n        throw e;\n      }\n    },\n\n    getSubscriptionState() {\n      try {\n        return getSubscriptionState(must(runner));\n      } catch (e) {\n        logCorruptionDiagnostics(e);\n        throw e;\n      }\n    },\n\n    processMessage(downstream: ChangeStreamData) {\n      try {\n        return must(processor).processMessage(must(lc), downstream);\n      } catch (e) {\n        logCorruptionDiagnostics(e);\n        throw e;\n      }\n    },\n\n    abort() {\n      must(processor).abort(must(lc));\n      createProcessor();\n    },\n\n    stop() {\n      db?.close();\n      db = undefined;\n      runner = undefined;\n      processor = undefined;\n      replicaDbPath = undefined;\n      unregisterCorruptionDiagnosticTarget?.();\n      unregisterCorruptionDiagnosticTarget = undefined;\n    },\n  };\n}\n\nconst api = createAPI();\n\nport.on('message', (msg: Request) => {\n  try {\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any -- TS can't narrow msg.method + msg.args together\n    const result = (api[msg.method] as (...args: any[]) => unknown)(\n      ...msg.args,\n    );\n    // abort is fire-and-forget — no pending slot on the client side.\n    if (msg.method !== 'abort') {\n      port.postMessage({method: msg.method, result} as Response);\n    }\n  } catch (e) {\n    if (msg.method !== 'abort') {\n      port.postMessage({\n        method: msg.method,\n        error: serializeError(e),\n      } as Response);\n    }\n  }\n});\n"],"mappings":";;;;;;;;;;AA2BA,IAAI,CAAC,YACH,MAAM,IAAI,MAAM,6CAA6C;AAG/D,IAAM,OAAO;AAIb,SAAS,YAAiB;CACxB,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CAEJ,SAAS,yBAAyB,KAAc;EAC9C,IAAI,MAAM,iBAAiB,mBAAmB,GAAG,GAC/C,+BAA+B,IAAI,gBAAgB,eAAe,GAAG;CAEzE;CAEA,SAAS,kBAAkB;EACzB,YAAY,IAAI,gBAAgB,KAAK,MAAM,GAAG,KAAK,IAAI,IAAI,KAAK,QAAQ;GACtE,yBAAyB,GAAG;GAC5B,KAAK,YAAY,EACf,YAAY,eAAe,GAAG,EAChC,CAAsB;EACxB,CAAC;CACH;CAEA,OAAO;EACL,KACE,QACA,QACA,SACA,WACA;GACA,gBAAgB;GAChB,KAAK,iBAAiB,EAAC,KAAK,UAAS,GAAG,cAAc;GACtD,uCAAuC;GACvC,uCACE,yCAAyC;IACvC,WAAW;IACX;GACF,CAAC;GACH,IAAI;IACF,KAAK,IAAI,SAAS,IAAI,MAAM;IAC5B,aAAa,IAAI,OAAO;IACxB,SAAS,IAAI,gBAAgB,EAAE;IAC/B,OAAO;IACP,gBAAgB;GAClB,SAAS,GAAG;IACV,yBAAyB,CAAC;IAC1B,MAAM;GACR;EACF;EAEA,uBAAuB;GACrB,IAAI;IACF,OAAO,qBAAqB,KAAK,MAAM,CAAC;GAC1C,SAAS,GAAG;IACV,yBAAyB,CAAC;IAC1B,MAAM;GACR;EACF;EAEA,eAAe,YAA8B;GAC3C,IAAI;IACF,OAAO,KAAK,SAAS,EAAE,eAAe,KAAK,EAAE,GAAG,UAAU;GAC5D,SAAS,GAAG;IACV,yBAAyB,CAAC;IAC1B,MAAM;GACR;EACF;EAEA,QAAQ;GACN,KAAK,SAAS,EAAE,MAAM,KAAK,EAAE,CAAC;GAC9B,gBAAgB;EAClB;EAEA,OAAO;GACL,IAAI,MAAM;GACV,KAAK,KAAA;GACL,SAAS,KAAA;GACT,YAAY,KAAA;GACZ,gBAAgB,KAAA;GAChB,uCAAuC;GACvC,uCAAuC,KAAA;EACzC;CACF;AACF;AAEA,IAAM,MAAM,UAAU;AAEtB,KAAK,GAAG,YAAY,QAAiB;CACnC,IAAI;EAEF,MAAM,SAAU,IAAI,IAAI,QACtB,GAAG,IAAI,IACT;EAEA,IAAI,IAAI,WAAW,SACjB,KAAK,YAAY;GAAC,QAAQ,IAAI;GAAQ;EAAM,CAAa;CAE7D,SAAS,GAAG;EACV,IAAI,IAAI,WAAW,SACjB,KAAK,YAAY;GACf,QAAQ,IAAI;GACZ,OAAO,eAAe,CAAC;EACzB,CAAa;CAEjB;AACF,CAAC"}
+{"version":3,"file":"write-worker.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker.ts"],"sourcesContent":["import {parentPort} from 'node:worker_threads';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n  registerSQLiteCorruptionDiagnosticTarget,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {createLogContext} from '../../server/logging.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {ChangeProcessor, type ChangeProcessorMode} from './change-processor.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport {\n  applyPragmas,\n  serializeError,\n  type ArgsMap,\n  type Method,\n  type PragmaConfig,\n  type Request,\n  type Response,\n  type ResultMap,\n  type WriteError,\n} from './write-worker-client.ts';\n\nif (!parentPort) {\n  throw new Error('write-worker must be run as a worker thread');\n}\n\nconst port = parentPort;\n\ntype API = {[M in Method]: (...args: ArgsMap[M]) => ResultMap[M]};\n\nfunction createAPI(): API {\n  let db: Database | undefined;\n  let runner: StatementRunner | undefined;\n  let processor: ChangeProcessor | undefined;\n  let mode: ChangeProcessorMode | undefined;\n  let lc: LogContext | undefined;\n  let replicaDbPath: string | undefined;\n  let unregisterCorruptionDiagnosticTarget: (() => void) | undefined;\n\n  function handleCorruptedDb(err: unknown) {\n    if (!lc || !replicaDbPath || !isSQLiteCorruption(err)) {\n      return;\n    }\n    logSQLiteCorruptionDiagnostics(lc, 'write-worker', replicaDbPath, err);\n    try {\n      lc.warn?.(`deleting corrupted db at ${replicaDbPath}`);\n      deleteLiteDB(replicaDbPath);\n    } catch (e) {\n      lc.warn?.(`error deleting corrupted db at ${replicaDbPath}`, e);\n    }\n  }\n\n  function createProcessor() {\n    processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {\n      handleCorruptedDb(err);\n      port.postMessage({\n        writeError: serializeError(err),\n      } satisfies WriteError);\n    });\n  }\n\n  return {\n    init(\n      dbPath: string,\n      cpMode: ChangeProcessorMode,\n      pragmas: PragmaConfig,\n      logConfig: LogConfig,\n    ) {\n      replicaDbPath = dbPath;\n      lc = createLogContext({log: logConfig}, 'write-worker');\n      unregisterCorruptionDiagnosticTarget?.();\n      unregisterCorruptionDiagnosticTarget =\n        registerSQLiteCorruptionDiagnosticTarget({\n          debugName: 'write-worker',\n          dbPath,\n        });\n      try {\n        db = new Database(lc, dbPath);\n        applyPragmas(db, pragmas);\n        runner = new StatementRunner(db);\n        mode = cpMode;\n        createProcessor();\n      } catch (e) {\n        handleCorruptedDb(e);\n        throw e;\n      }\n    },\n\n    getSubscriptionState() {\n      try {\n        return getSubscriptionState(must(runner));\n      } catch (e) {\n        handleCorruptedDb(e);\n        throw e;\n      }\n    },\n\n    processMessage(downstream: ChangeStreamData) {\n      try {\n        return must(processor).processMessage(must(lc), downstream);\n      } catch (e) {\n        handleCorruptedDb(e);\n        throw e;\n      }\n    },\n\n    abort() {\n      must(processor).abort(must(lc));\n      createProcessor();\n    },\n\n    stop() {\n      db?.close();\n      db = undefined;\n      runner = undefined;\n      processor = undefined;\n      replicaDbPath = undefined;\n      unregisterCorruptionDiagnosticTarget?.();\n      unregisterCorruptionDiagnosticTarget = undefined;\n    },\n  };\n}\n\nconst api = createAPI();\n\nport.on('message', (msg: Request) => {\n  try {\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any -- TS can't narrow msg.method + msg.args together\n    const result = (api[msg.method] as (...args: any[]) => unknown)(\n      ...msg.args,\n    );\n    // abort is fire-and-forget — no pending slot on the client side.\n    if (msg.method !== 'abort') {\n      port.postMessage({method: msg.method, result} as Response);\n    }\n  } catch (e) {\n    if (msg.method !== 'abort') {\n      port.postMessage({\n        method: msg.method,\n        error: serializeError(e),\n      } as Response);\n    }\n  }\n});\n"],"mappings":";;;;;;;;;;;AA4BA,IAAI,CAAC,YACH,MAAM,IAAI,MAAM,6CAA6C;AAG/D,IAAM,OAAO;AAIb,SAAS,YAAiB;CACxB,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CAEJ,SAAS,kBAAkB,KAAc;EACvC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,mBAAmB,GAAG,GAClD;EAEF,+BAA+B,IAAI,gBAAgB,eAAe,GAAG;EACrE,IAAI;GACF,GAAG,OAAO,4BAA4B,eAAe;GACrD,aAAa,aAAa;EAC5B,SAAS,GAAG;GACV,GAAG,OAAO,kCAAkC,iBAAiB,CAAC;EAChE;CACF;CAEA,SAAS,kBAAkB;EACzB,YAAY,IAAI,gBAAgB,KAAK,MAAM,GAAG,KAAK,IAAI,IAAI,KAAK,QAAQ;GACtE,kBAAkB,GAAG;GACrB,KAAK,YAAY,EACf,YAAY,eAAe,GAAG,EAChC,CAAsB;EACxB,CAAC;CACH;CAEA,OAAO;EACL,KACE,QACA,QACA,SACA,WACA;GACA,gBAAgB;GAChB,KAAK,iBAAiB,EAAC,KAAK,UAAS,GAAG,cAAc;GACtD,uCAAuC;GACvC,uCACE,yCAAyC;IACvC,WAAW;IACX;GACF,CAAC;GACH,IAAI;IACF,KAAK,IAAI,SAAS,IAAI,MAAM;IAC5B,aAAa,IAAI,OAAO;IACxB,SAAS,IAAI,gBAAgB,EAAE;IAC/B,OAAO;IACP,gBAAgB;GAClB,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,uBAAuB;GACrB,IAAI;IACF,OAAO,qBAAqB,KAAK,MAAM,CAAC;GAC1C,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,eAAe,YAA8B;GAC3C,IAAI;IACF,OAAO,KAAK,SAAS,EAAE,eAAe,KAAK,EAAE,GAAG,UAAU;GAC5D,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,QAAQ;GACN,KAAK,SAAS,EAAE,MAAM,KAAK,EAAE,CAAC;GAC9B,gBAAgB;EAClB;EAEA,OAAO;GACL,IAAI,MAAM;GACV,KAAK,KAAA;GACL,SAAS,KAAA;GACT,YAAY,KAAA;GACZ,gBAAgB,KAAA;GAChB,uCAAuC;GACvC,uCAAuC,KAAA;EACzC;CACF;AACF;AAEA,IAAM,MAAM,UAAU;AAEtB,KAAK,GAAG,YAAY,QAAiB;CACnC,IAAI;EAEF,MAAM,SAAU,IAAI,IAAI,QACtB,GAAG,IAAI,IACT;EAEA,IAAI,IAAI,WAAW,SACjB,KAAK,YAAY;GAAC,QAAQ,IAAI;GAAQ;EAAM,CAAa;CAE7D,SAAS,GAAG;EACV,IAAI,IAAI,WAAW,SACjB,KAAK,YAAY;GACf,QAAQ,IAAI;GACZ,OAAO,eAAe,CAAC;EACzB,CAAa;CAEjB;AACF,CAAC"}

package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.d.ts

@@ -0,0 +1,44 @@
+import type { ReplicaState } from '../replicator/replicator.ts';
+export type PendingUpstreamCommit = {
+    readonly watermark: string;
+    readonly commitTimeMs: number;
+};
+/**
+ * Pairs the upstream commit timestamps carried by `version-ready`
+ * notifications with the moment the corresponding change is poked to clients,
+ * yielding the end-to-end serving lag.
+ *
+ * This measures completion, not backlog: an observation is produced only when
+ * work actually reaches clients, so the resulting histogram is a latency
+ * distribution rather than a periodic snapshot of how far behind things are.
+ * A ViewSyncer that is stuck contributes nothing until it recovers, instead of
+ * re-reporting its age on every sample tick.
+ */
+export declare class E2EServingLagTracker {
+    #private;
+    get pending(): PendingUpstreamCommit | null;
+    /**
+     * Records the upstream commit behind a `version-ready` notification.
+     *
+     * Notifications coalesce when the ViewSyncer is busy, so one state may stand
+     * in for several commits. The *oldest* commit time is kept, since it bounds
+     * the lag of everything the notification subsumed, while the watermark
+     * advances to the newest — that is the one that must be served for all of
+     * the subsumed commits to have been delivered.
+     */
+    onVersionReady({ watermark, upstreamCommitTimeMs }: ReplicaState): void;
+    /**
+     * Called once a version has been poked to clients.
+     *
+     * @return the observation to record, or `null` if the served version does
+     *     not yet cover an outstanding upstream commit.
+     */
+    onVersionServed(servedVersion: string, nowMs: number): Observation | null;
+}
+export type Observation = {
+    /** End-to-end lag in milliseconds, clamped to be non-negative. */
+    readonly lagMs: number;
+    /** Whether the clamp was applied, i.e. the raw measurement was negative. */
+    readonly clamped: boolean;
+};
+//# sourceMappingURL=e2e-serving-lag.d.ts.map

package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"e2e-serving-lag.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/e2e-serving-lag.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAE9D,MAAM,MAAM,qBAAqB,GAAG;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,oBAAoB;;IAG/B,IAAI,OAAO,IAAI,qBAAqB,GAAG,IAAI,CAE1C;IAED;;;;;;;;OAQG;IACH,cAAc,CAAC,EAAC,SAAS,EAAE,oBAAoB,EAAC,EAAE,YAAY,GAAG,IAAI;IAcrE;;;;;OAKG;IACH,eAAe,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI;CAoB1E;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,kEAAkE;IAClE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,4EAA4E;IAC5E,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B,CAAC"}

package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.js

@@ -0,0 +1,59 @@
+//#region ../zero-cache/src/services/view-syncer/e2e-serving-lag.ts
+/**
+* Pairs the upstream commit timestamps carried by `version-ready`
+* notifications with the moment the corresponding change is poked to clients,
+* yielding the end-to-end serving lag.
+*
+* This measures completion, not backlog: an observation is produced only when
+* work actually reaches clients, so the resulting histogram is a latency
+* distribution rather than a periodic snapshot of how far behind things are.
+* A ViewSyncer that is stuck contributes nothing until it recovers, instead of
+* re-reporting its age on every sample tick.
+*/
+var E2EServingLagTracker = class {
+	#pending = null;
+	get pending() {
+		return this.#pending;
+	}
+	/**
+	* Records the upstream commit behind a `version-ready` notification.
+	*
+	* Notifications coalesce when the ViewSyncer is busy, so one state may stand
+	* in for several commits. The *oldest* commit time is kept, since it bounds
+	* the lag of everything the notification subsumed, while the watermark
+	* advances to the newest — that is the one that must be served for all of
+	* the subsumed commits to have been delivered.
+	*/
+	onVersionReady({ watermark, upstreamCommitTimeMs }) {
+		if (watermark === void 0 || upstreamCommitTimeMs === void 0) return;
+		const pending = this.#pending;
+		this.#pending = {
+			watermark,
+			commitTimeMs: pending === null ? upstreamCommitTimeMs : Math.min(pending.commitTimeMs, upstreamCommitTimeMs)
+		};
+	}
+	/**
+	* Called once a version has been poked to clients.
+	*
+	* @return the observation to record, or `null` if the served version does
+	*     not yet cover an outstanding upstream commit.
+	*/
+	onVersionServed(servedVersion, nowMs) {
+		const pending = this.#pending;
+		if (pending === null || servedVersion < pending.watermark) return null;
+		this.#pending = null;
+		const lagMs = nowMs - pending.commitTimeMs;
+		if (lagMs >= 0) return {
+			lagMs,
+			clamped: false
+		};
+		return {
+			lagMs: 0,
+			clamped: true
+		};
+	}
+};
+//#endregion
+export { E2EServingLagTracker };
+
+//# sourceMappingURL=e2e-serving-lag.js.map

package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"e2e-serving-lag.js","names":["#pending"],"sources":["../../../../../../zero-cache/src/services/view-syncer/e2e-serving-lag.ts"],"sourcesContent":["import type {ReplicaState} from '../replicator/replicator.ts';\n\nexport type PendingUpstreamCommit = {\n  readonly watermark: string;\n  readonly commitTimeMs: number;\n};\n\n/**\n * Pairs the upstream commit timestamps carried by `version-ready`\n * notifications with the moment the corresponding change is poked to clients,\n * yielding the end-to-end serving lag.\n *\n * This measures completion, not backlog: an observation is produced only when\n * work actually reaches clients, so the resulting histogram is a latency\n * distribution rather than a periodic snapshot of how far behind things are.\n * A ViewSyncer that is stuck contributes nothing until it recovers, instead of\n * re-reporting its age on every sample tick.\n */\nexport class E2EServingLagTracker {\n  #pending: PendingUpstreamCommit | null = null;\n\n  get pending(): PendingUpstreamCommit | null {\n    return this.#pending;\n  }\n\n  /**\n   * Records the upstream commit behind a `version-ready` notification.\n   *\n   * Notifications coalesce when the ViewSyncer is busy, so one state may stand\n   * in for several commits. The *oldest* commit time is kept, since it bounds\n   * the lag of everything the notification subsumed, while the watermark\n   * advances to the newest — that is the one that must be served for all of\n   * the subsumed commits to have been delivered.\n   */\n  onVersionReady({watermark, upstreamCommitTimeMs}: ReplicaState): void {\n    if (watermark === undefined || upstreamCommitTimeMs === undefined) {\n      return;\n    }\n    const pending = this.#pending;\n    this.#pending = {\n      watermark,\n      commitTimeMs:\n        pending === null\n          ? upstreamCommitTimeMs\n          : Math.min(pending.commitTimeMs, upstreamCommitTimeMs),\n    };\n  }\n\n  /**\n   * Called once a version has been poked to clients.\n   *\n   * @return the observation to record, or `null` if the served version does\n   *     not yet cover an outstanding upstream commit.\n   */\n  onVersionServed(servedVersion: string, nowMs: number): Observation | null {\n    const pending = this.#pending;\n    if (pending === null || servedVersion < pending.watermark) {\n      return null;\n    }\n    this.#pending = null;\n\n    const lagMs = nowMs - pending.commitTimeMs;\n    if (lagMs >= 0) {\n      return {lagMs, clamped: false};\n    }\n    // The commit time is on the upstream database's clock while `nowMs` is\n    // local, so a negative duration means upstream's clock is running ahead of\n    // ours by more than the entire pipeline latency. Clamp, because a negative\n    // value would corrupt the histogram's sum -- but report the clamp, since\n    // it is proof of gross clock skew, and skew in this direction biases the\n    // metric *low*, which reads as healthy serving rather than as a broken\n    // measurement. See replication.upstream_clock_skew for the magnitude.\n    return {lagMs: 0, clamped: true};\n  }\n}\n\nexport type Observation = {\n  /** End-to-end lag in milliseconds, clamped to be non-negative. */\n  readonly lagMs: number;\n  /** Whether the clamp was applied, i.e. the raw measurement was negative. */\n  readonly clamped: boolean;\n};\n"],"mappings":";;;;;;;;;;;;AAkBA,IAAa,uBAAb,MAAkC;CAChC,WAAyC;CAEzC,IAAI,UAAwC;EAC1C,OAAO,KAAKA;CACd;;;;;;;;;;CAWA,eAAe,EAAC,WAAW,wBAA2C;EACpE,IAAI,cAAc,KAAA,KAAa,yBAAyB,KAAA,GACtD;EAEF,MAAM,UAAU,KAAKA;EACrB,KAAKA,WAAW;GACd;GACA,cACE,YAAY,OACR,uBACA,KAAK,IAAI,QAAQ,cAAc,oBAAoB;EAC3D;CACF;;;;;;;CAQA,gBAAgB,eAAuB,OAAmC;EACxE,MAAM,UAAU,KAAKA;EACrB,IAAI,YAAY,QAAQ,gBAAgB,QAAQ,WAC9C,OAAO;EAET,KAAKA,WAAW;EAEhB,MAAM,QAAQ,QAAQ,QAAQ;EAC9B,IAAI,SAAS,GACX,OAAO;GAAC;GAAO,SAAS;EAAK;EAS/B,OAAO;GAAC,OAAO;GAAG,SAAS;EAAI;CACjC;AACF"}

package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAY,KAAK,SAAS,EAAC,MAAM,uCAAuC,CAAC;AAEhF,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,uCAAuC,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sCAAsC,CAAC;AAGrE,OAAO,KAAK,EACV,cAAc,EACd,+BAA+B,EAChC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAEL,KAAK,MAAM,EACX,KAAK,QAAQ,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,uBAAuB,CAAC;AAajD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,qBAAa,WAAW;;gBASpB,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAC,KAAK,EAAC,EAAE,KAAK,EACd,gBAAgB,CAAC,EAAE,MAAM;IAQ3B;;;;OAIG;IACH,IAAI,IAAI,IAAI;IAYZ,WAAW,IAAI,OAAO;IAItB,+EAA+E;IAC/E,OAAO,IAAI,QAAQ;IAKnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoCG;IACH,OAAO,CACL,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EAC3C,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,GACzB,YAAY;IAKf,kBAAkB;;;;IAelB;;;OAGG;IACH,OAAO;CAKR;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,WAAW,YAAa,SAAQ,QAAQ,CAAC,MAAM,CAAC;IACpD,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IAEF;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;GAIG;AACH,MAAM,MAAM,oBAAoB,GAC5B,qBAAqB,GACrB,iBAAiB,GACjB,eAAe,GACf,YAAY,GACZ,oBAAoB,CAAC;AAEzB,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,IAAI,0BAA0B;IACvC,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAC;gBAE1B,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB;CAItD;AAED,cAAM,QAAQ;;IACZ,MAAM,CAAC,MAAM,CACX,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,gBAAgB,EAAE,MAAM,GAAG,SAAS;IAsBtC,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;IAE7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;gBAEb,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM;IAY9C,eAAe,CAAC,WAAW,EAAE,MAAM;IAQnC,YAAY,CAAC,WAAW,EAAE,MAAM;;;;IAahC,MAAM,CAAC,KAAK,EAAE,+BAA+B,EAAE,MAAM,EAAE,SAAS;IAkBhE,OAAO,CACL,KAAK,EAAE,+BAA+B,EACtC,IAAI,EAAE,UAAU,EAAE,EAClB,GAAG,EAAE,QAAQ;IAgCf,WAAW,IAAI,QAAQ;CAIxB;AA8LD,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,GAAG,EAAE,MAAM;CAGxB"}
+{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAY,KAAK,SAAS,EAAC,MAAM,uCAAuC,CAAC;AAEhF,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,uCAAuC,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sCAAsC,CAAC;AAGrE,OAAO,KAAK,EACV,cAAc,EACd,+BAA+B,EAChC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAEL,KAAK,MAAM,EACX,KAAK,QAAQ,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,uBAAuB,CAAC;AAajD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,qBAAa,WAAW;;gBASpB,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAC,KAAK,EAAC,EAAE,KAAK,EACd,gBAAgB,CAAC,EAAE,MAAM;IAQ3B;;;;OAIG;IACH,IAAI,IAAI,IAAI;IAYZ,WAAW,IAAI,OAAO;IAItB,+EAA+E;IAC/E,OAAO,IAAI,QAAQ;IAKnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoCG;IACH,OAAO,CACL,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EAC3C,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,GACzB,YAAY;IAKf,kBAAkB;;;;IAelB;;;OAGG;IACH,OAAO;CAKR;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,WAAW,YAAa,SAAQ,QAAQ,CAAC,MAAM,CAAC;IACpD,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IAEF;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;GAIG;AACH,MAAM,MAAM,oBAAoB,GAC5B,qBAAqB,GACrB,iBAAiB,GACjB,eAAe,GACf,YAAY,GACZ,oBAAoB,CAAC;AAEzB,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,IAAI,0BAA0B;IACvC,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAC;gBAE1B,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB;CAItD;AAED,cAAM,QAAQ;;IACZ,MAAM,CAAC,MAAM,CACX,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,gBAAgB,EAAE,MAAM,GAAG,SAAS;IAuBtC,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;IAE7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;gBAEb,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM;IAY9C,eAAe,CAAC,WAAW,EAAE,MAAM;IAQnC,YAAY,CAAC,WAAW,EAAE,MAAM;;;;IAahC,MAAM,CAAC,KAAK,EAAE,+BAA+B,EAAE,MAAM,EAAE,SAAS;IAkBhE,OAAO,CACL,KAAK,EAAE,+BAA+B,EACtC,IAAI,EAAE,UAAU,EAAE,EAClB,GAAG,EAAE,QAAQ;IAgCf,WAAW,IAAI,QAAQ;CAIxB;AA8LD,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,GAAG,EAAE,MAAM;CAGxB"}

package/out/zero-cache/src/services/view-syncer/snapshotter.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshotter.js","names":["#lc","#dbFile","#appID","#pageCacheSizeKib","#curr","#prev","#permissionsTable","#syncableTables","#allTableNames"],"sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {stringify, type JSONValue} from '../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Row} from '../../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../../zero-types/src/schema.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {fromSQLiteTypes} from '../../../../zqlite/src/table-source.ts';\nimport type {\n  LiteAndZqlSpec,\n  LiteTableSpecWithKeysAndVersion,\n} from '../../db/specs.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {\n  normalizedKeyOrder,\n  type RowKey,\n  type RowValue,\n} from '../../types/row-key.ts';\nimport type {AppID} from '../../types/shards.ts';\nimport {id} from '../../types/sql.ts';\nimport {\n  RESET_OP,\n  changeLogEntrySchema as schema,\n  SET_OP,\n  TRUNCATE_OP,\n} from '../replicator/schema/change-log.ts';\nimport {\n  getReplicationState,\n  ZERO_VERSION_COLUMN_NAME as ROW_VERSION,\n} from '../replicator/schema/replication-state.ts';\n\n/**\n * A `Snapshotter` manages the progression of database snapshots for a\n * ViewSyncer.\n *\n * The Replicator and ViewSyncers operate on the same SQLite file, with the\n * Replicator being the sole writer to the database. The IVM logic in\n * ViewSyncers, however, rely on incrementally applying changes to the DB to\n * update the state of its pipelines.\n *\n * To avoid coupling the progress of the Replicator and all IVM pipelines on\n * each other, ViewSyncers operate on ephemeral forks of the database by holding\n * [concurrent](https://sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md)\n * snapshots of the database and simulating (but ultimately rolling back)\n * mutations on these snapshots.\n *\n * Example:\n * 1. ViewSyncer takes `snapshot_a` at version `t1` of the database and\n *    hydrates its pipeline(s).\n * 2. Replicator applies a new transaction to the database and notifies\n *    subscribers.\n * 3. ViewSyncer takes `snapshot_b` at `t2`, and queries the `ChangeLog` at\n *    that snapshot for changes since `t1`.\n * 4. ViewSyncer applies those changes to `snapshot_a` for IVM, but does not\n *    commit them. (Recall that the Replicator is the sole writer to the db, so\n *    the ViewSyncer never commits any writes.)\n * 5. Replicator applies the next transaction and advances the database to `t3`.\n * 6. ViewSyncer rolls back `snapshot_a` and opens `snapshot_c` at `t3`, using\n *    `snapshot_b` to simulate changes from `t2` to `t3`.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ```\n *\n * Note that the Replicator (and ViewSyncers) do not wait on the progress of\n * other ViewSyncers. If a ViewSyncer is busy hydrating at `t1`, the Replicator\n * and other ViewSyncers can progress through `t2`, `t3`, etc. independently,\n * as the busy ViewSyncer simply takes its own snapshot when it is ready.\n *\n * ```\n * Replicator:   t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer1:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ViewSyncer2:       [.......... snapshot_a ..........] ----> [snapshot_b]\n * ```\n *\n * To minimize Database connections (and statement preparation, etc.), the\n * Snapshotter reuses the connection from the previous (rolled back)\n * snapshot when opening the new one.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n *                     (conn_1)           (conn_2)           (conn_1)\n * ```\n *\n * In this manner, each ViewSyncer uses two connections that continually\n * \"leapfrog\" each other to replay the timeline of changes in isolation from\n * the Replicator and other ViewSyncers.\n */\nexport class Snapshotter {\n  readonly #lc: LogContext;\n  readonly #dbFile: string;\n  readonly #appID: string;\n  readonly #pageCacheSizeKib: number | undefined;\n  #curr: Snapshot | undefined;\n  #prev: Snapshot | undefined;\n\n  constructor(\n    lc: LogContext,\n    dbFile: string,\n    {appID}: AppID,\n    pageCacheSizeKib?: number,\n  ) {\n    this.#lc = lc;\n    this.#dbFile = dbFile;\n    this.#appID = appID;\n    this.#pageCacheSizeKib = pageCacheSizeKib;\n  }\n\n  /**\n   * Initializes the snapshot to the current head of the database. This must be\n   * only be called once. The state of whether a Snapshotter has been initialized\n   * can be determined by calling {@link initialized()}.\n   */\n  init(): this {\n    assert(this.#curr === undefined, 'Already initialized');\n    this.#curr = Snapshot.create(\n      this.#lc,\n      this.#dbFile,\n      this.#appID,\n      this.#pageCacheSizeKib,\n    );\n    this.#lc.debug?.(`Initial snapshot at version ${this.#curr.version}`);\n    return this;\n  }\n\n  initialized(): boolean {\n    return this.#curr !== undefined;\n  }\n\n  /** Returns the current snapshot. Asserts if {@link initialized()} is false. */\n  current(): Snapshot {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    return this.#curr;\n  }\n\n  /**\n   * Advances to the head of the Database, returning a diff between the\n   * previously current Snapshot and a new Snapshot at head. This is called\n   * in response to a notification from a Replicator subscription. Subsequent\n   * calls to {@link current()} return the new Snapshot. Note that the Snapshotter\n   * must be initialized before advancing.\n   *\n   * The returned {@link SnapshotDiff} contains snapshots at the endpoints\n   * of the database timeline. Iterating over the diff generates a sequence\n   * of {@link Change}s between the two snapshots.\n   *\n   * Note that this sequence is not chronological; rather, the sequence is\n   * ordered by `<table, row-key>`, such that a row can appear at most once\n   * in the common case, or twice if its table is `TRUNCATE`'d and a new value\n   * is subsequently `INSERT`'ed. This results in dropping most intermediate\n   * changes to a row and bounds the amount of work needed to catch up;\n   * however, as a consequence, a consistent database state is only guaranteed\n   * when the sequence has been fully consumed.\n   *\n   * Note that Change generation relies on the state of the underlying\n   * database connections, and because the connection for the previous snapshot\n   * is reused to produce the next snapshot, the diff object is only valid\n   * until the next call to `advance()`.\n   *\n   * It is okay for the caller to apply `Change`s to the `prev` snapshot\n   * during the iteration (e.g. this is necessary for IVM); the remainder\n   * of the iteration is not affected because a given row can appear at most\n   * once in the sequence (with the exception being TRUNCATE, after which the\n   * deleted rows can be re-inserted, but this will also behave correctly if\n   * the changes are applied).\n   *\n   * Once the changes have been applied, however, a _subsequent_ iteration\n   * will not produce the correct results. In order to perform multiple\n   * change-applying iterations, the caller must (1) create a save point\n   * on `prev` before each iteration, and (2) rollback to the save point after\n   * the iteration.\n   */\n  advance(\n    syncableTables: Map<string, LiteAndZqlSpec>,\n    allTableNames: Set<string>,\n  ): SnapshotDiff {\n    const {prev, curr} = this.advanceWithoutDiff();\n    return new Diff(this.#appID, syncableTables, allTableNames, prev, curr);\n  }\n\n  advanceWithoutDiff() {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    const next = this.#prev\n      ? this.#prev.resetToHead()\n      : Snapshot.create(\n          this.#lc,\n          this.#curr.db.db.name,\n          this.#appID,\n          this.#pageCacheSizeKib,\n        );\n    this.#prev = this.#curr;\n    this.#curr = next;\n    return {prev: this.#prev, curr: this.#curr};\n  }\n\n  /**\n   * Call this to close the database connections when the Snapshotter is\n   * no longer needed.\n   */\n  destroy() {\n    this.#curr?.db.db.close();\n    this.#prev?.db.db.close();\n    this.#lc.debug?.('closed database connections');\n  }\n}\n\nexport type Change = {\n  readonly table: string;\n  /**\n   * If this change represents a remove the row to remove,\n   * if nextValue is not null then all rows that have a unique constraint\n   * violation with nextValue.\n   * In both cases these rows should be removed.\n   */\n  readonly prevValues: Readonly<Row>[];\n  readonly nextValue: Readonly<Row> | null;\n  readonly rowKey: RowKey;\n};\n\n/**\n * Represents the difference between two database Snapshots.\n * Iterating over the object will produce a sequence of {@link Change}s\n * between the two snapshots.\n *\n * See {@link Snapshotter.advance()} for semantics and usage.\n */\nexport interface SnapshotDiff extends Iterable<Change> {\n  readonly prev: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n  readonly curr: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n\n  /**\n   * The number of ChangeLog entries between the snapshots. Note that this\n   * may not necessarily equal the number of `Change` objects that the iteration\n   * will produce, as `TRUNCATE` entries are counted as a single log entry which\n   * may be expanded into many changes (i.e. row deletes).\n   *\n   * TODO: Determine if it is worth changing the definition to count the\n   *       truncated rows. This would make diff computation more expensive\n   *       (requiring the count to be aggregated by operation type), which\n   *       may not be worth it for a presumable rare operation.\n   */\n  readonly changes: number;\n}\n\n/**\n * Thrown during an iteration of a {@link SnapshotDiff} when a schema\n * change or truncate is encountered, which result in aborting the\n * advancement and resetting / rehydrating the pipelines.\n */\nexport type ResetPipelinesReason =\n  | 'advancement-timeout'\n  | 'scalar-subquery'\n  | 'schema-change'\n  | 'truncation'\n  | 'permissions-change';\n\nexport class ResetPipelinesSignal extends Error {\n  readonly name = 'ResetPipelinesSignal';\n  readonly reason: ResetPipelinesReason;\n\n  constructor(msg: string, reason: ResetPipelinesReason) {\n    super(msg);\n    this.reason = reason;\n  }\n}\n\nclass Snapshot {\n  static create(\n    lc: LogContext,\n    dbFile: string,\n    appID: string,\n    pageCacheSizeKib: number | undefined,\n  ) {\n    const conn = new Database(lc, dbFile);\n    conn.pragma('synchronous = OFF'); // Applied changes are ephemeral; COMMIT is never called.\n    if (pageCacheSizeKib !== undefined) {\n      conn.pragma(`cache_size = -${pageCacheSizeKib}`); // Negative = size in KiB\n    }\n    const [{journal_mode: mode}] = conn.pragma('journal_mode') as [\n      {journal_mode: string},\n    ];\n    // The Snapshotter operates on the replica file with BEGIN CONCURRENT,\n    // which must be used in concert with the replicator using BEGIN CONCURRENT\n    // on a db in the wal2 journal_mode.\n    assert(\n      mode === 'wal2',\n      `replica db must be in wal2 mode (current: ${mode})`,\n    );\n\n    const db = new StatementRunner(conn);\n    return new Snapshot(db, appID);\n  }\n\n  readonly db: StatementRunner;\n  readonly #appID: string;\n  readonly version: string;\n\n  constructor(db: StatementRunner, appID: string) {\n    db.beginConcurrent();\n    // Note: The subsequent read is necessary to acquire the read lock\n    // (which results in the logical creation of the snapshot). Calling\n    // `BEGIN CONCURRENT` alone does not result in acquiring the read lock.\n    const {stateVersion} = getReplicationState(db);\n\n    this.db = db;\n    this.#appID = appID;\n    this.version = stateVersion;\n  }\n\n  numChangesSince(prevVersion: string) {\n    const {count} = this.db.get(\n      'SELECT COUNT(*) AS count FROM \"_zero.changeLog2\" WHERE stateVersion > ?',\n      prevVersion,\n    );\n    return count;\n  }\n\n  changesSince(prevVersion: string) {\n    // Note: The queried fields are constrained to only those that are relevant\n    // to the snapshot diff, i.e. those defined in the changeLogEntrySchema.\n    const cached = this.db.statementCache.get(\n      `SELECT \"stateVersion\", \"table\", \"rowKey\", \"op\" FROM \"_zero.changeLog2\"\n         WHERE \"stateVersion\" > ? ORDER BY \"stateVersion\" ASC, \"pos\" ASC`,\n    );\n    return {\n      changes: cached.statement.iterate(prevVersion),\n      cleanup: () => this.db.statementCache.return(cached),\n    };\n  }\n\n  getRow(table: LiteTableSpecWithKeysAndVersion, rowKey: JSONValue) {\n    const key = normalizedKeyOrder(rowKey as RowKey);\n    const conds = Object.keys(key).map(c => `${id(c)}=?`);\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.join(' AND ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.get<any>(Object.values(key));\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  getRows(\n    table: LiteTableSpecWithKeysAndVersion,\n    keys: PrimaryKey[],\n    row: RowValue,\n  ) {\n    // Filter out keys where any column is NULL. This is both correct and\n    // critical for performance:\n    // 1. Correctness: NULL values can't violate uniqueness (NULL != NULL in SQL)\n    // 2. Performance: SQLite's MULTI-INDEX OR optimization completely fails when\n    //    any branch involves NULL, falling back to a full table scan. This was\n    //    causing slowdowns of hundreds of times on tables with nullable unique columns.\n    const validKeys = keys.filter(key =>\n      key.every(column => row[column] !== null && row[column] !== undefined),\n    );\n    if (validKeys.length === 0) {\n      return [];\n    }\n    const conds = validKeys.map(key => key.map(c => `${id(c)}=?`));\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.map(cond => cond.join(' AND ')).join(' OR ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.all<any>(\n        validKeys.flatMap(key => key.map(column => row[column])),\n      );\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  resetToHead(): Snapshot {\n    this.db.rollback();\n    return new Snapshot(this.db, this.#appID);\n  }\n}\n\nclass Diff implements SnapshotDiff {\n  readonly #permissionsTable: string;\n  readonly #syncableTables: Map<string, LiteAndZqlSpec>;\n  readonly #allTableNames: Set<string>;\n  readonly prev: Snapshot;\n  readonly curr: Snapshot;\n  readonly changes: number;\n\n  constructor(\n    appID: string,\n    syncableTables: Map<string, LiteAndZqlSpec>,\n    allTableNames: Set<string>,\n    prev: Snapshot,\n    curr: Snapshot,\n  ) {\n    this.#permissionsTable = `${appID}.permissions`;\n    this.#syncableTables = syncableTables;\n    this.#allTableNames = allTableNames;\n    this.prev = prev;\n    this.curr = curr;\n    this.changes = curr.numChangesSince(prev.version);\n  }\n\n  [Symbol.iterator](): Iterator<Change> {\n    const {changes, cleanup: done} = this.curr.changesSince(this.prev.version);\n\n    const cleanup = () => {\n      try {\n        // Allow open iterators to clean up their state.\n        changes.return?.(undefined);\n      } finally {\n        done();\n      }\n    };\n\n    return {\n      next: () => {\n        try {\n          for (;;) {\n            const {value, done} = changes.next();\n            if (done) {\n              cleanup();\n              return {value, done: true};\n            }\n\n            const {table, rowKey, op, stateVersion} = v.parse(value, schema);\n            if (op === RESET_OP) {\n              // The current map of `TableSpec`s may not have the correct or complete information.\n              throw new ResetPipelinesSignal(\n                `schema for table ${table} has changed`,\n                'schema-change',\n              );\n            }\n            if (op === TRUNCATE_OP) {\n              // Truncates are also processed by rehydrating pipelines at current.\n              throw new ResetPipelinesSignal(\n                `table ${table} has been truncated`,\n                'truncation',\n              );\n            }\n            const specs = this.#syncableTables.get(table);\n            if (!specs) {\n              if (this.#allTableNames.has(table)) {\n                continue; // skip change log entries for non-syncable tables.\n              }\n              throw new Error(`change for unknown table ${table}`);\n            }\n            const {tableSpec, zqlSpec} = specs;\n\n            // Sanity check: All change log ops should have a stateVersion\n            // greater than minRowVersion in the table metadata. This is a\n            // mini-proof that the overlay does not need to be applied to\n            // rows produced in incremental catchup, based on the invariant in\n            // change-processor's #bumpVersions(), whereby the setting of the\n            // minRowVersion is always followed by a RESET OP, meaning that\n            // subsequent change-log traversal happens at a later version.\n            assert(\n              (tableSpec.minRowVersion ?? '') < stateVersion,\n              () =>\n                `unexpected change @${stateVersion} for table ${table} with ` +\n                `minRowVersion ${tableSpec.minRowVersion}: ${op}(${rowKey})`,\n            );\n\n            assert(rowKey !== null, 'rowKey must be present for row changes');\n            const nextValue =\n              op === SET_OP ? this.curr.getRow(tableSpec, rowKey) : null;\n            let prevValues;\n            if (nextValue) {\n              prevValues = this.prev.getRows(\n                tableSpec,\n                tableSpec.uniqueKeys,\n                nextValue,\n              );\n            } else {\n              const prevValue = this.prev.getRow(tableSpec, rowKey);\n              prevValues = prevValue ? [prevValue] : [];\n            }\n            if (nextValue === undefined) {\n              throw new Error(\n                `Missing value for ${table} ${stringify(rowKey)}`,\n              );\n            }\n            // Sanity check detects if the diff is being accessed after the Snapshots have advanced.\n            this.checkThatDiffIsValid(stateVersion, op, prevValues, nextValue);\n\n            if (prevValues.length === 0 && nextValue === null) {\n              // Filter out no-op changes (e.g. a delete of a row that does not exist in prev).\n              // TODO: Consider doing this for deep-equal values.\n              continue;\n            }\n\n            if (\n              table === this.#permissionsTable &&\n              prevValues.some(\n                prevValue => prevValue.permissions !== nextValue.permissions,\n              )\n            ) {\n              throw new ResetPipelinesSignal(\n                `Permissions have changed ${\n                  prevValues.find(\n                    prevValue =>\n                      prevValue.permissions !== nextValue.permissions,\n                  ).hash\n                } => ${nextValue.hash}`,\n                'permissions-change',\n              );\n            }\n\n            // Modify the values in place when converting to ZQL rows\n            // This is safe since we're the first node in the iterator chain.\n            // TODO Can we get rid of these RowValue casts?\n            return {\n              value: {\n                table,\n                prevValues: prevValues.map(prevValue =>\n                  fromSQLiteTypes(zqlSpec, prevValue, table),\n                ),\n                nextValue: nextValue\n                  ? fromSQLiteTypes(zqlSpec, nextValue, table)\n                  : null,\n                rowKey,\n              } satisfies Change,\n            };\n          }\n        } catch (e) {\n          // This control flow path is not covered by the return() method (i.e. `break`).\n          cleanup();\n          throw e;\n        }\n      },\n\n      return: (value: unknown) => {\n        cleanup();\n        return {value, done: true};\n      },\n    };\n  }\n\n  checkThatDiffIsValid(\n    stateVersion: string,\n    op: string,\n    prevValues: RowValue[],\n    nextValue: RowValue,\n  ) {\n    // Sanity checks to detect that the diff is not being accessed after\n    // the Snapshots have advanced.\n    if (stateVersion > this.curr.version) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. curr db has advanced past ${this.curr.version}`,\n      );\n    }\n    if (\n      prevValues.findIndex(\n        prevValue => (prevValue[ROW_VERSION] ?? '~') > this.prev.version,\n      ) !== -1\n    ) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. prev db has advanced past ${this.prev.version}.`,\n      );\n    }\n    if (op === SET_OP && nextValue[ROW_VERSION] !== stateVersion) {\n      throw new InvalidDiffError(\n        'Diff is no longer valid. curr db has advanced.',\n      );\n    }\n  }\n}\n\nexport class InvalidDiffError extends Error {\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0FA,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,EAAC,SACD,kBACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,SAAS;EACd,KAAKC,oBAAoB;CAC3B;;;;;;CAOA,OAAa;EACX,OAAO,KAAKC,UAAU,KAAA,GAAW,qBAAqB;EACtD,KAAKA,QAAQ,SAAS,OACpB,KAAKJ,KACL,KAAKC,SACL,KAAKC,QACL,KAAKC,iBACP;EACA,KAAKH,IAAI,QAAQ,+BAA+B,KAAKI,MAAM,SAAS;EACpE,OAAO;CACT;CAEA,cAAuB;EACrB,OAAO,KAAKA,UAAU,KAAA;CACxB;;CAGA,UAAoB;EAClB,OAAO,KAAKA,UAAU,KAAA,GAAW,sCAAsC;EACvE,OAAO,KAAKA;CACd;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuCA,QACE,gBACA,eACc;EACd,MAAM,EAAC,MAAM,SAAQ,KAAK,mBAAmB;EAC7C,OAAO,IAAI,KAAK,KAAKF,QAAQ,gBAAgB,eAAe,MAAM,IAAI;CACxE;CAEA,qBAAqB;EACnB,OAAO,KAAKE,UAAU,KAAA,GAAW,sCAAsC;EACvE,MAAM,OAAO,KAAKC,QACd,KAAKA,MAAM,YAAY,IACvB,SAAS,OACP,KAAKL,KACL,KAAKI,MAAM,GAAG,GAAG,MACjB,KAAKF,QACL,KAAKC,iBACP;EACJ,KAAKE,QAAQ,KAAKD;EAClB,KAAKA,QAAQ;EACb,OAAO;GAAC,MAAM,KAAKC;GAAO,MAAM,KAAKD;EAAK;CAC5C;;;;;CAMA,UAAU;EACR,KAAKA,OAAO,GAAG,GAAG,MAAM;EACxB,KAAKC,OAAO,GAAG,GAAG,MAAM;EACxB,KAAKL,IAAI,QAAQ,6BAA6B;CAChD;AACF;AA0DA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAgB;CAChB;CAEA,YAAY,KAAa,QAA8B;EACrD,MAAM,GAAG;EACT,KAAK,SAAS;CAChB;AACF;AAEA,IAAM,WAAN,MAAM,SAAS;CACb,OAAO,OACL,IACA,QACA,OACA,kBACA;EACA,MAAM,OAAO,IAAI,SAAS,IAAI,MAAM;EACpC,KAAK,OAAO,mBAAmB;EAC/B,IAAI,qBAAqB,KAAA,GACvB,KAAK,OAAO,iBAAiB,kBAAkB;EAEjD,MAAM,CAAC,EAAC,cAAc,UAAS,KAAK,OAAO,cAAc;EAMzD,OACE,SAAS,QACT,6CAA6C,KAAK,EACpD;EAGA,OAAO,IAAI,SAAS,IADL,gBAAgB,IACX,GAAI,KAAK;CAC/B;CAEA;CACA;CACA;CAEA,YAAY,IAAqB,OAAe;EAC9C,GAAG,gBAAgB;EAInB,MAAM,EAAC,iBAAgB,oBAAoB,EAAE;EAE7C,KAAK,KAAK;EACV,KAAKE,SAAS;EACd,KAAK,UAAU;CACjB;CAEA,gBAAgB,aAAqB;EACnC,MAAM,EAAC,UAAS,KAAK,GAAG,IACtB,6EACA,WACF;EACA,OAAO;CACT;CAEA,aAAa,aAAqB;EAGhC,MAAM,SAAS,KAAK,GAAG,eAAe,IACpC;yEAEF;EACA,OAAO;GACL,SAAS,OAAO,UAAU,QAAQ,WAAW;GAC7C,eAAe,KAAK,GAAG,eAAe,OAAO,MAAM;EACrD;CACF;CAEA,OAAO,OAAwC,QAAmB;EAChE,MAAM,MAAM,mBAAmB,MAAgB;EAC/C,MAAM,QAAQ,OAAO,KAAK,GAAG,EAAE,KAAI,MAAK,GAAG,GAAG,CAAC,EAAE,GAAG;EACpD,MAAM,OAAO,OAAO,KAAK,MAAM,OAAO;EACtC,MAAM,SAAS,KAAK,GAAG,eAAe,IACpC,UAAU,KAAK,KAAI,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,EAAE,QAAQ,GAC/C,MAAM,IACR,EAAE,SAAS,MAAM,KAAK,OAAO,GAC/B;EACA,OAAO,UAAU,aAAa,IAAI;EAClC,IAAI;GAEF,OAAO,OAAO,UAAU,IAAS,OAAO,OAAO,GAAG,CAAC;EACrD,UAAU;GACR,KAAK,GAAG,eAAe,OAAO,MAAM;EACtC;CACF;CAEA,QACE,OACA,MACA,KACA;EAOA,MAAM,YAAY,KAAK,QAAO,QAC5B,IAAI,OAAM,WAAU,IAAI,YAAY,QAAQ,IAAI,YAAY,KAAA,CAAS,CACvE;EACA,IAAI,UAAU,WAAW,GACvB,OAAO,CAAC;EAEV,MAAM,QAAQ,UAAU,KAAI,QAAO,IAAI,KAAI,MAAK,GAAG,GAAG,CAAC,EAAE,GAAG,CAAC;EAC7D,MAAM,OAAO,OAAO,KAAK,MAAM,OAAO;EACtC,MAAM,SAAS,KAAK,GAAG,eAAe,IACpC,UAAU,KAAK,KAAI,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,EAAE,QAAQ,GAC/C,MAAM,IACR,EAAE,SAAS,MAAM,KAAI,SAAQ,KAAK,KAAK,OAAO,CAAC,EAAE,KAAK,MAAM,GAC9D;EACA,OAAO,UAAU,aAAa,IAAI;EAClC,IAAI;GAEF,OAAO,OAAO,UAAU,IACtB,UAAU,SAAQ,QAAO,IAAI,KAAI,WAAU,IAAI,OAAO,CAAC,CACzD;EACF,UAAU;GACR,KAAK,GAAG,eAAe,OAAO,MAAM;EACtC;CACF;CAEA,cAAwB;EACtB,KAAK,GAAG,SAAS;EACjB,OAAO,IAAI,SAAS,KAAK,IAAI,KAAKA,MAAM;CAC1C;AACF;AAEA,IAAM,OAAN,MAAmC;CACjC;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,OACA,gBACA,eACA,MACA,MACA;EACA,KAAKI,oBAAoB,GAAG,MAAM;EAClC,KAAKC,kBAAkB;EACvB,KAAKC,iBAAiB;EACtB,KAAK,OAAO;EACZ,KAAK,OAAO;EACZ,KAAK,UAAU,KAAK,gBAAgB,KAAK,OAAO;CAClD;CAEA,CAAC,OAAO,YAA8B;EACpC,MAAM,EAAC,SAAS,SAAS,SAAQ,KAAK,KAAK,aAAa,KAAK,KAAK,OAAO;EAEzE,MAAM,gBAAgB;GACpB,IAAI;IAEF,QAAQ,SAAS,KAAA,CAAS;GAC5B,UAAU;IACR,KAAK;GACP;EACF;EAEA,OAAO;GACL,YAAY;IACV,IAAI;KACF,SAAS;MACP,MAAM,EAAC,OAAO,SAAQ,QAAQ,KAAK;MACnC,IAAI,MAAM;OACR,QAAQ;OACR,OAAO;QAAC;QAAO,MAAM;OAAI;MAC3B;MAEA,MAAM,EAAC,OAAO,QAAQ,IAAI,iBAAgB,MAAQ,OAAO,oBAAM;MAC/D,IAAI,OAAA,KAEF,MAAM,IAAI,qBACR,oBAAoB,MAAM,eAC1B,eACF;MAEF,IAAI,OAAA,KAEF,MAAM,IAAI,qBACR,SAAS,MAAM,sBACf,YACF;MAEF,MAAM,QAAQ,KAAKD,gBAAgB,IAAI,KAAK;MAC5C,IAAI,CAAC,OAAO;OACV,IAAI,KAAKC,eAAe,IAAI,KAAK,GAC/B;OAEF,MAAM,IAAI,MAAM,4BAA4B,OAAO;MACrD;MACA,MAAM,EAAC,WAAW,YAAW;MAS7B,QACG,UAAU,iBAAiB,MAAM,oBAEhC,sBAAsB,aAAa,aAAa,MAAM,sBACrC,UAAU,cAAc,IAAI,GAAG,GAAG,OAAO,EAC9D;MAEA,OAAO,WAAW,MAAM,wCAAwC;MAChE,MAAM,YACJ,OAAA,MAAgB,KAAK,KAAK,OAAO,WAAW,MAAM,IAAI;MACxD,IAAI;MACJ,IAAI,WACF,aAAa,KAAK,KAAK,QACrB,WACA,UAAU,YACV,SACF;WACK;OACL,MAAM,YAAY,KAAK,KAAK,OAAO,WAAW,MAAM;OACpD,aAAa,YAAY,CAAC,SAAS,IAAI,CAAC;MAC1C;MACA,IAAI,cAAc,KAAA,GAChB,MAAM,IAAI,MACR,qBAAqB,MAAM,GAAG,UAAU,MAAM,GAChD;MAGF,KAAK,qBAAqB,cAAc,IAAI,YAAY,SAAS;MAEjE,IAAI,WAAW,WAAW,KAAK,cAAc,MAG3C;MAGF,IACE,UAAU,KAAKF,qBACf,WAAW,MACT,cAAa,UAAU,gBAAgB,UAAU,WACnD,GAEA,MAAM,IAAI,qBACR,4BACE,WAAW,MACT,cACE,UAAU,gBAAgB,UAAU,WACxC,EAAE,KACH,MAAM,UAAU,QACjB,oBACF;MAMF,OAAO,EACL,OAAO;OACL;OACA,YAAY,WAAW,KAAI,cACzB,gBAAgB,SAAS,WAAW,KAAK,CAC3C;OACA,WAAW,YACP,gBAAgB,SAAS,WAAW,KAAK,IACzC;OACJ;MACF,EACF;KACF;IACF,SAAS,GAAG;KAEV,QAAQ;KACR,MAAM;IACR;GACF;GAEA,SAAS,UAAmB;IAC1B,QAAQ;IACR,OAAO;KAAC;KAAO,MAAM;IAAI;GAC3B;EACF;CACF;CAEA,qBACE,cACA,IACA,YACA,WACA;EAGA,IAAI,eAAe,KAAK,KAAK,SAC3B,MAAM,IAAI,iBACR,sDAAsD,KAAK,KAAK,SAClE;EAEF,IACE,WAAW,WACT,eAAc,UAAA,iBAA0B,OAAO,KAAK,KAAK,OAC3D,MAAM,IAEN,MAAM,IAAI,iBACR,sDAAsD,KAAK,KAAK,QAAQ,EAC1E;EAEF,IAAI,OAAA,OAAiB,UAAA,kBAA2B,cAC9C,MAAM,IAAI,iBACR,gDACF;CAEJ;AACF;AAEA,IAAa,mBAAb,cAAsC,MAAM;CAC1C,YAAY,KAAa;EACvB,MAAM,GAAG;CACX;AACF"}
+{"version":3,"file":"snapshotter.js","names":["#lc","#dbFile","#appID","#pageCacheSizeKib","#curr","#prev","#permissionsTable","#syncableTables","#allTableNames"],"sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {stringify, type JSONValue} from '../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Row} from '../../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../../zero-types/src/schema.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {fromSQLiteTypes} from '../../../../zqlite/src/table-source.ts';\nimport type {\n  LiteAndZqlSpec,\n  LiteTableSpecWithKeysAndVersion,\n} from '../../db/specs.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {\n  normalizedKeyOrder,\n  type RowKey,\n  type RowValue,\n} from '../../types/row-key.ts';\nimport type {AppID} from '../../types/shards.ts';\nimport {id} from '../../types/sql.ts';\nimport {\n  RESET_OP,\n  changeLogEntrySchema as schema,\n  SET_OP,\n  TRUNCATE_OP,\n} from '../replicator/schema/change-log.ts';\nimport {\n  getReplicationState,\n  ZERO_VERSION_COLUMN_NAME as ROW_VERSION,\n} from '../replicator/schema/replication-state.ts';\n\n/**\n * A `Snapshotter` manages the progression of database snapshots for a\n * ViewSyncer.\n *\n * The Replicator and ViewSyncers operate on the same SQLite file, with the\n * Replicator being the sole writer to the database. The IVM logic in\n * ViewSyncers, however, rely on incrementally applying changes to the DB to\n * update the state of its pipelines.\n *\n * To avoid coupling the progress of the Replicator and all IVM pipelines on\n * each other, ViewSyncers operate on ephemeral forks of the database by holding\n * [concurrent](https://sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md)\n * snapshots of the database and simulating (but ultimately rolling back)\n * mutations on these snapshots.\n *\n * Example:\n * 1. ViewSyncer takes `snapshot_a` at version `t1` of the database and\n *    hydrates its pipeline(s).\n * 2. Replicator applies a new transaction to the database and notifies\n *    subscribers.\n * 3. ViewSyncer takes `snapshot_b` at `t2`, and queries the `ChangeLog` at\n *    that snapshot for changes since `t1`.\n * 4. ViewSyncer applies those changes to `snapshot_a` for IVM, but does not\n *    commit them. (Recall that the Replicator is the sole writer to the db, so\n *    the ViewSyncer never commits any writes.)\n * 5. Replicator applies the next transaction and advances the database to `t3`.\n * 6. ViewSyncer rolls back `snapshot_a` and opens `snapshot_c` at `t3`, using\n *    `snapshot_b` to simulate changes from `t2` to `t3`.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ```\n *\n * Note that the Replicator (and ViewSyncers) do not wait on the progress of\n * other ViewSyncers. If a ViewSyncer is busy hydrating at `t1`, the Replicator\n * and other ViewSyncers can progress through `t2`, `t3`, etc. independently,\n * as the busy ViewSyncer simply takes its own snapshot when it is ready.\n *\n * ```\n * Replicator:   t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer1:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ViewSyncer2:       [.......... snapshot_a ..........] ----> [snapshot_b]\n * ```\n *\n * To minimize Database connections (and statement preparation, etc.), the\n * Snapshotter reuses the connection from the previous (rolled back)\n * snapshot when opening the new one.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n *                     (conn_1)           (conn_2)           (conn_1)\n * ```\n *\n * In this manner, each ViewSyncer uses two connections that continually\n * \"leapfrog\" each other to replay the timeline of changes in isolation from\n * the Replicator and other ViewSyncers.\n */\nexport class Snapshotter {\n  readonly #lc: LogContext;\n  readonly #dbFile: string;\n  readonly #appID: string;\n  readonly #pageCacheSizeKib: number | undefined;\n  #curr: Snapshot | undefined;\n  #prev: Snapshot | undefined;\n\n  constructor(\n    lc: LogContext,\n    dbFile: string,\n    {appID}: AppID,\n    pageCacheSizeKib?: number,\n  ) {\n    this.#lc = lc;\n    this.#dbFile = dbFile;\n    this.#appID = appID;\n    this.#pageCacheSizeKib = pageCacheSizeKib;\n  }\n\n  /**\n   * Initializes the snapshot to the current head of the database. This must be\n   * only be called once. The state of whether a Snapshotter has been initialized\n   * can be determined by calling {@link initialized()}.\n   */\n  init(): this {\n    assert(this.#curr === undefined, 'Already initialized');\n    this.#curr = Snapshot.create(\n      this.#lc,\n      this.#dbFile,\n      this.#appID,\n      this.#pageCacheSizeKib,\n    );\n    this.#lc.debug?.(`Initial snapshot at version ${this.#curr.version}`);\n    return this;\n  }\n\n  initialized(): boolean {\n    return this.#curr !== undefined;\n  }\n\n  /** Returns the current snapshot. Asserts if {@link initialized()} is false. */\n  current(): Snapshot {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    return this.#curr;\n  }\n\n  /**\n   * Advances to the head of the Database, returning a diff between the\n   * previously current Snapshot and a new Snapshot at head. This is called\n   * in response to a notification from a Replicator subscription. Subsequent\n   * calls to {@link current()} return the new Snapshot. Note that the Snapshotter\n   * must be initialized before advancing.\n   *\n   * The returned {@link SnapshotDiff} contains snapshots at the endpoints\n   * of the database timeline. Iterating over the diff generates a sequence\n   * of {@link Change}s between the two snapshots.\n   *\n   * Note that this sequence is not chronological; rather, the sequence is\n   * ordered by `<table, row-key>`, such that a row can appear at most once\n   * in the common case, or twice if its table is `TRUNCATE`'d and a new value\n   * is subsequently `INSERT`'ed. This results in dropping most intermediate\n   * changes to a row and bounds the amount of work needed to catch up;\n   * however, as a consequence, a consistent database state is only guaranteed\n   * when the sequence has been fully consumed.\n   *\n   * Note that Change generation relies on the state of the underlying\n   * database connections, and because the connection for the previous snapshot\n   * is reused to produce the next snapshot, the diff object is only valid\n   * until the next call to `advance()`.\n   *\n   * It is okay for the caller to apply `Change`s to the `prev` snapshot\n   * during the iteration (e.g. this is necessary for IVM); the remainder\n   * of the iteration is not affected because a given row can appear at most\n   * once in the sequence (with the exception being TRUNCATE, after which the\n   * deleted rows can be re-inserted, but this will also behave correctly if\n   * the changes are applied).\n   *\n   * Once the changes have been applied, however, a _subsequent_ iteration\n   * will not produce the correct results. In order to perform multiple\n   * change-applying iterations, the caller must (1) create a save point\n   * on `prev` before each iteration, and (2) rollback to the save point after\n   * the iteration.\n   */\n  advance(\n    syncableTables: Map<string, LiteAndZqlSpec>,\n    allTableNames: Set<string>,\n  ): SnapshotDiff {\n    const {prev, curr} = this.advanceWithoutDiff();\n    return new Diff(this.#appID, syncableTables, allTableNames, prev, curr);\n  }\n\n  advanceWithoutDiff() {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    const next = this.#prev\n      ? this.#prev.resetToHead()\n      : Snapshot.create(\n          this.#lc,\n          this.#curr.db.db.name,\n          this.#appID,\n          this.#pageCacheSizeKib,\n        );\n    this.#prev = this.#curr;\n    this.#curr = next;\n    return {prev: this.#prev, curr: this.#curr};\n  }\n\n  /**\n   * Call this to close the database connections when the Snapshotter is\n   * no longer needed.\n   */\n  destroy() {\n    this.#curr?.db.db.close();\n    this.#prev?.db.db.close();\n    this.#lc.debug?.('closed database connections');\n  }\n}\n\nexport type Change = {\n  readonly table: string;\n  /**\n   * If this change represents a remove the row to remove,\n   * if nextValue is not null then all rows that have a unique constraint\n   * violation with nextValue.\n   * In both cases these rows should be removed.\n   */\n  readonly prevValues: Readonly<Row>[];\n  readonly nextValue: Readonly<Row> | null;\n  readonly rowKey: RowKey;\n};\n\n/**\n * Represents the difference between two database Snapshots.\n * Iterating over the object will produce a sequence of {@link Change}s\n * between the two snapshots.\n *\n * See {@link Snapshotter.advance()} for semantics and usage.\n */\nexport interface SnapshotDiff extends Iterable<Change> {\n  readonly prev: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n  readonly curr: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n\n  /**\n   * The number of ChangeLog entries between the snapshots. Note that this\n   * may not necessarily equal the number of `Change` objects that the iteration\n   * will produce, as `TRUNCATE` entries are counted as a single log entry which\n   * may be expanded into many changes (i.e. row deletes).\n   *\n   * TODO: Determine if it is worth changing the definition to count the\n   *       truncated rows. This would make diff computation more expensive\n   *       (requiring the count to be aggregated by operation type), which\n   *       may not be worth it for a presumable rare operation.\n   */\n  readonly changes: number;\n}\n\n/**\n * Thrown during an iteration of a {@link SnapshotDiff} when a schema\n * change or truncate is encountered, which result in aborting the\n * advancement and resetting / rehydrating the pipelines.\n */\nexport type ResetPipelinesReason =\n  | 'advancement-timeout'\n  | 'scalar-subquery'\n  | 'schema-change'\n  | 'truncation'\n  | 'permissions-change';\n\nexport class ResetPipelinesSignal extends Error {\n  readonly name = 'ResetPipelinesSignal';\n  readonly reason: ResetPipelinesReason;\n\n  constructor(msg: string, reason: ResetPipelinesReason) {\n    super(msg);\n    this.reason = reason;\n  }\n}\n\nclass Snapshot {\n  static create(\n    lc: LogContext,\n    dbFile: string,\n    appID: string,\n    pageCacheSizeKib: number | undefined,\n  ) {\n    const conn = new Database(lc, dbFile);\n    conn.pragma('synchronous = OFF'); // Applied changes are ephemeral; COMMIT is never called.\n    if (pageCacheSizeKib !== undefined) {\n      conn.pragma(`cache_size = -${pageCacheSizeKib}`); // Negative = size in KiB\n    }\n    const [{journal_mode: mode}] = conn.pragma('journal_mode') as [\n      {journal_mode: string},\n    ];\n    // BEGIN CONCURRENT lets the Snapshotter simulate changes on a historic\n    // snapshot without taking the WAL writer lock. These private changes are\n    // always rolled back; the replicator is the only committer and can use\n    // BEGIN IMMEDIATE on the same WAL2 database.\n    assert(\n      mode === 'wal2',\n      `replica db must be in wal2 mode (current: ${mode})`,\n    );\n\n    const db = new StatementRunner(conn);\n    return new Snapshot(db, appID);\n  }\n\n  readonly db: StatementRunner;\n  readonly #appID: string;\n  readonly version: string;\n\n  constructor(db: StatementRunner, appID: string) {\n    db.beginConcurrent();\n    // Note: The subsequent read is necessary to acquire the read lock\n    // (which results in the logical creation of the snapshot). Calling\n    // `BEGIN CONCURRENT` alone does not result in acquiring the read lock.\n    const {stateVersion} = getReplicationState(db);\n\n    this.db = db;\n    this.#appID = appID;\n    this.version = stateVersion;\n  }\n\n  numChangesSince(prevVersion: string) {\n    const {count} = this.db.get(\n      'SELECT COUNT(*) AS count FROM \"_zero.changeLog2\" WHERE stateVersion > ?',\n      prevVersion,\n    );\n    return count;\n  }\n\n  changesSince(prevVersion: string) {\n    // Note: The queried fields are constrained to only those that are relevant\n    // to the snapshot diff, i.e. those defined in the changeLogEntrySchema.\n    const cached = this.db.statementCache.get(\n      `SELECT \"stateVersion\", \"table\", \"rowKey\", \"op\" FROM \"_zero.changeLog2\"\n         WHERE \"stateVersion\" > ? ORDER BY \"stateVersion\" ASC, \"pos\" ASC`,\n    );\n    return {\n      changes: cached.statement.iterate(prevVersion),\n      cleanup: () => this.db.statementCache.return(cached),\n    };\n  }\n\n  getRow(table: LiteTableSpecWithKeysAndVersion, rowKey: JSONValue) {\n    const key = normalizedKeyOrder(rowKey as RowKey);\n    const conds = Object.keys(key).map(c => `${id(c)}=?`);\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.join(' AND ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.get<any>(Object.values(key));\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  getRows(\n    table: LiteTableSpecWithKeysAndVersion,\n    keys: PrimaryKey[],\n    row: RowValue,\n  ) {\n    // Filter out keys where any column is NULL. This is both correct and\n    // critical for performance:\n    // 1. Correctness: NULL values can't violate uniqueness (NULL != NULL in SQL)\n    // 2. Performance: SQLite's MULTI-INDEX OR optimization completely fails when\n    //    any branch involves NULL, falling back to a full table scan. This was\n    //    causing slowdowns of hundreds of times on tables with nullable unique columns.\n    const validKeys = keys.filter(key =>\n      key.every(column => row[column] !== null && row[column] !== undefined),\n    );\n    if (validKeys.length === 0) {\n      return [];\n    }\n    const conds = validKeys.map(key => key.map(c => `${id(c)}=?`));\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.map(cond => cond.join(' AND ')).join(' OR ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.all<any>(\n        validKeys.flatMap(key => key.map(column => row[column])),\n      );\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  resetToHead(): Snapshot {\n    this.db.rollback();\n    return new Snapshot(this.db, this.#appID);\n  }\n}\n\nclass Diff implements SnapshotDiff {\n  readonly #permissionsTable: string;\n  readonly #syncableTables: Map<string, LiteAndZqlSpec>;\n  readonly #allTableNames: Set<string>;\n  readonly prev: Snapshot;\n  readonly curr: Snapshot;\n  readonly changes: number;\n\n  constructor(\n    appID: string,\n    syncableTables: Map<string, LiteAndZqlSpec>,\n    allTableNames: Set<string>,\n    prev: Snapshot,\n    curr: Snapshot,\n  ) {\n    this.#permissionsTable = `${appID}.permissions`;\n    this.#syncableTables = syncableTables;\n    this.#allTableNames = allTableNames;\n    this.prev = prev;\n    this.curr = curr;\n    this.changes = curr.numChangesSince(prev.version);\n  }\n\n  [Symbol.iterator](): Iterator<Change> {\n    const {changes, cleanup: done} = this.curr.changesSince(this.prev.version);\n\n    const cleanup = () => {\n      try {\n        // Allow open iterators to clean up their state.\n        changes.return?.(undefined);\n      } finally {\n        done();\n      }\n    };\n\n    return {\n      next: () => {\n        try {\n          for (;;) {\n            const {value, done} = changes.next();\n            if (done) {\n              cleanup();\n              return {value, done: true};\n            }\n\n            const {table, rowKey, op, stateVersion} = v.parse(value, schema);\n            if (op === RESET_OP) {\n              // The current map of `TableSpec`s may not have the correct or complete information.\n              throw new ResetPipelinesSignal(\n                `schema for table ${table} has changed`,\n                'schema-change',\n              );\n            }\n            if (op === TRUNCATE_OP) {\n              // Truncates are also processed by rehydrating pipelines at current.\n              throw new ResetPipelinesSignal(\n                `table ${table} has been truncated`,\n                'truncation',\n              );\n            }\n            const specs = this.#syncableTables.get(table);\n            if (!specs) {\n              if (this.#allTableNames.has(table)) {\n                continue; // skip change log entries for non-syncable tables.\n              }\n              throw new Error(`change for unknown table ${table}`);\n            }\n            const {tableSpec, zqlSpec} = specs;\n\n            // Sanity check: All change log ops should have a stateVersion\n            // greater than minRowVersion in the table metadata. This is a\n            // mini-proof that the overlay does not need to be applied to\n            // rows produced in incremental catchup, based on the invariant in\n            // change-processor's #bumpVersions(), whereby the setting of the\n            // minRowVersion is always followed by a RESET OP, meaning that\n            // subsequent change-log traversal happens at a later version.\n            assert(\n              (tableSpec.minRowVersion ?? '') < stateVersion,\n              () =>\n                `unexpected change @${stateVersion} for table ${table} with ` +\n                `minRowVersion ${tableSpec.minRowVersion}: ${op}(${rowKey})`,\n            );\n\n            assert(rowKey !== null, 'rowKey must be present for row changes');\n            const nextValue =\n              op === SET_OP ? this.curr.getRow(tableSpec, rowKey) : null;\n            let prevValues;\n            if (nextValue) {\n              prevValues = this.prev.getRows(\n                tableSpec,\n                tableSpec.uniqueKeys,\n                nextValue,\n              );\n            } else {\n              const prevValue = this.prev.getRow(tableSpec, rowKey);\n              prevValues = prevValue ? [prevValue] : [];\n            }\n            if (nextValue === undefined) {\n              throw new Error(\n                `Missing value for ${table} ${stringify(rowKey)}`,\n              );\n            }\n            // Sanity check detects if the diff is being accessed after the Snapshots have advanced.\n            this.checkThatDiffIsValid(stateVersion, op, prevValues, nextValue);\n\n            if (prevValues.length === 0 && nextValue === null) {\n              // Filter out no-op changes (e.g. a delete of a row that does not exist in prev).\n              // TODO: Consider doing this for deep-equal values.\n              continue;\n            }\n\n            if (\n              table === this.#permissionsTable &&\n              prevValues.some(\n                prevValue => prevValue.permissions !== nextValue.permissions,\n              )\n            ) {\n              throw new ResetPipelinesSignal(\n                `Permissions have changed ${\n                  prevValues.find(\n                    prevValue =>\n                      prevValue.permissions !== nextValue.permissions,\n                  ).hash\n                } => ${nextValue.hash}`,\n                'permissions-change',\n              );\n            }\n\n            // Modify the values in place when converting to ZQL rows\n            // This is safe since we're the first node in the iterator chain.\n            // TODO Can we get rid of these RowValue casts?\n            return {\n              value: {\n                table,\n                prevValues: prevValues.map(prevValue =>\n                  fromSQLiteTypes(zqlSpec, prevValue, table),\n                ),\n                nextValue: nextValue\n                  ? fromSQLiteTypes(zqlSpec, nextValue, table)\n                  : null,\n                rowKey,\n              } satisfies Change,\n            };\n          }\n        } catch (e) {\n          // This control flow path is not covered by the return() method (i.e. `break`).\n          cleanup();\n          throw e;\n        }\n      },\n\n      return: (value: unknown) => {\n        cleanup();\n        return {value, done: true};\n      },\n    };\n  }\n\n  checkThatDiffIsValid(\n    stateVersion: string,\n    op: string,\n    prevValues: RowValue[],\n    nextValue: RowValue,\n  ) {\n    // Sanity checks to detect that the diff is not being accessed after\n    // the Snapshots have advanced.\n    if (stateVersion > this.curr.version) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. curr db has advanced past ${this.curr.version}`,\n      );\n    }\n    if (\n      prevValues.findIndex(\n        prevValue => (prevValue[ROW_VERSION] ?? '~') > this.prev.version,\n      ) !== -1\n    ) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. prev db has advanced past ${this.prev.version}.`,\n      );\n    }\n    if (op === SET_OP && nextValue[ROW_VERSION] !== stateVersion) {\n      throw new InvalidDiffError(\n        'Diff is no longer valid. curr db has advanced.',\n      );\n    }\n  }\n}\n\nexport class InvalidDiffError extends Error {\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0FA,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,EAAC,SACD,kBACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,SAAS;EACd,KAAKC,oBAAoB;CAC3B;;;;;;CAOA,OAAa;EACX,OAAO,KAAKC,UAAU,KAAA,GAAW,qBAAqB;EACtD,KAAKA,QAAQ,SAAS,OACpB,KAAKJ,KACL,KAAKC,SACL,KAAKC,QACL,KAAKC,iBACP;EACA,KAAKH,IAAI,QAAQ,+BAA+B,KAAKI,MAAM,SAAS;EACpE,OAAO;CACT;CAEA,cAAuB;EACrB,OAAO,KAAKA,UAAU,KAAA;CACxB;;CAGA,UAAoB;EAClB,OAAO,KAAKA,UAAU,KAAA,GAAW,sCAAsC;EACvE,OAAO,KAAKA;CACd;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuCA,QACE,gBACA,eACc;EACd,MAAM,EAAC,MAAM,SAAQ,KAAK,mBAAmB;EAC7C,OAAO,IAAI,KAAK,KAAKF,QAAQ,gBAAgB,eAAe,MAAM,IAAI;CACxE;CAEA,qBAAqB;EACnB,OAAO,KAAKE,UAAU,KAAA,GAAW,sCAAsC;EACvE,MAAM,OAAO,KAAKC,QACd,KAAKA,MAAM,YAAY,IACvB,SAAS,OACP,KAAKL,KACL,KAAKI,MAAM,GAAG,GAAG,MACjB,KAAKF,QACL,KAAKC,iBACP;EACJ,KAAKE,QAAQ,KAAKD;EAClB,KAAKA,QAAQ;EACb,OAAO;GAAC,MAAM,KAAKC;GAAO,MAAM,KAAKD;EAAK;CAC5C;;;;;CAMA,UAAU;EACR,KAAKA,OAAO,GAAG,GAAG,MAAM;EACxB,KAAKC,OAAO,GAAG,GAAG,MAAM;EACxB,KAAKL,IAAI,QAAQ,6BAA6B;CAChD;AACF;AA0DA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAgB;CAChB;CAEA,YAAY,KAAa,QAA8B;EACrD,MAAM,GAAG;EACT,KAAK,SAAS;CAChB;AACF;AAEA,IAAM,WAAN,MAAM,SAAS;CACb,OAAO,OACL,IACA,QACA,OACA,kBACA;EACA,MAAM,OAAO,IAAI,SAAS,IAAI,MAAM;EACpC,KAAK,OAAO,mBAAmB;EAC/B,IAAI,qBAAqB,KAAA,GACvB,KAAK,OAAO,iBAAiB,kBAAkB;EAEjD,MAAM,CAAC,EAAC,cAAc,UAAS,KAAK,OAAO,cAAc;EAOzD,OACE,SAAS,QACT,6CAA6C,KAAK,EACpD;EAGA,OAAO,IAAI,SAAS,IADL,gBAAgB,IACX,GAAI,KAAK;CAC/B;CAEA;CACA;CACA;CAEA,YAAY,IAAqB,OAAe;EAC9C,GAAG,gBAAgB;EAInB,MAAM,EAAC,iBAAgB,oBAAoB,EAAE;EAE7C,KAAK,KAAK;EACV,KAAKE,SAAS;EACd,KAAK,UAAU;CACjB;CAEA,gBAAgB,aAAqB;EACnC,MAAM,EAAC,UAAS,KAAK,GAAG,IACtB,6EACA,WACF;EACA,OAAO;CACT;CAEA,aAAa,aAAqB;EAGhC,MAAM,SAAS,KAAK,GAAG,eAAe,IACpC;yEAEF;EACA,OAAO;GACL,SAAS,OAAO,UAAU,QAAQ,WAAW;GAC7C,eAAe,KAAK,GAAG,eAAe,OAAO,MAAM;EACrD;CACF;CAEA,OAAO,OAAwC,QAAmB;EAChE,MAAM,MAAM,mBAAmB,MAAgB;EAC/C,MAAM,QAAQ,OAAO,KAAK,GAAG,EAAE,KAAI,MAAK,GAAG,GAAG,CAAC,EAAE,GAAG;EACpD,MAAM,OAAO,OAAO,KAAK,MAAM,OAAO;EACtC,MAAM,SAAS,KAAK,GAAG,eAAe,IACpC,UAAU,KAAK,KAAI,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,EAAE,QAAQ,GAC/C,MAAM,IACR,EAAE,SAAS,MAAM,KAAK,OAAO,GAC/B;EACA,OAAO,UAAU,aAAa,IAAI;EAClC,IAAI;GAEF,OAAO,OAAO,UAAU,IAAS,OAAO,OAAO,GAAG,CAAC;EACrD,UAAU;GACR,KAAK,GAAG,eAAe,OAAO,MAAM;EACtC;CACF;CAEA,QACE,OACA,MACA,KACA;EAOA,MAAM,YAAY,KAAK,QAAO,QAC5B,IAAI,OAAM,WAAU,IAAI,YAAY,QAAQ,IAAI,YAAY,KAAA,CAAS,CACvE;EACA,IAAI,UAAU,WAAW,GACvB,OAAO,CAAC;EAEV,MAAM,QAAQ,UAAU,KAAI,QAAO,IAAI,KAAI,MAAK,GAAG,GAAG,CAAC,EAAE,GAAG,CAAC;EAC7D,MAAM,OAAO,OAAO,KAAK,MAAM,OAAO;EACtC,MAAM,SAAS,KAAK,GAAG,eAAe,IACpC,UAAU,KAAK,KAAI,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,EAAE,QAAQ,GAC/C,MAAM,IACR,EAAE,SAAS,MAAM,KAAI,SAAQ,KAAK,KAAK,OAAO,CAAC,EAAE,KAAK,MAAM,GAC9D;EACA,OAAO,UAAU,aAAa,IAAI;EAClC,IAAI;GAEF,OAAO,OAAO,UAAU,IACtB,UAAU,SAAQ,QAAO,IAAI,KAAI,WAAU,IAAI,OAAO,CAAC,CACzD;EACF,UAAU;GACR,KAAK,GAAG,eAAe,OAAO,MAAM;EACtC;CACF;CAEA,cAAwB;EACtB,KAAK,GAAG,SAAS;EACjB,OAAO,IAAI,SAAS,KAAK,IAAI,KAAKA,MAAM;CAC1C;AACF;AAEA,IAAM,OAAN,MAAmC;CACjC;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,OACA,gBACA,eACA,MACA,MACA;EACA,KAAKI,oBAAoB,GAAG,MAAM;EAClC,KAAKC,kBAAkB;EACvB,KAAKC,iBAAiB;EACtB,KAAK,OAAO;EACZ,KAAK,OAAO;EACZ,KAAK,UAAU,KAAK,gBAAgB,KAAK,OAAO;CAClD;CAEA,CAAC,OAAO,YAA8B;EACpC,MAAM,EAAC,SAAS,SAAS,SAAQ,KAAK,KAAK,aAAa,KAAK,KAAK,OAAO;EAEzE,MAAM,gBAAgB;GACpB,IAAI;IAEF,QAAQ,SAAS,KAAA,CAAS;GAC5B,UAAU;IACR,KAAK;GACP;EACF;EAEA,OAAO;GACL,YAAY;IACV,IAAI;KACF,SAAS;MACP,MAAM,EAAC,OAAO,SAAQ,QAAQ,KAAK;MACnC,IAAI,MAAM;OACR,QAAQ;OACR,OAAO;QAAC;QAAO,MAAM;OAAI;MAC3B;MAEA,MAAM,EAAC,OAAO,QAAQ,IAAI,iBAAgB,MAAQ,OAAO,oBAAM;MAC/D,IAAI,OAAA,KAEF,MAAM,IAAI,qBACR,oBAAoB,MAAM,eAC1B,eACF;MAEF,IAAI,OAAA,KAEF,MAAM,IAAI,qBACR,SAAS,MAAM,sBACf,YACF;MAEF,MAAM,QAAQ,KAAKD,gBAAgB,IAAI,KAAK;MAC5C,IAAI,CAAC,OAAO;OACV,IAAI,KAAKC,eAAe,IAAI,KAAK,GAC/B;OAEF,MAAM,IAAI,MAAM,4BAA4B,OAAO;MACrD;MACA,MAAM,EAAC,WAAW,YAAW;MAS7B,QACG,UAAU,iBAAiB,MAAM,oBAEhC,sBAAsB,aAAa,aAAa,MAAM,sBACrC,UAAU,cAAc,IAAI,GAAG,GAAG,OAAO,EAC9D;MAEA,OAAO,WAAW,MAAM,wCAAwC;MAChE,MAAM,YACJ,OAAA,MAAgB,KAAK,KAAK,OAAO,WAAW,MAAM,IAAI;MACxD,IAAI;MACJ,IAAI,WACF,aAAa,KAAK,KAAK,QACrB,WACA,UAAU,YACV,SACF;WACK;OACL,MAAM,YAAY,KAAK,KAAK,OAAO,WAAW,MAAM;OACpD,aAAa,YAAY,CAAC,SAAS,IAAI,CAAC;MAC1C;MACA,IAAI,cAAc,KAAA,GAChB,MAAM,IAAI,MACR,qBAAqB,MAAM,GAAG,UAAU,MAAM,GAChD;MAGF,KAAK,qBAAqB,cAAc,IAAI,YAAY,SAAS;MAEjE,IAAI,WAAW,WAAW,KAAK,cAAc,MAG3C;MAGF,IACE,UAAU,KAAKF,qBACf,WAAW,MACT,cAAa,UAAU,gBAAgB,UAAU,WACnD,GAEA,MAAM,IAAI,qBACR,4BACE,WAAW,MACT,cACE,UAAU,gBAAgB,UAAU,WACxC,EAAE,KACH,MAAM,UAAU,QACjB,oBACF;MAMF,OAAO,EACL,OAAO;OACL;OACA,YAAY,WAAW,KAAI,cACzB,gBAAgB,SAAS,WAAW,KAAK,CAC3C;OACA,WAAW,YACP,gBAAgB,SAAS,WAAW,KAAK,IACzC;OACJ;MACF,EACF;KACF;IACF,SAAS,GAAG;KAEV,QAAQ;KACR,MAAM;IACR;GACF;GAEA,SAAS,UAAmB;IAC1B,QAAQ;IACR,OAAO;KAAC;KAAO,MAAM;IAAI;GAC3B;EACF;CACF;CAEA,qBACE,cACA,IACA,YACA,WACA;EAGA,IAAI,eAAe,KAAK,KAAK,SAC3B,MAAM,IAAI,iBACR,sDAAsD,KAAK,KAAK,SAClE;EAEF,IACE,WAAW,WACT,eAAc,UAAA,iBAA0B,OAAO,KAAK,KAAK,OAC3D,MAAM,IAEN,MAAM,IAAI,iBACR,sDAAsD,KAAK,KAAK,QAAQ,EAC1E;EAEF,IAAI,OAAA,OAAiB,UAAA,kBAA2B,cAC9C,MAAM,IAAI,iBACR,gDACF;CAEJ;AACF;AAEA,IAAa,mBAAb,cAAsC,MAAM;CAC1C,YAAY,KAAa;EACvB,MAAM,GAAG;CACX;AACF"}

package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"view-syncer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/view-syncer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,EAAC,2BAA2B,EAAC,MAAM,yDAAyD,CAAC;AACzG,OAAO,KAAK,EAEV,qBAAqB,EACtB,MAAM,0CAA0C,CAAC;AAElD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,iDAAiD,CAAC;AAC1F,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAQtE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAGpF,OAAO,EAAkB,KAAK,IAAI,EAAC,MAAM,oBAAoB,CAAC;AAK9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EACV,sBAAsB,EAEvB,MAAM,yCAAyC,CAAC;AAOjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AAM1E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAE9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAQxD,OAAO,KAAK,EAEV,wBAAwB,EACxB,kBAAkB,EAEnB,MAAM,iCAAiC,CAAC;AAUzC,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAE7D,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAuCzD,MAAM,WAAW,UAAU;IACzB,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC,CAAC;IAEtB,oBAAoB,CAClB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,aAAa,CACX,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAErB,OAAO,CAAC,QAAQ,EAAE,kBAAkB,EAAE,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,UAAU,CACR,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,iBAAiB,EACtB,mBAAmB,EAAE,OAAO,GAC3B,OAAO,CAAC,IAAI,CAAC,CAAC;IAGjB,kBAAkB,EAAE,wBAAwB,CAAC;IAE7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;CACtC;AAED,MAAM,MAAM,WAAW,GAAG,kBAAkB,GAAG;IAC7C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,CAAC;CACjC,CAAC;AAmBF,KAAK,UAAU,GAAG,CAChB,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAChC,KAAK,CAAC,EAAE,MAAM,KACX,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,QAAS,CAAC;AAEzC;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAIvC,qBAAa,iBAAkB,YAAW,UAAU,EAAE,oBAAoB;;IACxE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,SAAc;IAGlC,QAAQ,CAAC,kBAAkB,EAAE,wBAAwB,CAAC;gBA+JpD,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,UAAU,EACjB,cAAc,EAAE,cAAc,EAC9B,cAAc,EAAE,YAAY,CAAC,YAAY,CAAC,EAC1C,gBAAgB,EAAE,gBAAgB,EAClC,oBAAoB,EAAE,MAAM,EAC5B,iBAAiB,EAAE,iBAAiB,EACpC,kBAAkB,EAAE,wBAAwB,EAC5C,sBAAsB,EAAE,sBAAsB,GAAG,SAAS,EAC1D,aAAa,EAAE,CAAC,CAAC,EACf,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,KACjB,OAAO,CAAC,CAAC,CAAC,EACf,WAAW,SAAuB,EAClC,YAAY,GAAE,UAAwC;IAiGxD,UAAU,IAAI,OAAO,CAAC,aAAa,GAAG,UAAU,CAAC;IAO3C,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAgI1B,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAEjC;IAED,IAAI,kBAAkB,IAAI,OAAO,CAKhC;IAQD;;;;;;;;OAQG;IACH,SAAS,IAAI,OAAO;IAkKpB,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC;IAuGf,oBAAoB,CACxB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC;IAiBV,UAAU,CACd,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,iBAAiB,EACtB,mBAAmB,EAAE,OAAO,GAC3B,OAAO,CAAC,IAAI,CAAC;IAsCV,aAAa,CACjB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC;IAikDd,OAAO,CACX,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,IAAI,CAAC;IAmKhB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAgCrB;;;OAGG;IACH,eAAe;CAGhB;AAsGD,qBAAa,cAAc;;gBAKb,EAAE,EAAE,UAAU;IAIpB,KAAK;IAOX,oBAAoB;IAMd,YAAY,CAAC,cAAc,CAAC,EAAE,MAAM;IAW1C,UAAU;IAWV,sCAAsC;IACtC,IAAI,IAAI,MAAM;IAKd;;;OAGG;IACH,YAAY,IAAI,MAAM;CAKvB"}
+{"version":3,"file":"view-syncer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/view-syncer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,EAAC,2BAA2B,EAAC,MAAM,yDAAyD,CAAC;AACzG,OAAO,KAAK,EAEV,qBAAqB,EACtB,MAAM,0CAA0C,CAAC;AAElD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,iDAAiD,CAAC;AAC1F,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAQtE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAGpF,OAAO,EAAkB,KAAK,IAAI,EAAC,MAAM,oBAAoB,CAAC;AAK9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EACV,sBAAsB,EAEvB,MAAM,yCAAyC,CAAC;AAOjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AAM1E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAE9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAQxD,OAAO,KAAK,EAEV,wBAAwB,EACxB,kBAAkB,EAEnB,MAAM,iCAAiC,CAAC;AAUzC,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAG7D,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAuCzD,MAAM,WAAW,UAAU;IACzB,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC,CAAC;IAEtB,oBAAoB,CAClB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,aAAa,CACX,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAErB,OAAO,CAAC,QAAQ,EAAE,kBAAkB,EAAE,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,UAAU,CACR,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,iBAAiB,EACtB,mBAAmB,EAAE,OAAO,GAC3B,OAAO,CAAC,IAAI,CAAC,CAAC;IAGjB,kBAAkB,EAAE,wBAAwB,CAAC;IAE7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;CACtC;AAED,MAAM,MAAM,WAAW,GAAG,kBAAkB,GAAG;IAC7C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,CAAC;CACjC,CAAC;AAmBF,KAAK,UAAU,GAAG,CAChB,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAChC,KAAK,CAAC,EAAE,MAAM,KACX,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,QAAS,CAAC;AAEzC;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAIvC,qBAAa,iBAAkB,YAAW,UAAU,EAAE,oBAAoB;;IACxE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,SAAc;IAGlC,QAAQ,CAAC,kBAAkB,EAAE,wBAAwB,CAAC;gBA0LpD,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,UAAU,EACjB,cAAc,EAAE,cAAc,EAC9B,cAAc,EAAE,YAAY,CAAC,YAAY,CAAC,EAC1C,gBAAgB,EAAE,gBAAgB,EAClC,oBAAoB,EAAE,MAAM,EAC5B,iBAAiB,EAAE,iBAAiB,EACpC,kBAAkB,EAAE,wBAAwB,EAC5C,sBAAsB,EAAE,sBAAsB,GAAG,SAAS,EAC1D,aAAa,EAAE,CAAC,CAAC,EACf,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,KACjB,OAAO,CAAC,CAAC,CAAC,EACf,WAAW,SAAuB,EAClC,YAAY,GAAE,UAAwC;IAiGxD,UAAU,IAAI,OAAO,CAAC,aAAa,GAAG,UAAU,CAAC;IAO3C,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAkI1B,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAEjC;IAED,IAAI,kBAAkB,IAAI,OAAO,CAKhC;IAkBD;;;;;;;;OAQG;IACH,SAAS,IAAI,OAAO;IAkKpB,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC;IAuGf,oBAAoB,CACxB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC;IAiBV,UAAU,CACd,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,iBAAiB,EACtB,mBAAmB,EAAE,OAAO,GAC3B,OAAO,CAAC,IAAI,CAAC;IAsCV,aAAa,CACjB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC;IAikDd,OAAO,CACX,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,IAAI,CAAC;IAmKhB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAgCrB;;;OAGG;IACH,eAAe;CAGhB;AAsGD,qBAAa,cAAc;;gBAKb,EAAE,EAAE,UAAU;IAIpB,KAAK;IAOX,oBAAoB;IAMd,YAAY,CAAC,cAAc,CAAC,EAAE,MAAM;IAW1C,UAAU;IAWV,sCAAsC;IACtC,IAAI,IAAI,MAAM;IAKd;;;OAGG;IACH,YAAY,IAAI,MAAM;CAKvB"}

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -25,6 +25,7 @@ import { ClientHandler, startPoke } from "./client-handler.js";
 import { tracer } from "./tracer.js";
 import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
 import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, nextEvictionTime } from "./cvr.js";
+import { E2EServingLagTracker } from "./e2e-serving-lag.js";
 import { handleInspect } from "./inspect-handler.js";
 import { QueryCoveringIndex } from "./query-covering.js";
 import { resolver } from "@rocicorp/resolver";
@@ -90,6 +91,7 @@ var ViewSyncerService = class {
 	#cvr;
 	#pipelinesSynced = false;
 	#servedVersion = null;
+	#e2eServingLagTracker = new E2EServingLagTracker();
 	#expiredQueriesTimer = 0;
 	#authMaintenanceTimer = 0;
 	#setTimeout;
@@ -102,6 +104,14 @@ var ViewSyncerService = class {
 		description: "Time from ViewSyncer query sync requiring hydration to output for a client group. Includes query transformation, query materialization, CVR flush, catchup, and pokeEnd.",
 		unit: "s"
 	});
+	#e2eServingLag = getOrCreateNativeHistogram("sync", "e2e_serving_lag", {
+		description: "End-to-end lag from upstream commit to ViewSyncer output. Spans the whole pipeline: the upstream transaction commit, replication to the replica, IVM advancement, CVR flush, and pokeEnd. Recorded once per served version, not sampled, so each observation is the completion latency of real replicated work.",
+		unit: "s"
+	});
+	#e2eServingLagClamps = getOrCreateCounter("sync", "e2e_serving_lag_clamps", {
+		description: "Observations of sync.e2e_serving_lag that came out negative and were clamped to zero. Non-zero means the upstream database clock is running ahead of this pod by more than the entire pipeline latency, so sync.e2e_serving_lag is biased low and reads healthier than reality. See replication.upstream_clock_skew for the magnitude.",
+		unit: "{observation}"
+	});
 	#transactionAdvanceTime = getOrCreateLatencyHistogram("sync", "advance-time", "Time to advance all queries for a given client group after applying a new transaction to the replica.");
 	#queryTransformations = getOrCreateCounter("sync", "query.transformations", "Number of query transformations performed");
 	#queryTransformationTime = getOrCreateLatencyHistogram("sync", "query.transformation-time", "Time to transform custom queries via API server.");
@@ -186,12 +196,14 @@ var ViewSyncerService = class {
 				this.#lc.debug?.(`draining view-syncer ${this.id} before running`);
 				this.stop();
 			}
-			for await (const { state } of this.#stateChanges) {
+			for await (const replicaState of this.#stateChanges) {
+				const { state } = replicaState;
 				if (this.#drainCoordinator.shouldDrain()) {
 					this.#lc.debug?.(`draining view-syncer ${this.id} (elective)`);
 					break;
 				}
 				assert(state === "version-ready", "state should be version-ready");
+				this.#e2eServingLagTracker.onVersionReady(replicaState);
 				await this.#runInLockWithCVR(async (lc, cvr) => {
 					const clientSchema = must(cvr.clientSchema, "cvr.clientSchema missing after initialization");
 					if (!this.#pipelines.initialized()) this.#pipelines.init(clientSchema);
@@ -258,6 +270,11 @@ var ViewSyncerService = class {
 	}
 	#markVersionServed(version) {
 		this.#servedVersion = version.stateVersion;
+		const observation = this.#e2eServingLagTracker.onVersionServed(version.stateVersion, Date.now());
+		if (observation !== null) {
+			this.#e2eServingLag.recordMs(observation.lagMs);
+			if (observation.clamped) this.#e2eServingLagClamps.add(1);
+		}
 	}
 	#keepAliveUntil = 0;
 	/**