@rocicorp/zero 1.11.0-canary.3 → 1.11.0-canary.30

package/out/zero-cache/src/services/change-streamer/snapshot.js

@@ -1,67 +1,35 @@
+import { AbortError } from "../../../../shared/src/abort-error.js";
 import { sleep } from "../../../../shared/src/sleep.js";
-import { valita_exports } from "../../../../shared/src/valita.js";
-import { getShardConfig } from "../../types/shards.js";
-import "../../config/normalize.js";
-import { ChangeStreamerHttpClient } from "./change-streamer-http.js";
-import { resolver } from "@rocicorp/resolver";
+import { promiseOrAbort } from "../../../../shared/src/promise-race.js";
 //#region ../zero-cache/src/services/change-streamer/snapshot.ts
-var statusSchema = valita_exports.object({
-	tag: valita_exports.literal("status"),
-	/**
-	* The location from which litestream should perform the restore.
-	*/
-	backupURL: valita_exports.string(),
-	/**
-	* The `replicaVersion` of the backup. If a subscriber's restored or
-	* existing replica is of a different version, it should delete it and
-	* retry the restore from litestream (i.e. equivalent to a
-	* `WrongReplicaVersion` response from a `/changes` subscription).
-	*/
-	replicaVersion: valita_exports.string(),
-	/**
-	* The earliest watermark from which catchup is possible. If the
-	* subscriber's replica is older that this watermark, it should delete it
-	* and (retry the) restore from litestream (i.e. equivalent to a
-	* `WatermarkTooOld` response from a `/changes` subscription).
-	*/
-	minWatermark: valita_exports.string()
-});
-var statusMessageSchema = valita_exports.tuple([valita_exports.literal("status"), statusSchema]);
-var snapshotMessageSchema = valita_exports.union(statusMessageSchema);
-function reserveAndGetSnapshotStatus(lc, config) {
-	const { promise: status, resolve, reject } = resolver();
-	(async function() {
-		const abort = new AbortController();
-		process.on("SIGINT", () => abort.abort());
-		process.on("SIGTERM", () => abort.abort());
+async function reserveAndGetSnapshotStatus(lc, taskID, changeStreamer) {
+	const abort = new AbortController();
+	const { signal } = abort;
+	const onSignal = () => abort.abort(new AbortError("SIGTERM/SIGINT"));
+	process.on("SIGINT", onSignal);
+	process.on("SIGTERM", onSignal);
+	try {
 		for (let i = 0;; i++) {
 			let err;
+			const reservation = changeStreamer.reserveSnapshot(taskID);
 			try {
-				let resolved = false;
-				const stream = await reserveSnapshot(lc, config);
-				for await (const msg of stream) {
-					resolve(msg[1]);
-					resolved = true;
-				}
-				if (resolved) break;
+				return await promiseOrAbort(reservation, signal);
 			} catch (e) {
 				err = e;
 			}
-			lc.warn?.(`Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`, String(err));
-			try {
-				await sleep(5e3, abort.signal);
-			} catch (e) {
-				return reject(e);
+			if (signal.aborted) {
+				reservation.then(({ followup }) => followup.cancel(), () => {});
+				throw err;
 			}
+			lc.warn?.(`Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`, String(err));
+			await sleep(5e3, signal);
 		}
-	})();
-	return status;
-}
-function reserveSnapshot(lc, config) {
-	const { taskID, change, changeStreamer } = config;
-	return new ChangeStreamerHttpClient(lc, getShardConfig(config), change.db, changeStreamer.uri).reserveSnapshot(taskID);
+	} finally {
+		process.off("SIGINT", onSignal);
+		process.off("SIGTERM", onSignal);
+	}
 }
 //#endregion
-export { reserveAndGetSnapshotStatus, snapshotMessageSchema };
+export { reserveAndGetSnapshotStatus };
 
 //# sourceMappingURL=snapshot.js.map

package/out/zero-cache/src/services/change-streamer/snapshot.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot.ts"],"sourcesContent":["/**\n * The `snapshot` API serves the purpose of:\n * - informing subscribers (i.e. view-syncers) of the (litestream)\n *   backup location from which to restore a replica snapshot\n * - checking whether a restored backup or existing replica is\n *   compatible with the change-streamer\n * - preventing change-log cleanup while a snapshot restore is in\n *   progress\n * - tracking the approximate time it takes from the beginning of\n *   snapshot \"reservation\" to the subsequent subscription, which\n *   serves as the minimum interval to wait before cleaning up\n *   backed up changes.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {type NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from './change-streamer-http.ts';\n\nconst statusSchema = v.object({\n  tag: v.literal('status'),\n\n  /**\n   * The location from which litestream should perform the restore.\n   */\n  backupURL: v.string(),\n\n  /**\n   * The `replicaVersion` of the backup. If a subscriber's restored or\n   * existing replica is of a different version, it should delete it and\n   * retry the restore from litestream (i.e. equivalent to a\n   * `WrongReplicaVersion` response from a `/changes` subscription).\n   */\n  replicaVersion: v.string(),\n\n  /**\n   * The earliest watermark from which catchup is possible. If the\n   * subscriber's replica is older that this watermark, it should delete it\n   * and (retry the) restore from litestream (i.e. equivalent to a\n   * `WatermarkTooOld` response from a `/changes` subscription).\n   */\n  minWatermark: v.string(),\n});\n\nexport type SnapshotStatus = v.Infer<typeof statusSchema>;\n\nconst statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\nexport const snapshotMessageSchema = v.union(statusMessageSchema);\n\nexport type SnapshotMessage = v.Infer<typeof statusMessageSchema>;\n\nexport function reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown;\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // backup monitor returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;AAuBA,IAAM,eAAe,eAAE,OAAO;CAC5B,KAAK,eAAE,QAAQ,QAAQ;;;;CAKvB,WAAW,eAAE,OAAO;;;;;;;CAQpB,gBAAgB,eAAE,OAAO;;;;;;;CAQzB,cAAc,eAAE,OAAO;AACzB,CAAC;AAID,IAAM,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAEvE,IAAa,wBAAwB,eAAE,MAAM,mBAAmB;AAIhE,SAAgB,4BACd,IACA,QACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,SAAyB;CAEpE,CAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,QAAQ,GAAG,gBAAgB,MAAM,MAAM,CAAC;EACxC,QAAQ,GAAG,iBAAiB,MAAM,MAAM,CAAC;EAEzC,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,IAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;IAC/C,WAAW,MAAM,OAAO,QAAQ;KAI9B,QAAQ,IAAI,EAAE;KACd,WAAW;IACb;IAGA,IAAI,UACF;GAEJ,SAAS,GAAG;IACV,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,IAAI;IACF,MAAM,MAAM,KAAM,MAAM,MAAM;GAChC,SAAS,GAAG;IACV,OAAO,OAAO,CAAC;GACjB;EACF;CACF,GAAG;CAEH,OAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;CAClC,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;CAUzC,OAAO,IAP0B,yBAC/B,IAHc,eAAe,MAI7B,GACA,OAAO,IACP,eAAe,GAGV,EAAqB,gBAAgB,MAAM;AACpD"}
+{"version":3,"file":"snapshot.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot.ts"],"sourcesContent":["/**\n * The `snapshot` API serves the purpose of:\n * - informing subscribers (i.e. view-syncers) of the (litestream)\n *   backup location from which to restore a replica snapshot\n * - checking whether a restored backup or existing replica is\n *   compatible with the change-streamer\n * - preventing change-log cleanup while a snapshot restore is in\n *   progress\n * - tracking the approximate time it takes from the beginning of\n *   snapshot \"reservation\" to the subsequent subscription, which\n *   serves as the minimum interval to wait before cleaning up\n *   backed up changes.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {promiseOrAbort} from '../../../../shared/src/promise-race.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\n\n// The schema definitions live in the pure schema module (with no dependency on\n// the HTTP client) so that subscribe.ts can import them without forming an\n// import cycle through change-streamer-http.ts. Imported here for local use and\n// re-exported for existing consumers.\nimport type {\n  ReservationFollowup,\n  SnapshotReserver,\n} from './change-streamer-http.ts';\nimport type {SnapshotStatus} from './subscribe.ts';\n\nexport type {SnapshotStatus};\n\nexport async function reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  taskID: string,\n  changeStreamer: SnapshotReserver,\n): Promise<{reserved: SnapshotStatus; followup: ReservationFollowup}> {\n  const abort = new AbortController();\n  const {signal} = abort;\n  // Use our own AbortError as the abort reason (rather than the native\n  // DOMException that AbortController.abort() defaults to) so that\n  // promiseOrAbort()'s rejection, which propagates `signal.reason` verbatim,\n  // is always an instance of the shared AbortError type.\n  const onSignal = () => abort.abort(new AbortError('SIGTERM/SIGINT'));\n  process.on('SIGINT', onSignal);\n  process.on('SIGTERM', onSignal);\n\n  try {\n    for (let i = 0; ; i++) {\n      let err: unknown;\n      const reservation = changeStreamer.reserveSnapshot(taskID);\n      try {\n        return await promiseOrAbort(reservation, signal);\n      } catch (e) {\n        err = e;\n      }\n      if (signal.aborted) {\n        // promiseOrAbort() does not cancel the loser of the race: the\n        // reservation may still resolve later. If it does, cancel its\n        // connection rather than leaking it, since this function has already\n        // given up and nobody will call followup.subscribe().\n        reservation.then(\n          ({followup}) => followup.cancel(),\n          () => {},\n        );\n        throw err;\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      await sleep(5000, signal);\n    }\n  } finally {\n    // This function is called repeatedly (e.g. by the replicator's restore\n    // loop), so the signal handlers must not outlive the reservation.\n    process.off('SIGINT', onSignal);\n    process.off('SIGTERM', onSignal);\n  }\n}\n"],"mappings":";;;;AA+BA,eAAsB,4BACpB,IACA,QACA,gBACoE;CACpE,MAAM,QAAQ,IAAI,gBAAgB;CAClC,MAAM,EAAC,WAAU;CAKjB,MAAM,iBAAiB,MAAM,MAAM,IAAI,WAAW,gBAAgB,CAAC;CACnE,QAAQ,GAAG,UAAU,QAAQ;CAC7B,QAAQ,GAAG,WAAW,QAAQ;CAE9B,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,MAAM,cAAc,eAAe,gBAAgB,MAAM;GACzD,IAAI;IACF,OAAO,MAAM,eAAe,aAAa,MAAM;GACjD,SAAS,GAAG;IACV,MAAM;GACR;GACA,IAAI,OAAO,SAAS;IAKlB,YAAY,MACT,EAAC,eAAc,SAAS,OAAO,SAC1B,CAAC,CACT;IACA,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,MAAM,MAAM,KAAM,MAAM;EAC1B;CACF,UAAU;EAGR,QAAQ,IAAI,UAAU,QAAQ;EAC9B,QAAQ,IAAI,WAAW,QAAQ;CACjC;AACF"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts

@@ -25,6 +25,11 @@ export type SQLiteChangeLogWriterOptions = {
     onRebuilt?: (() => void) | undefined;
     /** Overridable for tests. */
     now?: (() => number) | undefined;
+    /**
+     * Percentage of transactions that retain the rollout-only per-message
+     * timing and source-vs-stored hash validation. Defaults to 1.
+     */
+    shadowValidationPercent?: number | undefined;
 };
 /**
  * Appends the canonical change stream to the SQLite change log, from the
@@ -98,7 +103,7 @@ export declare class SQLiteChangeLogWriter {
      * Writes one stream message, committing at transaction boundaries. Never
      * throws: a write failure disables the writer instead of failing the stream.
      */
-    write(change: ChangeStreamData, json: string): void;
+    write(change: ChangeStreamData, json: string, storedChange?: string): void;
     /**
      * Discards the open transaction, if any. Called when the change stream is
      * interrupted mid-transaction.

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-writer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-writer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAM3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AAKtF,OAAO,EAUL,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EAE1B,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAML,KAAK,iCAAiC,EACvC,MAAM,kDAAkD,CAAC;AAE1D,MAAM,MAAM,4BAA4B,GAAG;IACzC,iFAAiF;IACjF,WAAW,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,QAAQ,EAAE,iBAAiB,CAAC;IAC5B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IACrD;;;;;OAKG;IACH,UAAU,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,CAAC;IACtC,6EAA6E;IAC7E,SAAS,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,CAAC;IACrC,6BAA6B;IAC7B,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,qBAAqB;;IAUhC,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,4BAA4B,EAI7D;IAED,qEAAqE;IACrE,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;;;;;;OAOG;IACH,IAAI,UAAU,IAAI,QAAQ,GAAG,SAAS,CAErC;IAED,2EAA2E;IAC3E,KAAK,IAAI,iCAAiC,GAAG,SAAS,CAErD;IAED;;;;;;;;;;;;;OAaG;IACH,SAAS,CACP,UAAU,EAAE,oBAAoB,GAC/B,oBAAoB,GAAG,SAAS,CAElC;IAED;;;;;OAKG;IACH,gBAAgB,CACd,IAAI,EAAE,MAAM,oBAAoB,GAC/B,oBAAoB,GAAG,SAAS,CAElC;IA2GD;;;OAGG;IACH,KAAK,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAgElD;IAED;;;OAGG;IACH,KAAK,IAAI,IAAI,CAWZ;IAED,KAAK,IAAI,IAAI,CAWZ;CA6BF"}
+{"version":3,"file":"sqlite-change-log-writer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-writer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAM3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AAKtF,OAAO,EAUL,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EAE1B,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAML,KAAK,iCAAiC,EACvC,MAAM,kDAAkD,CAAC;AAG1D,MAAM,MAAM,4BAA4B,GAAG;IACzC,iFAAiF;IACjF,WAAW,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,QAAQ,EAAE,iBAAiB,CAAC;IAC5B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IACrD;;;;;OAKG;IACH,UAAU,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,CAAC;IACtC,6EAA6E;IAC7E,SAAS,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,CAAC;IACrC,6BAA6B;IAC7B,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACjC;;;OAGG;IACH,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9C,CAAC;AAIF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,qBAAqB;;IAYhC,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,4BAA4B,EAY7D;IAED,qEAAqE;IACrE,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;;;;;;OAOG;IACH,IAAI,UAAU,IAAI,QAAQ,GAAG,SAAS,CAErC;IAED,2EAA2E;IAC3E,KAAK,IAAI,iCAAiC,GAAG,SAAS,CAErD;IAED;;;;;;;;;;;;;OAaG;IACH,SAAS,CACP,UAAU,EAAE,oBAAoB,GAC/B,oBAAoB,GAAG,SAAS,CAElC;IAED;;;;;OAKG;IACH,gBAAgB,CACd,IAAI,EAAE,MAAM,oBAAoB,GAC/B,oBAAoB,GAAG,SAAS,CAElC;IA2GD;;;OAGG;IACH,KAAK,CACH,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,MAAM,EACZ,YAAY,SAA8C,GACzD,IAAI,CA4FN;IAeD;;;OAGG;IACH,KAAK,IAAI,IAAI,CAYZ;IAED,KAAK,IAAI,IAAI,CAYZ;CA6BF"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js

@@ -1,12 +1,16 @@
+import { assert } from "../../../../shared/src/asserts.js";
 import { must } from "../../../../shared/src/must.js";
+import { h32 } from "../../../../shared/src/hash.js";
 import { readCookieRowCounts, readCookies } from "../replicator/change-log-cookies.js";
 import { StatementRunner } from "../../db/statements.js";
 import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics } from "../../db/sqlite-corruption.js";
 import { ChangeLogRebuildRequired, changeLogFileName, changeLogWipeReason, deleteChangeLogDB, openChangeLogDBForWriting, readChangeLogHead, rebuildChangeLogDBForWriting, reconcileChangeLog } from "../replicator/change-log-db.js";
+import { extractChangeSubstring } from "./change-log-codec.js";
 import { ChangeLogStreamWriter } from "../replicator/change-log-stream-writer.js";
 import { SQLiteChangeLogObserver, getSQLiteChangeLogInfo, logSQLiteChangeLogStartup, recordSQLiteChangeLogReconcile } from "../replicator/sqlite-change-log-observability.js";
 import { SqliteError } from "@rocicorp/zero-sqlite3";
 //#region ../zero-cache/src/services/change-streamer/sqlite-change-log-writer.ts
+var DEFAULT_SHADOW_VALIDATION_PERCENT = 1;
 /**
 * Appends the canonical change stream to the SQLite change log, from the
 * change-streamer's own stream loop.
@@ -41,14 +45,18 @@ var SQLiteChangeLogWriter = class {
 	#lc;
 	#opts;
 	#now;
+	#shadowValidationPercent;
 	#db;
 	#writer;
 	#observer;
 	#disabled = false;
+	#validateTransaction = false;
 	constructor(lc, opts) {
 		this.#lc = lc.withContext("component", "sqlite-change-log-writer");
 		this.#opts = opts;
 		this.#now = opts.now ?? Date.now;
+		this.#shadowValidationPercent = opts.shadowValidationPercent ?? DEFAULT_SHADOW_VALIDATION_PERCENT;
+		assert(Number.isInteger(this.#shadowValidationPercent) && this.#shadowValidationPercent >= 0 && this.#shadowValidationPercent <= 100, "SQLite change-log shadow validation percentage must be an integer between 0 and 100");
 	}
 	/** False once the writer has failed soft, or after {@link close}. */
 	get enabled() {
@@ -161,26 +169,28 @@ var SQLiteChangeLogWriter = class {
 	* Writes one stream message, committing at transaction boundaries. Never
 	* throws: a write failure disables the writer instead of failing the stream.
 	*/
-	write(change, json) {
+	write(change, json, storedChange = extractChangeSubstring(json, change[1].tag)) {
 		const writer = this.#writer;
 		const db = this.#db;
 		if (this.#disabled || writer === void 0 || db === void 0) return;
-		const observer = this.#observer;
-		observer?.messageReceived(change, json);
 		const [type, msg] = change;
-		const startedAt = performance.now();
+		if (type === "begin") this.#validateTransaction = this.#isSelectedForShadowValidation(change[2].commitWatermark);
+		const validateTransaction = this.#validateTransaction;
+		const observer = this.#observer;
+		observer?.messageReceived(change, json, validateTransaction, storedChange);
+		const startedAt = validateTransaction ? performance.now() : void 0;
 		try {
 			let commit = null;
 			switch (type) {
 				case "begin":
-					writer.begin(change[2].commitWatermark, json);
+					writer.begin(change[2].commitWatermark, json, storedChange, validateTransaction);
 					break;
 				case "commit": {
 					const { watermark } = change[2];
 					const commitStart = performance.now();
 					commit = {
 						watermark,
-						stats: writer.commit(watermark, json, this.#now()),
+						stats: writer.commit(watermark, json, this.#now(), storedChange),
 						logCommitMs: performance.now() - commitStart
 					};
 					break;
@@ -189,14 +199,16 @@ var SQLiteChangeLogWriter = class {
 					writer.rollback();
 					break;
 				default:
-					writer.append(json, msg);
+					writer.append(json, msg, storedChange);
 					break;
 			}
-			observer?.messageProcessed(change, commit, performance.now() - startedAt);
+			observer?.messageProcessed(change, commit, startedAt === void 0 ? void 0 : performance.now() - startedAt);
 			if (commit !== null && commit.stats.cookieMutations.length > 0) observer?.cookiesMutated(commit.stats.cookieMutations, readCookieRowCounts(db));
 			if (commit !== null) this.#opts.onCommit?.(commit.watermark);
+			if (type === "commit" || type === "rollback") this.#validateTransaction = false;
 		} catch (e) {
-			observer?.messageFailed(change, e, performance.now() - startedAt);
+			observer?.messageFailed(change, e, startedAt === void 0 ? void 0 : performance.now() - startedAt);
+			this.#validateTransaction = false;
 			if (isConstraintViolation(e)) observer?.invariantFailure("SQLite change-log insert violated a constraint, which means a transaction above the resume watermark was not truncated", {
 				tag: msg.tag,
 				error: String(e)
@@ -204,12 +216,17 @@ var SQLiteChangeLogWriter = class {
 			this.#failSoft("writing to the SQLite change log", e);
 		}
 	}
+	#isSelectedForShadowValidation(watermark) {
+		const percent = this.#shadowValidationPercent;
+		return percent >= 100 || percent > 0 && h32(`${this.#opts.identity.generation}/${this.#opts.identity.replicaID ?? ""}:${watermark}`) % 100 < percent;
+	}
 	/**
 	* Discards the open transaction, if any. Called when the change stream is
 	* interrupted mid-transaction.
 	*/
 	abort() {
 		const writer = this.#writer;
+		this.#validateTransaction = false;
 		if (this.#disabled || writer === void 0) return;
 		this.#observer?.abort();
 		try {
@@ -220,6 +237,7 @@ var SQLiteChangeLogWriter = class {
 	}
 	close() {
 		this.#disabled = true;
+		this.#validateTransaction = false;
 		this.#writer = void 0;
 		this.#observer = void 0;
 		const db = this.#db;

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-writer.js","names":["#lc","#opts","#now","#disabled","#db","#observer","#reconcile","#resumeFromLog","#writer","#failSoft"],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-writer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n  readCookieRowCounts,\n  readCookies,\n} from '../replicator/change-log-cookies.ts';\nimport {\n  ChangeLogRebuildRequired,\n  changeLogFileName,\n  changeLogWipeReason,\n  deleteChangeLogDB,\n  openChangeLogDBForWriting,\n  readChangeLogHead,\n  rebuildChangeLogDBForWriting,\n  reconcileChangeLog,\n  type ChangeLogAnchor,\n  type ChangeLogIdentity,\n  type ChangeLogResumePoint,\n  type ReconcileResult,\n} from '../replicator/change-log-db.ts';\nimport {ChangeLogStreamWriter} from '../replicator/change-log-stream-writer.ts';\nimport {\n  getSQLiteChangeLogInfo,\n  logSQLiteChangeLogStartup,\n  recordSQLiteChangeLogReconcile,\n  SQLiteChangeLogObserver,\n  type ChangeLogCommit,\n  type SQLiteChangeLogObservabilityState,\n} from '../replicator/sqlite-change-log-observability.ts';\n\nexport type SQLiteChangeLogWriterOptions = {\n  /** The replica file the log is named after, i.e. `${replicaFile}-change-log`. */\n  replicaFile: string;\n  /** The replica whose contents this log belongs to. */\n  identity: ChangeLogIdentity;\n  /**\n   * Called after each transaction's log commit, with the log's new head. This\n   * is what releases the catchup barrier: after the writer moved in-process,\n   * the head advances here rather than at a subscriber's ACK.\n   */\n  onCommit?: ((watermark: string) => void) | undefined;\n  /**\n   * Called when the writer disables itself and deletes the file. The reader is\n   * cached on the service, so a delete that leaves it open would leave an\n   * in-process reader serving an unlinked inode while every new open sees\n   * nothing.\n   */\n  onDisabled?: (() => void) | undefined;\n  /** Called when reconciliation replaces the file, to close cached readers. */\n  onRebuilt?: (() => void) | undefined;\n  /** Overridable for tests. */\n  now?: (() => number) | undefined;\n};\n\n/**\n * Appends the canonical change stream to the SQLite change log, from the\n * change-streamer's own stream loop.\n *\n * ### Ordering\n *\n * The log's commit for a transaction precedes the forward of that transaction's\n * `commit` message, and nothing `await`s in between. Everything that can\n * advance the watermark the stream would resume from — the storer's Postgres\n * commit, `lastWatermark`, the upstream ACK — runs strictly after it, so the\n * log's head is always at or above the resume point. A *hole* is therefore\n * unreachable at any crash timing, and a *phantom* is the normal state, which is\n * what makes truncate-above the whole of reconciliation.\n *\n * ### Fail-soft\n *\n * The log is a catchup cache: nothing in it is not either already durable in the\n * replica's litestream backup or re-derivable from the replication slot. A write\n * failure must therefore not stop replication for the shard. Corruption is\n * rebuilt once at open; any other write error — a full disk, an I/O error, a\n * permissions problem — logs, disables the writer, deletes the file, closes the\n * reader, and lets the stream continue. The cost is catchup reach, and reach is\n * recoverable: `too-old` sends a subscriber to a restore.\n *\n * One carve-out keeps that from hiding the bug class this topology introduces. A\n * missed truncate-above surfaces as `SQLITE_CONSTRAINT` on the writer's plain\n * `INSERT`, which is \"any write error other than corruption\" and would be\n * absorbed quietly. It is counted as an invariant failure and alerted on before\n * failing soft.\n */\nexport class SQLiteChangeLogWriter {\n  readonly #lc: LogContext;\n  readonly #opts: SQLiteChangeLogWriterOptions;\n  readonly #now: () => number;\n\n  #db: Database | undefined;\n  #writer: ChangeLogStreamWriter | undefined;\n  #observer: SQLiteChangeLogObserver | undefined;\n  #disabled = false;\n\n  constructor(lc: LogContext, opts: SQLiteChangeLogWriterOptions) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-writer');\n    this.#opts = opts;\n    this.#now = opts.now ?? Date.now;\n  }\n\n  /** False once the writer has failed soft, or after {@link close}. */\n  get enabled(): boolean {\n    return !this.#disabled;\n  }\n\n  /**\n   * The writer's own connection, which is the one purge runs on (§3.3): one\n   * process appends *and* deletes, on one handle, with no second writable\n   * connection and no cross-process barrier. `undefined` until the stream\n   * loop's first reconcile creates the log, and again once the writer has\n   * failed soft and deleted the file — both of which a purge cycle treats as\n   * \"skip\", not as an error.\n   */\n  get connection(): Database | undefined {\n    return this.#db;\n  }\n\n  /** Exposed for tests and for the integration assertions on the metrics. */\n  state(): SQLiteChangeLogObservabilityState | undefined {\n    return this.#observer?.state();\n  }\n\n  /**\n   * Opens the log if necessary and reconciles it with the Postgres resume\n   * point. Returns the resume point stored in the log.\n   *\n   * Returns `undefined` when the writer is disabled. The caller must then use\n   * another source.\n   *\n   * This runs for every connection because Postgres supplies a new resume point\n   * after each reconnect. The log can contain transactions above that point.\n   * Reconciliation removes them before Postgres sends them again.\n   *\n   * `resumeFrom` includes cookies because the cookies must match the resume\n   * watermark.\n   */\n  reconcile(\n    resumeFrom: ChangeLogResumePoint,\n  ): ChangeLogResumePoint | undefined {\n    return this.#reconcile(() => resumeFrom);\n  }\n\n  /**\n   * Reconciles the log without a Postgres resume point.\n   *\n   * A valid log uses its own head and cookies. A new or invalid log uses the\n   * replica values returned by `seed`.\n   */\n  reconcileFromLog(\n    seed: () => ChangeLogResumePoint,\n  ): ChangeLogResumePoint | undefined {\n    return this.#reconcile(db => this.#resumeFromLog(db, seed));\n  }\n\n  #reconcile(\n    resolve: (db: Database) => ChangeLogResumePoint,\n  ): ChangeLogResumePoint | undefined {\n    if (this.#disabled) {\n      return undefined;\n    }\n    const {replicaFile, identity} = this.#opts;\n    const nowMs = this.#now();\n    const anchor = (db: Database): ChangeLogAnchor => ({\n      ...resolve(db),\n      identity,\n      nowMs,\n    });\n    try {\n      const db = this.#db;\n      if (db === undefined) {\n        const opened = openChangeLogDBForWriting(this.#lc, replicaFile, anchor);\n        this.#db = opened.db;\n        if (opened.replacedFile) {\n          this.#opts.onRebuilt?.();\n        }\n        recordSQLiteChangeLogReconcile(this.#lc, opened.result);\n        const info = getSQLiteChangeLogInfo(opened.db);\n        logSQLiteChangeLogStartup(\n          this.#lc,\n          changeLogFileName(replicaFile),\n          info,\n        );\n        this.#observer = new SQLiteChangeLogObserver(this.#lc, info);\n      } else {\n        const resolved = anchor(db);\n        let result: ReconcileResult;\n        try {\n          result = reconcileChangeLog(this.#lc, db, resolved, {\n            rebuildInsteadOfUnboundedWork: true,\n          });\n        } catch (e) {\n          if (!(e instanceof ChangeLogRebuildRequired)) {\n            throw e;\n          }\n          // Close every handle to the old inode before unlinking it. The writer\n          // is reconstructed below against the replacement connection.\n          this.#writer = undefined;\n          this.#db = undefined;\n          db.close();\n          this.#opts.onRebuilt?.();\n          const rebuilt = rebuildChangeLogDBForWriting(\n            this.#lc,\n            replicaFile,\n            resolved,\n            e,\n          );\n          this.#db = rebuilt.db;\n          result = rebuilt.result;\n        }\n        recordSQLiteChangeLogReconcile(this.#lc, result);\n        if (result.action !== 'none') {\n          this.#observer?.reconciled(\n            getSQLiteChangeLogInfo(\n              must(this.#db, 'the SQLite change log is not open'),\n            ),\n          );\n        }\n      }\n      // A reseed drops and recreates the stream table, so the append statements\n      // are prepared fresh against whatever the reconciliation left behind.\n      const reconciledDB = must(this.#db, 'the SQLite change log is not open');\n      this.#writer = new ChangeLogStreamWriter(\n        new StatementRunner(reconciledDB),\n      );\n      // Read both values after reconciliation. A truncate or reseed updates the\n      // head and cookies in the same transaction.\n      return {\n        resumeWatermark: must(\n          readChangeLogHead(reconciledDB),\n          'the SQLite change log has no head after reconciliation',\n        ),\n        cookies: readCookies(reconciledDB),\n      };\n    } catch (e) {\n      this.#failSoft('reconciling the SQLite change log', e);\n      return undefined;\n    }\n  }\n\n  /**\n   * Returns the resume point supplied by the log.\n   *\n   * A valid log uses its own head and cookies. A log that must be recreated uses\n   * the replica values returned by `seed`.\n   */\n  #resumeFromLog(\n    db: Database,\n    seed: () => ChangeLogResumePoint,\n  ): ChangeLogResumePoint {\n    if (changeLogWipeReason(db, this.#opts.identity) !== undefined) {\n      return seed();\n    }\n    const head = readChangeLogHead(db);\n    // An empty log has no resume point, so use the replica seed.\n    return head === null\n      ? seed()\n      : {resumeWatermark: head, cookies: readCookies(db)};\n  }\n\n  /**\n   * Writes one stream message, committing at transaction boundaries. Never\n   * throws: a write failure disables the writer instead of failing the stream.\n   */\n  write(change: ChangeStreamData, json: string): void {\n    const writer = this.#writer;\n    const db = this.#db;\n    if (this.#disabled || writer === undefined || db === undefined) {\n      return;\n    }\n    const observer = this.#observer;\n    observer?.messageReceived(change, json);\n    const [type, msg] = change;\n    const startedAt = performance.now();\n    try {\n      let commit: ChangeLogCommit | null = null;\n      switch (type) {\n        case 'begin':\n          writer.begin(change[2].commitWatermark, json);\n          break;\n        case 'commit': {\n          const {watermark} = change[2];\n          const commitStart = performance.now();\n          const stats = writer.commit(watermark, json, this.#now());\n          commit = {\n            watermark,\n            stats,\n            logCommitMs: performance.now() - commitStart,\n          };\n          break;\n        }\n        case 'rollback':\n          writer.rollback();\n          break;\n        default:\n          writer.append(json, msg);\n          break;\n      }\n      observer?.messageProcessed(change, commit, performance.now() - startedAt);\n      if (commit !== null && commit.stats.cookieMutations.length > 0) {\n        // Only re-counted when a schema change actually moved the cookie jar,\n        // which keeps the count off the forward path of every other\n        // transaction. The counts are read after the commit, so they are what\n        // a reader of the file would see.\n        observer?.cookiesMutated(\n          commit.stats.cookieMutations,\n          readCookieRowCounts(db),\n        );\n      }\n      if (commit !== null) {\n        // The head advanced, and it is durable. Releasing the barrier here is\n        // what makes its poll interval a backstop rather than the mechanism.\n        this.#opts.onCommit?.(commit.watermark);\n      }\n    } catch (e) {\n      observer?.messageFailed(change, e, performance.now() - startedAt);\n      if (isConstraintViolation(e)) {\n        // Reconciliation should have truncated whatever this collided with.\n        // Report it as an invariant failure before failing soft, so that a\n        // truncate-above bug cannot look like a lost cache.\n        observer?.invariantFailure(\n          'SQLite change-log insert violated a constraint, which means a ' +\n            'transaction above the resume watermark was not truncated',\n          {tag: msg.tag, error: String(e)},\n        );\n      }\n      this.#failSoft('writing to the SQLite change log', e);\n    }\n  }\n\n  /**\n   * Discards the open transaction, if any. Called when the change stream is\n   * interrupted mid-transaction.\n   */\n  abort(): void {\n    const writer = this.#writer;\n    if (this.#disabled || writer === undefined) {\n      return;\n    }\n    this.#observer?.abort();\n    try {\n      writer.rollback();\n    } catch (e) {\n      this.#failSoft('rolling back the SQLite change log', e);\n    }\n  }\n\n  close(): void {\n    this.#disabled = true;\n    this.#writer = undefined;\n    this.#observer = undefined;\n    const db = this.#db;\n    this.#db = undefined;\n    try {\n      db?.close();\n    } catch (e) {\n      this.#lc.warn?.('error closing the SQLite change log', e);\n    }\n  }\n\n  /**\n   * Logs, disables the writer, deletes the file, and closes the reader, leaving\n   * replication running. Disabling is for the life of the process: a log that\n   * failed a write is not re-opened until the task restarts, so the behavior a\n   * subscriber sees is the same as `sqliteChangeLogMode=off`.\n   */\n  #failSoft(what: string, e: unknown): void {\n    const file = changeLogFileName(this.#opts.replicaFile);\n    if (isSQLiteCorruption(e)) {\n      // Open-time corruption is rebuilt by openChangeLogDBForWriting. Reaching\n      // here means the file went bad under an open handle, which the same\n      // delete-and-reseed answers -- on the next process start rather than now.\n      logSQLiteCorruptionDiagnostics(this.#lc, 'change-log', file, e);\n    }\n    this.#lc.error?.(\n      `error ${what}: disabling the writer and deleting ${file}. ` +\n        `Replication continues; catchup reach is lost until this task restarts.`,\n      e,\n    );\n    this.close();\n    try {\n      deleteChangeLogDB(this.#opts.replicaFile);\n    } catch (deleteError) {\n      this.#lc.error?.(`unable to delete ${file}`, deleteError);\n    }\n    this.#opts.onDisabled?.();\n  }\n}\n\nfunction isConstraintViolation(e: unknown): boolean {\n  return e instanceof SqliteError && e.code.startsWith('SQLITE_CONSTRAINT');\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4FA,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CAEA;CACA;CACA;CACA,YAAY;CAEZ,YAAY,IAAgB,MAAoC;EAC9D,KAAKA,MAAM,GAAG,YAAY,aAAa,0BAA0B;EACjE,KAAKC,QAAQ;EACb,KAAKC,OAAO,KAAK,OAAO,KAAK;CAC/B;;CAGA,IAAI,UAAmB;EACrB,OAAO,CAAC,KAAKC;CACf;;;;;;;;;CAUA,IAAI,aAAmC;EACrC,OAAO,KAAKC;CACd;;CAGA,QAAuD;EACrD,OAAO,KAAKC,WAAW,MAAM;CAC/B;;;;;;;;;;;;;;;CAgBA,UACE,YACkC;EAClC,OAAO,KAAKC,iBAAiB,UAAU;CACzC;;;;;;;CAQA,iBACE,MACkC;EAClC,OAAO,KAAKA,YAAW,OAAM,KAAKC,eAAe,IAAI,IAAI,CAAC;CAC5D;CAEA,WACE,SACkC;EAClC,IAAI,KAAKJ,WACP;EAEF,MAAM,EAAC,aAAa,aAAY,KAAKF;EACrC,MAAM,QAAQ,KAAKC,KAAK;EACxB,MAAM,UAAU,QAAmC;GACjD,GAAG,QAAQ,EAAE;GACb;GACA;EACF;EACA,IAAI;GACF,MAAM,KAAK,KAAKE;GAChB,IAAI,OAAO,KAAA,GAAW;IACpB,MAAM,SAAS,0BAA0B,KAAKJ,KAAK,aAAa,MAAM;IACtE,KAAKI,MAAM,OAAO;IAClB,IAAI,OAAO,cACT,KAAKH,MAAM,YAAY;IAEzB,+BAA+B,KAAKD,KAAK,OAAO,MAAM;IACtD,MAAM,OAAO,uBAAuB,OAAO,EAAE;IAC7C,0BACE,KAAKA,KACL,kBAAkB,WAAW,GAC7B,IACF;IACA,KAAKK,YAAY,IAAI,wBAAwB,KAAKL,KAAK,IAAI;GAC7D,OAAO;IACL,MAAM,WAAW,OAAO,EAAE;IAC1B,IAAI;IACJ,IAAI;KACF,SAAS,mBAAmB,KAAKA,KAAK,IAAI,UAAU,EAClD,+BAA+B,KACjC,CAAC;IACH,SAAS,GAAG;KACV,IAAI,EAAE,aAAa,2BACjB,MAAM;KAIR,KAAKQ,UAAU,KAAA;KACf,KAAKJ,MAAM,KAAA;KACX,GAAG,MAAM;KACT,KAAKH,MAAM,YAAY;KACvB,MAAM,UAAU,6BACd,KAAKD,KACL,aACA,UACA,CACF;KACA,KAAKI,MAAM,QAAQ;KACnB,SAAS,QAAQ;IACnB;IACA,+BAA+B,KAAKJ,KAAK,MAAM;IAC/C,IAAI,OAAO,WAAW,QACpB,KAAKK,WAAW,WACd,uBACE,KAAK,KAAKD,KAAK,mCAAmC,CACpD,CACF;GAEJ;GAGA,MAAM,eAAe,KAAK,KAAKA,KAAK,mCAAmC;GACvE,KAAKI,UAAU,IAAI,sBACjB,IAAI,gBAAgB,YAAY,CAClC;GAGA,OAAO;IACL,iBAAiB,KACf,kBAAkB,YAAY,GAC9B,wDACF;IACA,SAAS,YAAY,YAAY;GACnC;EACF,SAAS,GAAG;GACV,KAAKC,UAAU,qCAAqC,CAAC;GACrD;EACF;CACF;;;;;;;CAQA,eACE,IACA,MACsB;EACtB,IAAI,oBAAoB,IAAI,KAAKR,MAAM,QAAQ,MAAM,KAAA,GACnD,OAAO,KAAK;EAEd,MAAM,OAAO,kBAAkB,EAAE;EAEjC,OAAO,SAAS,OACZ,KAAK,IACL;GAAC,iBAAiB;GAAM,SAAS,YAAY,EAAE;EAAC;CACtD;;;;;CAMA,MAAM,QAA0B,MAAoB;EAClD,MAAM,SAAS,KAAKO;EACpB,MAAM,KAAK,KAAKJ;EAChB,IAAI,KAAKD,aAAa,WAAW,KAAA,KAAa,OAAO,KAAA,GACnD;EAEF,MAAM,WAAW,KAAKE;EACtB,UAAU,gBAAgB,QAAQ,IAAI;EACtC,MAAM,CAAC,MAAM,OAAO;EACpB,MAAM,YAAY,YAAY,IAAI;EAClC,IAAI;GACF,IAAI,SAAiC;GACrC,QAAQ,MAAR;IACE,KAAK;KACH,OAAO,MAAM,OAAO,GAAG,iBAAiB,IAAI;KAC5C;IACF,KAAK,UAAU;KACb,MAAM,EAAC,cAAa,OAAO;KAC3B,MAAM,cAAc,YAAY,IAAI;KAEpC,SAAS;MACP;MACA,OAHY,OAAO,OAAO,WAAW,MAAM,KAAKH,KAAK,CAGrD;MACA,aAAa,YAAY,IAAI,IAAI;KACnC;KACA;IACF;IACA,KAAK;KACH,OAAO,SAAS;KAChB;IACF;KACE,OAAO,OAAO,MAAM,GAAG;KACvB;GACJ;GACA,UAAU,iBAAiB,QAAQ,QAAQ,YAAY,IAAI,IAAI,SAAS;GACxE,IAAI,WAAW,QAAQ,OAAO,MAAM,gBAAgB,SAAS,GAK3D,UAAU,eACR,OAAO,MAAM,iBACb,oBAAoB,EAAE,CACxB;GAEF,IAAI,WAAW,MAGb,KAAKD,MAAM,WAAW,OAAO,SAAS;EAE1C,SAAS,GAAG;GACV,UAAU,cAAc,QAAQ,GAAG,YAAY,IAAI,IAAI,SAAS;GAChE,IAAI,sBAAsB,CAAC,GAIzB,UAAU,iBACR,0HAEA;IAAC,KAAK,IAAI;IAAK,OAAO,OAAO,CAAC;GAAC,CACjC;GAEF,KAAKQ,UAAU,oCAAoC,CAAC;EACtD;CACF;;;;;CAMA,QAAc;EACZ,MAAM,SAAS,KAAKD;EACpB,IAAI,KAAKL,aAAa,WAAW,KAAA,GAC/B;EAEF,KAAKE,WAAW,MAAM;EACtB,IAAI;GACF,OAAO,SAAS;EAClB,SAAS,GAAG;GACV,KAAKI,UAAU,sCAAsC,CAAC;EACxD;CACF;CAEA,QAAc;EACZ,KAAKN,YAAY;EACjB,KAAKK,UAAU,KAAA;EACf,KAAKH,YAAY,KAAA;EACjB,MAAM,KAAK,KAAKD;EAChB,KAAKA,MAAM,KAAA;EACX,IAAI;GACF,IAAI,MAAM;EACZ,SAAS,GAAG;GACV,KAAKJ,IAAI,OAAO,uCAAuC,CAAC;EAC1D;CACF;;;;;;;CAQA,UAAU,MAAc,GAAkB;EACxC,MAAM,OAAO,kBAAkB,KAAKC,MAAM,WAAW;EACrD,IAAI,mBAAmB,CAAC,GAItB,+BAA+B,KAAKD,KAAK,cAAc,MAAM,CAAC;EAEhE,KAAKA,IAAI,QACP,SAAS,KAAK,sCAAsC,KAAK,2EAEzD,CACF;EACA,KAAK,MAAM;EACX,IAAI;GACF,kBAAkB,KAAKC,MAAM,WAAW;EAC1C,SAAS,aAAa;GACpB,KAAKD,IAAI,QAAQ,oBAAoB,QAAQ,WAAW;EAC1D;EACA,KAAKC,MAAM,aAAa;CAC1B;AACF;AAEA,SAAS,sBAAsB,GAAqB;CAClD,OAAO,aAAa,eAAe,EAAE,KAAK,WAAW,mBAAmB;AAC1E"}
+{"version":3,"file":"sqlite-change-log-writer.js","names":["#lc","#opts","#now","#shadowValidationPercent","#disabled","#db","#observer","#reconcile","#resumeFromLog","#writer","#failSoft","#validateTransaction","#isSelectedForShadowValidation"],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-writer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {h32} from '../../../../shared/src/hash.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n  readCookieRowCounts,\n  readCookies,\n} from '../replicator/change-log-cookies.ts';\nimport {\n  ChangeLogRebuildRequired,\n  changeLogFileName,\n  changeLogWipeReason,\n  deleteChangeLogDB,\n  openChangeLogDBForWriting,\n  readChangeLogHead,\n  rebuildChangeLogDBForWriting,\n  reconcileChangeLog,\n  type ChangeLogAnchor,\n  type ChangeLogIdentity,\n  type ChangeLogResumePoint,\n  type ReconcileResult,\n} from '../replicator/change-log-db.ts';\nimport {ChangeLogStreamWriter} from '../replicator/change-log-stream-writer.ts';\nimport {\n  getSQLiteChangeLogInfo,\n  logSQLiteChangeLogStartup,\n  recordSQLiteChangeLogReconcile,\n  SQLiteChangeLogObserver,\n  type ChangeLogCommit,\n  type SQLiteChangeLogObservabilityState,\n} from '../replicator/sqlite-change-log-observability.ts';\nimport {extractChangeSubstring} from './change-log-codec.ts';\n\nexport type SQLiteChangeLogWriterOptions = {\n  /** The replica file the log is named after, i.e. `${replicaFile}-change-log`. */\n  replicaFile: string;\n  /** The replica whose contents this log belongs to. */\n  identity: ChangeLogIdentity;\n  /**\n   * Called after each transaction's log commit, with the log's new head. This\n   * is what releases the catchup barrier: after the writer moved in-process,\n   * the head advances here rather than at a subscriber's ACK.\n   */\n  onCommit?: ((watermark: string) => void) | undefined;\n  /**\n   * Called when the writer disables itself and deletes the file. The reader is\n   * cached on the service, so a delete that leaves it open would leave an\n   * in-process reader serving an unlinked inode while every new open sees\n   * nothing.\n   */\n  onDisabled?: (() => void) | undefined;\n  /** Called when reconciliation replaces the file, to close cached readers. */\n  onRebuilt?: (() => void) | undefined;\n  /** Overridable for tests. */\n  now?: (() => number) | undefined;\n  /**\n   * Percentage of transactions that retain the rollout-only per-message\n   * timing and source-vs-stored hash validation. Defaults to 1.\n   */\n  shadowValidationPercent?: number | undefined;\n};\n\nconst DEFAULT_SHADOW_VALIDATION_PERCENT = 1;\n\n/**\n * Appends the canonical change stream to the SQLite change log, from the\n * change-streamer's own stream loop.\n *\n * ### Ordering\n *\n * The log's commit for a transaction precedes the forward of that transaction's\n * `commit` message, and nothing `await`s in between. Everything that can\n * advance the watermark the stream would resume from — the storer's Postgres\n * commit, `lastWatermark`, the upstream ACK — runs strictly after it, so the\n * log's head is always at or above the resume point. A *hole* is therefore\n * unreachable at any crash timing, and a *phantom* is the normal state, which is\n * what makes truncate-above the whole of reconciliation.\n *\n * ### Fail-soft\n *\n * The log is a catchup cache: nothing in it is not either already durable in the\n * replica's litestream backup or re-derivable from the replication slot. A write\n * failure must therefore not stop replication for the shard. Corruption is\n * rebuilt once at open; any other write error — a full disk, an I/O error, a\n * permissions problem — logs, disables the writer, deletes the file, closes the\n * reader, and lets the stream continue. The cost is catchup reach, and reach is\n * recoverable: `too-old` sends a subscriber to a restore.\n *\n * One carve-out keeps that from hiding the bug class this topology introduces. A\n * missed truncate-above surfaces as `SQLITE_CONSTRAINT` on the writer's plain\n * `INSERT`, which is \"any write error other than corruption\" and would be\n * absorbed quietly. It is counted as an invariant failure and alerted on before\n * failing soft.\n */\nexport class SQLiteChangeLogWriter {\n  readonly #lc: LogContext;\n  readonly #opts: SQLiteChangeLogWriterOptions;\n  readonly #now: () => number;\n  readonly #shadowValidationPercent: number;\n\n  #db: Database | undefined;\n  #writer: ChangeLogStreamWriter | undefined;\n  #observer: SQLiteChangeLogObserver | undefined;\n  #disabled = false;\n  #validateTransaction = false;\n\n  constructor(lc: LogContext, opts: SQLiteChangeLogWriterOptions) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-writer');\n    this.#opts = opts;\n    this.#now = opts.now ?? Date.now;\n    this.#shadowValidationPercent =\n      opts.shadowValidationPercent ?? DEFAULT_SHADOW_VALIDATION_PERCENT;\n    assert(\n      Number.isInteger(this.#shadowValidationPercent) &&\n        this.#shadowValidationPercent >= 0 &&\n        this.#shadowValidationPercent <= 100,\n      'SQLite change-log shadow validation percentage must be an integer between 0 and 100',\n    );\n  }\n\n  /** False once the writer has failed soft, or after {@link close}. */\n  get enabled(): boolean {\n    return !this.#disabled;\n  }\n\n  /**\n   * The writer's own connection, which is the one purge runs on (§3.3): one\n   * process appends *and* deletes, on one handle, with no second writable\n   * connection and no cross-process barrier. `undefined` until the stream\n   * loop's first reconcile creates the log, and again once the writer has\n   * failed soft and deleted the file — both of which a purge cycle treats as\n   * \"skip\", not as an error.\n   */\n  get connection(): Database | undefined {\n    return this.#db;\n  }\n\n  /** Exposed for tests and for the integration assertions on the metrics. */\n  state(): SQLiteChangeLogObservabilityState | undefined {\n    return this.#observer?.state();\n  }\n\n  /**\n   * Opens the log if necessary and reconciles it with the Postgres resume\n   * point. Returns the resume point stored in the log.\n   *\n   * Returns `undefined` when the writer is disabled. The caller must then use\n   * another source.\n   *\n   * This runs for every connection because Postgres supplies a new resume point\n   * after each reconnect. The log can contain transactions above that point.\n   * Reconciliation removes them before Postgres sends them again.\n   *\n   * `resumeFrom` includes cookies because the cookies must match the resume\n   * watermark.\n   */\n  reconcile(\n    resumeFrom: ChangeLogResumePoint,\n  ): ChangeLogResumePoint | undefined {\n    return this.#reconcile(() => resumeFrom);\n  }\n\n  /**\n   * Reconciles the log without a Postgres resume point.\n   *\n   * A valid log uses its own head and cookies. A new or invalid log uses the\n   * replica values returned by `seed`.\n   */\n  reconcileFromLog(\n    seed: () => ChangeLogResumePoint,\n  ): ChangeLogResumePoint | undefined {\n    return this.#reconcile(db => this.#resumeFromLog(db, seed));\n  }\n\n  #reconcile(\n    resolve: (db: Database) => ChangeLogResumePoint,\n  ): ChangeLogResumePoint | undefined {\n    if (this.#disabled) {\n      return undefined;\n    }\n    const {replicaFile, identity} = this.#opts;\n    const nowMs = this.#now();\n    const anchor = (db: Database): ChangeLogAnchor => ({\n      ...resolve(db),\n      identity,\n      nowMs,\n    });\n    try {\n      const db = this.#db;\n      if (db === undefined) {\n        const opened = openChangeLogDBForWriting(this.#lc, replicaFile, anchor);\n        this.#db = opened.db;\n        if (opened.replacedFile) {\n          this.#opts.onRebuilt?.();\n        }\n        recordSQLiteChangeLogReconcile(this.#lc, opened.result);\n        const info = getSQLiteChangeLogInfo(opened.db);\n        logSQLiteChangeLogStartup(\n          this.#lc,\n          changeLogFileName(replicaFile),\n          info,\n        );\n        this.#observer = new SQLiteChangeLogObserver(this.#lc, info);\n      } else {\n        const resolved = anchor(db);\n        let result: ReconcileResult;\n        try {\n          result = reconcileChangeLog(this.#lc, db, resolved, {\n            rebuildInsteadOfUnboundedWork: true,\n          });\n        } catch (e) {\n          if (!(e instanceof ChangeLogRebuildRequired)) {\n            throw e;\n          }\n          // Close every handle to the old inode before unlinking it. The writer\n          // is reconstructed below against the replacement connection.\n          this.#writer = undefined;\n          this.#db = undefined;\n          db.close();\n          this.#opts.onRebuilt?.();\n          const rebuilt = rebuildChangeLogDBForWriting(\n            this.#lc,\n            replicaFile,\n            resolved,\n            e,\n          );\n          this.#db = rebuilt.db;\n          result = rebuilt.result;\n        }\n        recordSQLiteChangeLogReconcile(this.#lc, result);\n        if (result.action !== 'none') {\n          this.#observer?.reconciled(\n            getSQLiteChangeLogInfo(\n              must(this.#db, 'the SQLite change log is not open'),\n            ),\n          );\n        }\n      }\n      // A reseed drops and recreates the stream table, so the append statements\n      // are prepared fresh against whatever the reconciliation left behind.\n      const reconciledDB = must(this.#db, 'the SQLite change log is not open');\n      this.#writer = new ChangeLogStreamWriter(\n        new StatementRunner(reconciledDB),\n      );\n      // Read both values after reconciliation. A truncate or reseed updates the\n      // head and cookies in the same transaction.\n      return {\n        resumeWatermark: must(\n          readChangeLogHead(reconciledDB),\n          'the SQLite change log has no head after reconciliation',\n        ),\n        cookies: readCookies(reconciledDB),\n      };\n    } catch (e) {\n      this.#failSoft('reconciling the SQLite change log', e);\n      return undefined;\n    }\n  }\n\n  /**\n   * Returns the resume point supplied by the log.\n   *\n   * A valid log uses its own head and cookies. A log that must be recreated uses\n   * the replica values returned by `seed`.\n   */\n  #resumeFromLog(\n    db: Database,\n    seed: () => ChangeLogResumePoint,\n  ): ChangeLogResumePoint {\n    if (changeLogWipeReason(db, this.#opts.identity) !== undefined) {\n      return seed();\n    }\n    const head = readChangeLogHead(db);\n    // An empty log has no resume point, so use the replica seed.\n    return head === null\n      ? seed()\n      : {resumeWatermark: head, cookies: readCookies(db)};\n  }\n\n  /**\n   * Writes one stream message, committing at transaction boundaries. Never\n   * throws: a write failure disables the writer instead of failing the stream.\n   */\n  write(\n    change: ChangeStreamData,\n    json: string,\n    storedChange = extractChangeSubstring(json, change[1].tag),\n  ): void {\n    const writer = this.#writer;\n    const db = this.#db;\n    if (this.#disabled || writer === undefined || db === undefined) {\n      return;\n    }\n    const [type, msg] = change;\n    if (type === 'begin') {\n      this.#validateTransaction = this.#isSelectedForShadowValidation(\n        change[2].commitWatermark,\n      );\n    }\n    const validateTransaction = this.#validateTransaction;\n    const observer = this.#observer;\n    observer?.messageReceived(change, json, validateTransaction, storedChange);\n    const startedAt = validateTransaction ? performance.now() : undefined;\n    try {\n      let commit: ChangeLogCommit | null = null;\n      switch (type) {\n        case 'begin':\n          writer.begin(\n            change[2].commitWatermark,\n            json,\n            storedChange,\n            validateTransaction,\n          );\n          break;\n        case 'commit': {\n          const {watermark} = change[2];\n          const commitStart = performance.now();\n          const stats = writer.commit(\n            watermark,\n            json,\n            this.#now(),\n            storedChange,\n          );\n          commit = {\n            watermark,\n            stats,\n            logCommitMs: performance.now() - commitStart,\n          };\n          break;\n        }\n        case 'rollback':\n          writer.rollback();\n          break;\n        default:\n          writer.append(json, msg, storedChange);\n          break;\n      }\n      observer?.messageProcessed(\n        change,\n        commit,\n        startedAt === undefined ? undefined : performance.now() - startedAt,\n      );\n      if (commit !== null && commit.stats.cookieMutations.length > 0) {\n        // Only re-counted when a schema change actually moved the cookie jar,\n        // which keeps the count off the forward path of every other\n        // transaction. The counts are read after the commit, so they are what\n        // a reader of the file would see.\n        observer?.cookiesMutated(\n          commit.stats.cookieMutations,\n          readCookieRowCounts(db),\n        );\n      }\n      if (commit !== null) {\n        // The head advanced, and it is durable. Releasing the barrier here is\n        // what makes its poll interval a backstop rather than the mechanism.\n        this.#opts.onCommit?.(commit.watermark);\n      }\n      if (type === 'commit' || type === 'rollback') {\n        this.#validateTransaction = false;\n      }\n    } catch (e) {\n      observer?.messageFailed(\n        change,\n        e,\n        startedAt === undefined ? undefined : performance.now() - startedAt,\n      );\n      this.#validateTransaction = false;\n      if (isConstraintViolation(e)) {\n        // Reconciliation should have truncated whatever this collided with.\n        // Report it as an invariant failure before failing soft, so that a\n        // truncate-above bug cannot look like a lost cache.\n        observer?.invariantFailure(\n          'SQLite change-log insert violated a constraint, which means a ' +\n            'transaction above the resume watermark was not truncated',\n          {tag: msg.tag, error: String(e)},\n        );\n      }\n      this.#failSoft('writing to the SQLite change log', e);\n    }\n  }\n\n  #isSelectedForShadowValidation(watermark: string): boolean {\n    const percent = this.#shadowValidationPercent;\n    return (\n      percent >= 100 ||\n      (percent > 0 &&\n        h32(\n          `${this.#opts.identity.generation}/${this.#opts.identity.replicaID ?? ''}:${watermark}`,\n        ) %\n          100 <\n          percent)\n    );\n  }\n\n  /**\n   * Discards the open transaction, if any. Called when the change stream is\n   * interrupted mid-transaction.\n   */\n  abort(): void {\n    const writer = this.#writer;\n    this.#validateTransaction = false;\n    if (this.#disabled || writer === undefined) {\n      return;\n    }\n    this.#observer?.abort();\n    try {\n      writer.rollback();\n    } catch (e) {\n      this.#failSoft('rolling back the SQLite change log', e);\n    }\n  }\n\n  close(): void {\n    this.#disabled = true;\n    this.#validateTransaction = false;\n    this.#writer = undefined;\n    this.#observer = undefined;\n    const db = this.#db;\n    this.#db = undefined;\n    try {\n      db?.close();\n    } catch (e) {\n      this.#lc.warn?.('error closing the SQLite change log', e);\n    }\n  }\n\n  /**\n   * Logs, disables the writer, deletes the file, and closes the reader, leaving\n   * replication running. Disabling is for the life of the process: a log that\n   * failed a write is not re-opened until the task restarts, so the behavior a\n   * subscriber sees is the same as `sqliteChangeLogMode=off`.\n   */\n  #failSoft(what: string, e: unknown): void {\n    const file = changeLogFileName(this.#opts.replicaFile);\n    if (isSQLiteCorruption(e)) {\n      // Open-time corruption is rebuilt by openChangeLogDBForWriting. Reaching\n      // here means the file went bad under an open handle, which the same\n      // delete-and-reseed answers -- on the next process start rather than now.\n      logSQLiteCorruptionDiagnostics(this.#lc, 'change-log', file, e);\n    }\n    this.#lc.error?.(\n      `error ${what}: disabling the writer and deleting ${file}. ` +\n        `Replication continues; catchup reach is lost until this task restarts.`,\n      e,\n    );\n    this.close();\n    try {\n      deleteChangeLogDB(this.#opts.replicaFile);\n    } catch (deleteError) {\n      this.#lc.error?.(`unable to delete ${file}`, deleteError);\n    }\n    this.#opts.onDisabled?.();\n  }\n}\n\nfunction isConstraintViolation(e: unknown): boolean {\n  return e instanceof SqliteError && e.code.startsWith('SQLITE_CONSTRAINT');\n}\n"],"mappings":";;;;;;;;;;;;AAsEA,IAAM,oCAAoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgC1C,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CACA;CAEA;CACA;CACA;CACA,YAAY;CACZ,uBAAuB;CAEvB,YAAY,IAAgB,MAAoC;EAC9D,KAAKA,MAAM,GAAG,YAAY,aAAa,0BAA0B;EACjE,KAAKC,QAAQ;EACb,KAAKC,OAAO,KAAK,OAAO,KAAK;EAC7B,KAAKC,2BACH,KAAK,2BAA2B;EAClC,OACE,OAAO,UAAU,KAAKA,wBAAwB,KAC5C,KAAKA,4BAA4B,KACjC,KAAKA,4BAA4B,KACnC,qFACF;CACF;;CAGA,IAAI,UAAmB;EACrB,OAAO,CAAC,KAAKC;CACf;;;;;;;;;CAUA,IAAI,aAAmC;EACrC,OAAO,KAAKC;CACd;;CAGA,QAAuD;EACrD,OAAO,KAAKC,WAAW,MAAM;CAC/B;;;;;;;;;;;;;;;CAgBA,UACE,YACkC;EAClC,OAAO,KAAKC,iBAAiB,UAAU;CACzC;;;;;;;CAQA,iBACE,MACkC;EAClC,OAAO,KAAKA,YAAW,OAAM,KAAKC,eAAe,IAAI,IAAI,CAAC;CAC5D;CAEA,WACE,SACkC;EAClC,IAAI,KAAKJ,WACP;EAEF,MAAM,EAAC,aAAa,aAAY,KAAKH;EACrC,MAAM,QAAQ,KAAKC,KAAK;EACxB,MAAM,UAAU,QAAmC;GACjD,GAAG,QAAQ,EAAE;GACb;GACA;EACF;EACA,IAAI;GACF,MAAM,KAAK,KAAKG;GAChB,IAAI,OAAO,KAAA,GAAW;IACpB,MAAM,SAAS,0BAA0B,KAAKL,KAAK,aAAa,MAAM;IACtE,KAAKK,MAAM,OAAO;IAClB,IAAI,OAAO,cACT,KAAKJ,MAAM,YAAY;IAEzB,+BAA+B,KAAKD,KAAK,OAAO,MAAM;IACtD,MAAM,OAAO,uBAAuB,OAAO,EAAE;IAC7C,0BACE,KAAKA,KACL,kBAAkB,WAAW,GAC7B,IACF;IACA,KAAKM,YAAY,IAAI,wBAAwB,KAAKN,KAAK,IAAI;GAC7D,OAAO;IACL,MAAM,WAAW,OAAO,EAAE;IAC1B,IAAI;IACJ,IAAI;KACF,SAAS,mBAAmB,KAAKA,KAAK,IAAI,UAAU,EAClD,+BAA+B,KACjC,CAAC;IACH,SAAS,GAAG;KACV,IAAI,EAAE,aAAa,2BACjB,MAAM;KAIR,KAAKS,UAAU,KAAA;KACf,KAAKJ,MAAM,KAAA;KACX,GAAG,MAAM;KACT,KAAKJ,MAAM,YAAY;KACvB,MAAM,UAAU,6BACd,KAAKD,KACL,aACA,UACA,CACF;KACA,KAAKK,MAAM,QAAQ;KACnB,SAAS,QAAQ;IACnB;IACA,+BAA+B,KAAKL,KAAK,MAAM;IAC/C,IAAI,OAAO,WAAW,QACpB,KAAKM,WAAW,WACd,uBACE,KAAK,KAAKD,KAAK,mCAAmC,CACpD,CACF;GAEJ;GAGA,MAAM,eAAe,KAAK,KAAKA,KAAK,mCAAmC;GACvE,KAAKI,UAAU,IAAI,sBACjB,IAAI,gBAAgB,YAAY,CAClC;GAGA,OAAO;IACL,iBAAiB,KACf,kBAAkB,YAAY,GAC9B,wDACF;IACA,SAAS,YAAY,YAAY;GACnC;EACF,SAAS,GAAG;GACV,KAAKC,UAAU,qCAAqC,CAAC;GACrD;EACF;CACF;;;;;;;CAQA,eACE,IACA,MACsB;EACtB,IAAI,oBAAoB,IAAI,KAAKT,MAAM,QAAQ,MAAM,KAAA,GACnD,OAAO,KAAK;EAEd,MAAM,OAAO,kBAAkB,EAAE;EAEjC,OAAO,SAAS,OACZ,KAAK,IACL;GAAC,iBAAiB;GAAM,SAAS,YAAY,EAAE;EAAC;CACtD;;;;;CAMA,MACE,QACA,MACA,eAAe,uBAAuB,MAAM,OAAO,GAAG,GAAG,GACnD;EACN,MAAM,SAAS,KAAKQ;EACpB,MAAM,KAAK,KAAKJ;EAChB,IAAI,KAAKD,aAAa,WAAW,KAAA,KAAa,OAAO,KAAA,GACnD;EAEF,MAAM,CAAC,MAAM,OAAO;EACpB,IAAI,SAAS,SACX,KAAKO,uBAAuB,KAAKC,+BAC/B,OAAO,GAAG,eACZ;EAEF,MAAM,sBAAsB,KAAKD;EACjC,MAAM,WAAW,KAAKL;EACtB,UAAU,gBAAgB,QAAQ,MAAM,qBAAqB,YAAY;EACzE,MAAM,YAAY,sBAAsB,YAAY,IAAI,IAAI,KAAA;EAC5D,IAAI;GACF,IAAI,SAAiC;GACrC,QAAQ,MAAR;IACE,KAAK;KACH,OAAO,MACL,OAAO,GAAG,iBACV,MACA,cACA,mBACF;KACA;IACF,KAAK,UAAU;KACb,MAAM,EAAC,cAAa,OAAO;KAC3B,MAAM,cAAc,YAAY,IAAI;KAOpC,SAAS;MACP;MACA,OARY,OAAO,OACnB,WACA,MACA,KAAKJ,KAAK,GACV,YAIA;MACA,aAAa,YAAY,IAAI,IAAI;KACnC;KACA;IACF;IACA,KAAK;KACH,OAAO,SAAS;KAChB;IACF;KACE,OAAO,OAAO,MAAM,KAAK,YAAY;KACrC;GACJ;GACA,UAAU,iBACR,QACA,QACA,cAAc,KAAA,IAAY,KAAA,IAAY,YAAY,IAAI,IAAI,SAC5D;GACA,IAAI,WAAW,QAAQ,OAAO,MAAM,gBAAgB,SAAS,GAK3D,UAAU,eACR,OAAO,MAAM,iBACb,oBAAoB,EAAE,CACxB;GAEF,IAAI,WAAW,MAGb,KAAKD,MAAM,WAAW,OAAO,SAAS;GAExC,IAAI,SAAS,YAAY,SAAS,YAChC,KAAKU,uBAAuB;EAEhC,SAAS,GAAG;GACV,UAAU,cACR,QACA,GACA,cAAc,KAAA,IAAY,KAAA,IAAY,YAAY,IAAI,IAAI,SAC5D;GACA,KAAKA,uBAAuB;GAC5B,IAAI,sBAAsB,CAAC,GAIzB,UAAU,iBACR,0HAEA;IAAC,KAAK,IAAI;IAAK,OAAO,OAAO,CAAC;GAAC,CACjC;GAEF,KAAKD,UAAU,oCAAoC,CAAC;EACtD;CACF;CAEA,+BAA+B,WAA4B;EACzD,MAAM,UAAU,KAAKP;EACrB,OACE,WAAW,OACV,UAAU,KACT,IACE,GAAG,KAAKF,MAAM,SAAS,WAAW,GAAG,KAAKA,MAAM,SAAS,aAAa,GAAG,GAAG,WAC9E,IACE,MACA;CAER;;;;;CAMA,QAAc;EACZ,MAAM,SAAS,KAAKQ;EACpB,KAAKE,uBAAuB;EAC5B,IAAI,KAAKP,aAAa,WAAW,KAAA,GAC/B;EAEF,KAAKE,WAAW,MAAM;EACtB,IAAI;GACF,OAAO,SAAS;EAClB,SAAS,GAAG;GACV,KAAKI,UAAU,sCAAsC,CAAC;EACxD;CACF;CAEA,QAAc;EACZ,KAAKN,YAAY;EACjB,KAAKO,uBAAuB;EAC5B,KAAKF,UAAU,KAAA;EACf,KAAKH,YAAY,KAAA;EACjB,MAAM,KAAK,KAAKD;EAChB,KAAKA,MAAM,KAAA;EACX,IAAI;GACF,IAAI,MAAM;EACZ,SAAS,GAAG;GACV,KAAKL,IAAI,OAAO,uCAAuC,CAAC;EAC1D;CACF;;;;;;;CAQA,UAAU,MAAc,GAAkB;EACxC,MAAM,OAAO,kBAAkB,KAAKC,MAAM,WAAW;EACrD,IAAI,mBAAmB,CAAC,GAItB,+BAA+B,KAAKD,KAAK,cAAc,MAAM,CAAC;EAEhE,KAAKA,IAAI,QACP,SAAS,KAAK,sCAAsC,KAAK,2EAEzD,CACF;EACA,KAAK,MAAM;EACX,IAAI;GACF,kBAAkB,KAAKC,MAAM,WAAW;EAC1C,SAAS,aAAa;GACpB,KAAKD,IAAI,QAAQ,oBAAoB,QAAQ,WAAW;EAC1D;EACA,KAAKC,MAAM,aAAa;CAC1B;AACF;AAEA,SAAS,sBAAsB,GAAqB;CAClD,OAAO,aAAa,eAAe,EAAE,KAAK,WAAW,mBAAmB;AAC1E"}

package/out/zero-cache/src/services/change-streamer/storer.d.ts

@@ -9,13 +9,27 @@ import type { UpstreamStatusMessage } from '../change-source/protocol/current/st
 import { type CookieSet } from '../replicator/change-log-cookies.ts';
 import type { ReplicatorMode } from '../replicator/replicator.ts';
 import type { Service } from '../service.ts';
-import { type ChangeLogEntry } from './change-log-codec.ts';
+import { type ChangeLogEntry, type SerializedChangeStreamData } from './change-log-codec.ts';
+import { type ChangeLogReinitialization } from './schema/tables.ts';
 import type { Subscriber } from './subscriber.ts';
 /**
  * Factory for creating dynamic connection pools used by the Storer to
  * isolate concurrent transactions into separate pg clients.
  */
 export type PostgresDBProvider = (applicationName: string, maxConns: number) => PostgresDB;
+export type OwnershipTakeover = {
+    /**
+     * The lock that prevents the previous owner from purging the change log
+     * (from which this change-streamer's replica was restored), released once
+     * ownership is assumed.
+     */
+    purgeLock: PurgeLock | null;
+    /**
+     * Requests a restart of the change stream (at a transaction boundary),
+     * which is used to start pending backfills once ownership is assumed.
+     */
+    requestRestart: (reason: string) => void;
+};
 export type TuningOptions = {
     backPressureLimitHeapProportion: number;
     statementTimeoutMs: number;
@@ -56,7 +70,65 @@ export declare class Storer implements Service {
     #private;
     readonly id = "storer";
     constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, dbProvider: PostgresDBProvider, replicaVersion: string, onCommitted: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs, }: TuningOptions);
-    assumeOwnership(purgeLock?: PurgeLock | null): Promise<void>;
+    /**
+     * Re-initializes a change log that is no longer maintained from the
+     * replica's state, taking ownership of it in the process (which is then
+     * assumed again at its new head, see {@link assumeOwnershipAtHead}).
+     * See {@link reinitializeChangeLog}.
+     */
+    reinitializeFromReplica(params: Omit<ChangeLogReinitialization, 'owner' | 'ownerAddress'>): Promise<'reinitialized' | 'maintained'>;
+    /**
+     * Prepares to assume ownership of the change log at its head, while it may
+     * still be owned and maintained by another change-streamer. This allows the
+     * previous owner to continue to serve subscribers while this change-source's
+     * upstream decodes the WAL to catch up to the head of the change log, which
+     * may be many minutes worth of transactions for large DBs.
+     *
+     * Once the change stream is started from the head of the change log:
+     *
+     * * Transactions that the change log already contains (i.e. stored by the
+     *   previous owner) are not stored again, but are otherwise processed
+     *   normally (e.g. for upstream acks, since they have been stored).
+     * * Ownership is assumed in the transaction that stores the first
+     *   transaction past the head of the change log. The previous owner's
+     *   (subsequent) transactions fail its ownership check, which makes this
+     *   transaction the one immediately after the previous owner's last one.
+     * * Ownership is also assumed upon a {@link status} at or past the head of
+     *   the change log (outside of a transaction), which covers an idle
+     *   upstream: all transactions up to the status position have been
+     *   received, and any past the head would have been stored.
+     *
+     * Subscribers are caught up from the change log as usual, as the stream
+     * (which is forwarded to them) starts from its head.
+     *
+     * Backfill transactions are synthesized by each change-streamer's change
+     * source, and thus never correspond to those stored by the previous owner.
+     * Until ownership is assumed, they are not stored, and no backfills are
+     * requested from the change log (see
+     * {@link getStartStreamInitializationParameters}). Once ownership is
+     * assumed, a restart of the change stream is requested if the change log has
+     * pending backfills or if a backfill transaction was skipped.
+     */
+    assumeOwnershipAtHead(takeover: OwnershipTakeover): void;
+    /**
+     * Whether ownership of the change log is pending, i.e. after
+     * {@link assumeOwnershipAtHead} and before ownership is assumed.
+     */
+    get ownershipPending(): boolean;
+    /**
+     * Relays a status message received outside of a transaction, which is
+     * used to assume ownership of the change log on an idle upstream. A no-op
+     * unless ownership is pending.
+     */
+    status(watermark: string): void;
+    /**
+     * Assumes ownership of the change log immediately, regardless of its head.
+     * This is used for in tests.
+     *
+     * A production change-streamer instead takes over the change log at its head
+     * (see {@link assumeOwnershipAtHead}).
+     */
+    assumeOwnership(): Promise<void>;
     /**
      * The whole of what a stream connection starts from: where to resume, what to
      * ask the change source to backfill, and the raw cookie set those requests
@@ -106,7 +178,7 @@ export declare class Storer implements Service {
     /**
      * @returns The JSON stringified stream message to be sent downstream.
      */
-    store(watermark: string, data: ChangeStreamData): string;
+    store(watermark: string, data: ChangeStreamData, serialized?: SerializedChangeStreamData): string;
     abort(): void;
     catchup(subscriber: Subscriber, mode: ReplicatorMode): void;
     readyForMore(): Promise<void> | undefined;
@@ -131,13 +203,32 @@ export declare class PurgeLock {
     #private;
     readonly replicaVersion: string;
     readonly minWatermark: string;
-    constructor(lc: LogContext, tx: TransactionPool, replicaVersion: string, watermark: string);
+    /**
+     * The head of the change log (i.e. its `lastWatermark`) when the lock was
+     * acquired. The change log is resumed from its head, which only advances,
+     * so a stream cannot (correctly) be started from before it.
+     */
+    readonly headWatermark: string;
+    constructor(lc: LogContext, tx: TransactionPool, replicaVersion: string, watermark: string, headWatermark: string);
     release(): Promise<void>;
 }
 export declare class PurgeLocker {
     #private;
     constructor(lc: LogContext, shard: ShardID, db: PostgresDB);
+    /**
+     * Locks the oldest row of the change log, preventing it from being purged.
+     *
+     * @param slotWatermark The position of the replication slot from which the
+     *     change log is to be resumed (as a watermark). If specified and the
+     *     head of the change log is behind it, the change log may be missing
+     *     transactions that the slot will not stream, in which case nothing is
+     *     locked and `'behind-slot'` is returned. Conversely, a head at or past
+     *     the slot's position means that the stream (which resumes from the
+     *     head) will not skip anything.
+     * @returns `null` if the change log is empty.
+     */
     acquire(): Promise<PurgeLock | null>;
+    acquire(slotWatermark?: string | undefined): Promise<PurgeLock | null | 'behind-slot'>;
 }
 type TaskKey = JSONObject;
 type TaskDoneFn = () => void;

package/out/zero-cache/src/services/change-streamer/storer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,uCAAuC,CAAC;AAM/C,OAAO,EAAiB,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,6CAA6C,CAAC;AACvF,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,EAIL,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAQ/B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM,KACb,UAAU,CAAC;AAsChB,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;IA4BvB,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,kBAAkB,EAC9B,cAAc,EAAE,MAAM,EACtB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACxD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa,EAkCjB;IAgCK,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI,iBA4BjD;IAED;;;;;;;;;;;;;;;;OAgBG;IACG,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;QACpC,OAAO,EAAE,SAAS,CAAC;KACpB,CAAC,CAsDD;IAEK,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAOxD;IAED;;;;OAIG;IACG,gBAAgB,IAAI,OAAO,CAAC;QAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CASD;IAED;;;OAGG;IACG,oBAAoB,CACxB,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB;IAED;;;;;;OAMG;IACH,gBAAgB,CACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,SAAS,SAAO,GACf,aAAa,CAAC,cAAc,EAAE,CAAC,CAYjC;IAMD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAuCrD;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,UAgB9C;IAED,KAAK,SAEJ;IAED,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,QAEnD;IAID,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAwBxC;IAmDD;;;OAGG;IACG,GAAG,kBA6BR;IAmfD;;;OAGG;IACG,YAAY,kBAMjB;IAED;;;;OAIG;IACG,IAAI,kBAcT;CACF;AAED,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAE9B,YACE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,EAMlB;IAIK,OAAO,kBAUZ;CACF;AAED,qBAAa,WAAW;;IAKtB,YAAY,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,EAIzD;IAOK,OAAO,8BA2BZ;CACF;AAKD,KAAK,OAAO,GAAG,UAAU,CAAC;AAE1B,KAAK,UAAU,GAAG,MAAM,IAAI,CAAC;AAE7B;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,eAAe;;IAQ1B,YACE,EAAE,EAAE,UAAU,EACd,kBAAkB,EAAE,MAAM,EAC1B,SAAS,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAKhC;IAED,KAAK,SAMJ;IAGD,iBAAiB,CAAC,GAAG,EAAE,IAAI,QAgB1B;IAED;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,OAAa,GAAG,UAAU,CAKrD;IAED,IAAI,SAGH;CACF"}
+{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,uCAAuC,CAAC;AAO/C,OAAO,EAAiB,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,6CAA6C,CAAC;AACvF,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,0BAA0B,EAChC,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAKL,KAAK,yBAAyB,EAE/B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM,KACb,UAAU,CAAC;AAsDhB,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;;;OAIG;IACH,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;IAC5B;;;OAGG;IACH,cAAc,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1C,CAAC;AAQF,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;IA0CvB,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,kBAAkB,EAC9B,cAAc,EAAE,MAAM,EACtB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACxD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa,EAkCjB;IAyCD;;;;;OAKG;IACH,uBAAuB,CACrB,MAAM,EAAE,IAAI,CAAC,yBAAyB,EAAE,OAAO,GAAG,cAAc,CAAC,GAChE,OAAO,CAAC,eAAe,GAAG,YAAY,CAAC,CAUzC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,QAOhD;IAED;;;OAGG;IACH,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED;;;;OAIG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,QAIvB;IA6DD;;;;;;OAMG;IACG,eAAe,kBAcpB;IAED;;;;;;;;;;;;;;;;OAgBG;IACG,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;QACpC,OAAO,EAAE,SAAS,CAAC;KACpB,CAAC,CA4DD;IAEK,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAOxD;IAED;;;;OAIG;IACG,gBAAgB,IAAI,OAAO,CAAC;QAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CASD;IAED;;;OAGG;IACG,oBAAoB,CACxB,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB;IAED;;;;;;OAMG;IACH,gBAAgB,CACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,SAAS,SAAO,GACf,aAAa,CAAC,cAAc,EAAE,CAAC,CAYjC;IAMD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAuCrD;IAED;;OAEG;IACH,KAAK,CACH,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,gBAAgB,EACtB,UAAU,GAAE,0BAEX,UAkBF;IAED,KAAK,SAEJ;IAED,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,QAEnD;IAID,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAwBxC;IAmDD;;;OAGG;IACG,GAAG,kBA6BR;IA6kBD;;;OAGG;IACG,YAAY,kBAMjB;IAED;;;;OAIG;IACG,IAAI,kBAcT;CACF;AAED,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;OAIG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAE/B,YACE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EAOtB;IAIK,OAAO,kBAUZ;CACF;AAED,qBAAa,WAAW;;IAKtB,YAAY,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,EAIzD;IAOD;;;;;;;;;;;OAWG;IACH,OAAO,IAAI,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IACrC,OAAO,CACL,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,GACjC,OAAO,CAAC,SAAS,GAAG,IAAI,GAAG,aAAa,CAAC,CAAC;CA6D9C;AAKD,KAAK,OAAO,GAAG,UAAU,CAAC;AAE1B,KAAK,UAAU,GAAG,MAAM,IAAI,CAAC;AAE7B;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,eAAe;;IAQ1B,YACE,EAAE,EAAE,UAAU,EACd,kBAAkB,EAAE,MAAM,EAC1B,SAAS,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAKhC;IAED,KAAK,SAMJ;IAGD,iBAAiB,CAAC,GAAG,EAAE,IAAI,QAgB1B;IAED;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,OAAa,GAAG,UAAU,CAKrD;IAED,IAAI,SAGH;CACF"}