@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.10

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js

@@ -0,0 +1,258 @@
+import { must } from "../../../../shared/src/must.js";
+import { readCookieRowCounts, readCookies } from "../replicator/change-log-cookies.js";
+import { StatementRunner } from "../../db/statements.js";
+import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics } from "../../db/sqlite-corruption.js";
+import { ChangeLogRebuildRequired, changeLogFileName, changeLogWipeReason, deleteChangeLogDB, openChangeLogDBForWriting, readChangeLogHead, rebuildChangeLogDBForWriting, reconcileChangeLog } from "../replicator/change-log-db.js";
+import { ChangeLogStreamWriter } from "../replicator/change-log-stream-writer.js";
+import { SQLiteChangeLogObserver, getSQLiteChangeLogInfo, logSQLiteChangeLogStartup, recordSQLiteChangeLogReconcile } from "../replicator/sqlite-change-log-observability.js";
+import { SqliteError } from "@rocicorp/zero-sqlite3";
+//#region ../zero-cache/src/services/change-streamer/sqlite-change-log-writer.ts
+/**
+* Appends the canonical change stream to the SQLite change log, from the
+* change-streamer's own stream loop.
+*
+* ### Ordering
+*
+* The log's commit for a transaction precedes the forward of that transaction's
+* `commit` message, and nothing `await`s in between. Everything that can
+* advance the watermark the stream would resume from — the storer's Postgres
+* commit, `lastWatermark`, the upstream ACK — runs strictly after it, so the
+* log's head is always at or above the resume point. A *hole* is therefore
+* unreachable at any crash timing, and a *phantom* is the normal state, which is
+* what makes truncate-above the whole of reconciliation.
+*
+* ### Fail-soft
+*
+* The log is a catchup cache: nothing in it is not either already durable in the
+* replica's litestream backup or re-derivable from the replication slot. A write
+* failure must therefore not stop replication for the shard. Corruption is
+* rebuilt once at open; any other write error — a full disk, an I/O error, a
+* permissions problem — logs, disables the writer, deletes the file, closes the
+* reader, and lets the stream continue. The cost is catchup reach, and reach is
+* recoverable: `too-old` sends a subscriber to a restore.
+*
+* One carve-out keeps that from hiding the bug class this topology introduces. A
+* missed truncate-above surfaces as `SQLITE_CONSTRAINT` on the writer's plain
+* `INSERT`, which is "any write error other than corruption" and would be
+* absorbed quietly. It is counted as an invariant failure and alerted on before
+* failing soft.
+*/
+var SQLiteChangeLogWriter = class {
+	#lc;
+	#opts;
+	#now;
+	#db;
+	#writer;
+	#observer;
+	#disabled = false;
+	constructor(lc, opts) {
+		this.#lc = lc.withContext("component", "sqlite-change-log-writer");
+		this.#opts = opts;
+		this.#now = opts.now ?? Date.now;
+	}
+	/** False once the writer has failed soft, or after {@link close}. */
+	get enabled() {
+		return !this.#disabled;
+	}
+	/**
+	* The writer's own connection, which is the one purge runs on (§3.3): one
+	* process appends *and* deletes, on one handle, with no second writable
+	* connection and no cross-process barrier. `undefined` until the stream
+	* loop's first reconcile creates the log, and again once the writer has
+	* failed soft and deleted the file — both of which a purge cycle treats as
+	* "skip", not as an error.
+	*/
+	get connection() {
+		return this.#db;
+	}
+	/** Exposed for tests and for the integration assertions on the metrics. */
+	state() {
+		return this.#observer?.state();
+	}
+	/**
+	* Opens the log if necessary and reconciles it with the Postgres resume
+	* point. Returns the resume point stored in the log.
+	*
+	* Returns `undefined` when the writer is disabled. The caller must then use
+	* another source.
+	*
+	* This runs for every connection because Postgres supplies a new resume point
+	* after each reconnect. The log can contain transactions above that point.
+	* Reconciliation removes them before Postgres sends them again.
+	*
+	* `resumeFrom` includes cookies because the cookies must match the resume
+	* watermark.
+	*/
+	reconcile(resumeFrom) {
+		return this.#reconcile(() => resumeFrom);
+	}
+	/**
+	* Reconciles the log without a Postgres resume point.
+	*
+	* A valid log uses its own head and cookies. A new or invalid log uses the
+	* replica values returned by `seed`.
+	*/
+	reconcileFromLog(seed) {
+		return this.#reconcile((db) => this.#resumeFromLog(db, seed));
+	}
+	#reconcile(resolve) {
+		if (this.#disabled) return;
+		const { replicaFile, identity } = this.#opts;
+		const nowMs = this.#now();
+		const anchor = (db) => ({
+			...resolve(db),
+			identity,
+			nowMs
+		});
+		try {
+			const db = this.#db;
+			if (db === void 0) {
+				const opened = openChangeLogDBForWriting(this.#lc, replicaFile, anchor);
+				this.#db = opened.db;
+				if (opened.replacedFile) this.#opts.onRebuilt?.();
+				recordSQLiteChangeLogReconcile(this.#lc, opened.result);
+				const info = getSQLiteChangeLogInfo(opened.db);
+				logSQLiteChangeLogStartup(this.#lc, changeLogFileName(replicaFile), info);
+				this.#observer = new SQLiteChangeLogObserver(this.#lc, info);
+			} else {
+				const resolved = anchor(db);
+				let result;
+				try {
+					result = reconcileChangeLog(this.#lc, db, resolved, { rebuildInsteadOfUnboundedWork: true });
+				} catch (e) {
+					if (!(e instanceof ChangeLogRebuildRequired)) throw e;
+					this.#writer = void 0;
+					this.#db = void 0;
+					db.close();
+					this.#opts.onRebuilt?.();
+					const rebuilt = rebuildChangeLogDBForWriting(this.#lc, replicaFile, resolved, e);
+					this.#db = rebuilt.db;
+					result = rebuilt.result;
+				}
+				recordSQLiteChangeLogReconcile(this.#lc, result);
+				if (result.action !== "none") this.#observer?.reconciled(getSQLiteChangeLogInfo(must(this.#db, "the SQLite change log is not open")));
+			}
+			const reconciledDB = must(this.#db, "the SQLite change log is not open");
+			this.#writer = new ChangeLogStreamWriter(new StatementRunner(reconciledDB));
+			return {
+				resumeWatermark: must(readChangeLogHead(reconciledDB), "the SQLite change log has no head after reconciliation"),
+				cookies: readCookies(reconciledDB)
+			};
+		} catch (e) {
+			this.#failSoft("reconciling the SQLite change log", e);
+			return;
+		}
+	}
+	/**
+	* Returns the resume point supplied by the log.
+	*
+	* A valid log uses its own head and cookies. A log that must be recreated uses
+	* the replica values returned by `seed`.
+	*/
+	#resumeFromLog(db, seed) {
+		if (changeLogWipeReason(db, this.#opts.identity) !== void 0) return seed();
+		const head = readChangeLogHead(db);
+		return head === null ? seed() : {
+			resumeWatermark: head,
+			cookies: readCookies(db)
+		};
+	}
+	/**
+	* Writes one stream message, committing at transaction boundaries. Never
+	* throws: a write failure disables the writer instead of failing the stream.
+	*/
+	write(change, json) {
+		const writer = this.#writer;
+		const db = this.#db;
+		if (this.#disabled || writer === void 0 || db === void 0) return;
+		const observer = this.#observer;
+		observer?.messageReceived(change, json);
+		const [type, msg] = change;
+		const startedAt = performance.now();
+		try {
+			let commit = null;
+			switch (type) {
+				case "begin":
+					writer.begin(change[2].commitWatermark, json);
+					break;
+				case "commit": {
+					const { watermark } = change[2];
+					const commitStart = performance.now();
+					commit = {
+						watermark,
+						stats: writer.commit(watermark, json, this.#now()),
+						logCommitMs: performance.now() - commitStart
+					};
+					break;
+				}
+				case "rollback":
+					writer.rollback();
+					break;
+				default:
+					writer.append(json, msg);
+					break;
+			}
+			observer?.messageProcessed(change, commit, performance.now() - startedAt);
+			if (commit !== null && commit.stats.cookieMutations.length > 0) observer?.cookiesMutated(commit.stats.cookieMutations, readCookieRowCounts(db));
+			if (commit !== null) this.#opts.onCommit?.(commit.watermark);
+		} catch (e) {
+			observer?.messageFailed(change, e, performance.now() - startedAt);
+			if (isConstraintViolation(e)) observer?.invariantFailure("SQLite change-log insert violated a constraint, which means a transaction above the resume watermark was not truncated", {
+				tag: msg.tag,
+				error: String(e)
+			});
+			this.#failSoft("writing to the SQLite change log", e);
+		}
+	}
+	/**
+	* Discards the open transaction, if any. Called when the change stream is
+	* interrupted mid-transaction.
+	*/
+	abort() {
+		const writer = this.#writer;
+		if (this.#disabled || writer === void 0) return;
+		this.#observer?.abort();
+		try {
+			writer.rollback();
+		} catch (e) {
+			this.#failSoft("rolling back the SQLite change log", e);
+		}
+	}
+	close() {
+		this.#disabled = true;
+		this.#writer = void 0;
+		this.#observer = void 0;
+		const db = this.#db;
+		this.#db = void 0;
+		try {
+			db?.close();
+		} catch (e) {
+			this.#lc.warn?.("error closing the SQLite change log", e);
+		}
+	}
+	/**
+	* Logs, disables the writer, deletes the file, and closes the reader, leaving
+	* replication running. Disabling is for the life of the process: a log that
+	* failed a write is not re-opened until the task restarts, so the behavior a
+	* subscriber sees is the same as `sqliteChangeLogMode=off`.
+	*/
+	#failSoft(what, e) {
+		const file = changeLogFileName(this.#opts.replicaFile);
+		if (isSQLiteCorruption(e)) logSQLiteCorruptionDiagnostics(this.#lc, "change-log", file, e);
+		this.#lc.error?.(`error ${what}: disabling the writer and deleting ${file}. Replication continues; catchup reach is lost until this task restarts.`, e);
+		this.close();
+		try {
+			deleteChangeLogDB(this.#opts.replicaFile);
+		} catch (deleteError) {
+			this.#lc.error?.(`unable to delete ${file}`, deleteError);
+		}
+		this.#opts.onDisabled?.();
+	}
+};
+function isConstraintViolation(e) {
+	return e instanceof SqliteError && e.code.startsWith("SQLITE_CONSTRAINT");
+}
+//#endregion
+export { SQLiteChangeLogWriter };
+
+//# sourceMappingURL=sqlite-change-log-writer.js.map

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlite-change-log-writer.js","names":["#lc","#opts","#now","#disabled","#db","#observer","#reconcile","#resumeFromLog","#writer","#failSoft"],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-writer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n  readCookieRowCounts,\n  readCookies,\n} from '../replicator/change-log-cookies.ts';\nimport {\n  ChangeLogRebuildRequired,\n  changeLogFileName,\n  changeLogWipeReason,\n  deleteChangeLogDB,\n  openChangeLogDBForWriting,\n  readChangeLogHead,\n  rebuildChangeLogDBForWriting,\n  reconcileChangeLog,\n  type ChangeLogAnchor,\n  type ChangeLogIdentity,\n  type ChangeLogResumePoint,\n  type ReconcileResult,\n} from '../replicator/change-log-db.ts';\nimport {ChangeLogStreamWriter} from '../replicator/change-log-stream-writer.ts';\nimport {\n  getSQLiteChangeLogInfo,\n  logSQLiteChangeLogStartup,\n  recordSQLiteChangeLogReconcile,\n  SQLiteChangeLogObserver,\n  type ChangeLogCommit,\n  type SQLiteChangeLogObservabilityState,\n} from '../replicator/sqlite-change-log-observability.ts';\n\nexport type SQLiteChangeLogWriterOptions = {\n  /** The replica file the log is named after, i.e. `${replicaFile}-change-log`. */\n  replicaFile: string;\n  /** The replica whose contents this log belongs to. */\n  identity: ChangeLogIdentity;\n  /**\n   * Called after each transaction's log commit, with the log's new head. This\n   * is what releases the catchup barrier: after the writer moved in-process,\n   * the head advances here rather than at a subscriber's ACK.\n   */\n  onCommit?: ((watermark: string) => void) | undefined;\n  /**\n   * Called when the writer disables itself and deletes the file. The reader is\n   * cached on the service, so a delete that leaves it open would leave an\n   * in-process reader serving an unlinked inode while every new open sees\n   * nothing.\n   */\n  onDisabled?: (() => void) | undefined;\n  /** Called when reconciliation replaces the file, to close cached readers. */\n  onRebuilt?: (() => void) | undefined;\n  /** Overridable for tests. */\n  now?: (() => number) | undefined;\n};\n\n/**\n * Appends the canonical change stream to the SQLite change log, from the\n * change-streamer's own stream loop.\n *\n * ### Ordering\n *\n * The log's commit for a transaction precedes the forward of that transaction's\n * `commit` message, and nothing `await`s in between. Everything that can\n * advance the watermark the stream would resume from — the storer's Postgres\n * commit, `lastWatermark`, the upstream ACK — runs strictly after it, so the\n * log's head is always at or above the resume point. A *hole* is therefore\n * unreachable at any crash timing, and a *phantom* is the normal state, which is\n * what makes truncate-above the whole of reconciliation.\n *\n * ### Fail-soft\n *\n * The log is a catchup cache: nothing in it is not either already durable in the\n * replica's litestream backup or re-derivable from the replication slot. A write\n * failure must therefore not stop replication for the shard. Corruption is\n * rebuilt once at open; any other write error — a full disk, an I/O error, a\n * permissions problem — logs, disables the writer, deletes the file, closes the\n * reader, and lets the stream continue. The cost is catchup reach, and reach is\n * recoverable: `too-old` sends a subscriber to a restore.\n *\n * One carve-out keeps that from hiding the bug class this topology introduces. A\n * missed truncate-above surfaces as `SQLITE_CONSTRAINT` on the writer's plain\n * `INSERT`, which is \"any write error other than corruption\" and would be\n * absorbed quietly. It is counted as an invariant failure and alerted on before\n * failing soft.\n */\nexport class SQLiteChangeLogWriter {\n  readonly #lc: LogContext;\n  readonly #opts: SQLiteChangeLogWriterOptions;\n  readonly #now: () => number;\n\n  #db: Database | undefined;\n  #writer: ChangeLogStreamWriter | undefined;\n  #observer: SQLiteChangeLogObserver | undefined;\n  #disabled = false;\n\n  constructor(lc: LogContext, opts: SQLiteChangeLogWriterOptions) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-writer');\n    this.#opts = opts;\n    this.#now = opts.now ?? Date.now;\n  }\n\n  /** False once the writer has failed soft, or after {@link close}. */\n  get enabled(): boolean {\n    return !this.#disabled;\n  }\n\n  /**\n   * The writer's own connection, which is the one purge runs on (§3.3): one\n   * process appends *and* deletes, on one handle, with no second writable\n   * connection and no cross-process barrier. `undefined` until the stream\n   * loop's first reconcile creates the log, and again once the writer has\n   * failed soft and deleted the file — both of which a purge cycle treats as\n   * \"skip\", not as an error.\n   */\n  get connection(): Database | undefined {\n    return this.#db;\n  }\n\n  /** Exposed for tests and for the integration assertions on the metrics. */\n  state(): SQLiteChangeLogObservabilityState | undefined {\n    return this.#observer?.state();\n  }\n\n  /**\n   * Opens the log if necessary and reconciles it with the Postgres resume\n   * point. Returns the resume point stored in the log.\n   *\n   * Returns `undefined` when the writer is disabled. The caller must then use\n   * another source.\n   *\n   * This runs for every connection because Postgres supplies a new resume point\n   * after each reconnect. The log can contain transactions above that point.\n   * Reconciliation removes them before Postgres sends them again.\n   *\n   * `resumeFrom` includes cookies because the cookies must match the resume\n   * watermark.\n   */\n  reconcile(\n    resumeFrom: ChangeLogResumePoint,\n  ): ChangeLogResumePoint | undefined {\n    return this.#reconcile(() => resumeFrom);\n  }\n\n  /**\n   * Reconciles the log without a Postgres resume point.\n   *\n   * A valid log uses its own head and cookies. A new or invalid log uses the\n   * replica values returned by `seed`.\n   */\n  reconcileFromLog(\n    seed: () => ChangeLogResumePoint,\n  ): ChangeLogResumePoint | undefined {\n    return this.#reconcile(db => this.#resumeFromLog(db, seed));\n  }\n\n  #reconcile(\n    resolve: (db: Database) => ChangeLogResumePoint,\n  ): ChangeLogResumePoint | undefined {\n    if (this.#disabled) {\n      return undefined;\n    }\n    const {replicaFile, identity} = this.#opts;\n    const nowMs = this.#now();\n    const anchor = (db: Database): ChangeLogAnchor => ({\n      ...resolve(db),\n      identity,\n      nowMs,\n    });\n    try {\n      const db = this.#db;\n      if (db === undefined) {\n        const opened = openChangeLogDBForWriting(this.#lc, replicaFile, anchor);\n        this.#db = opened.db;\n        if (opened.replacedFile) {\n          this.#opts.onRebuilt?.();\n        }\n        recordSQLiteChangeLogReconcile(this.#lc, opened.result);\n        const info = getSQLiteChangeLogInfo(opened.db);\n        logSQLiteChangeLogStartup(\n          this.#lc,\n          changeLogFileName(replicaFile),\n          info,\n        );\n        this.#observer = new SQLiteChangeLogObserver(this.#lc, info);\n      } else {\n        const resolved = anchor(db);\n        let result: ReconcileResult;\n        try {\n          result = reconcileChangeLog(this.#lc, db, resolved, {\n            rebuildInsteadOfUnboundedWork: true,\n          });\n        } catch (e) {\n          if (!(e instanceof ChangeLogRebuildRequired)) {\n            throw e;\n          }\n          // Close every handle to the old inode before unlinking it. The writer\n          // is reconstructed below against the replacement connection.\n          this.#writer = undefined;\n          this.#db = undefined;\n          db.close();\n          this.#opts.onRebuilt?.();\n          const rebuilt = rebuildChangeLogDBForWriting(\n            this.#lc,\n            replicaFile,\n            resolved,\n            e,\n          );\n          this.#db = rebuilt.db;\n          result = rebuilt.result;\n        }\n        recordSQLiteChangeLogReconcile(this.#lc, result);\n        if (result.action !== 'none') {\n          this.#observer?.reconciled(\n            getSQLiteChangeLogInfo(\n              must(this.#db, 'the SQLite change log is not open'),\n            ),\n          );\n        }\n      }\n      // A reseed drops and recreates the stream table, so the append statements\n      // are prepared fresh against whatever the reconciliation left behind.\n      const reconciledDB = must(this.#db, 'the SQLite change log is not open');\n      this.#writer = new ChangeLogStreamWriter(\n        new StatementRunner(reconciledDB),\n      );\n      // Read both values after reconciliation. A truncate or reseed updates the\n      // head and cookies in the same transaction.\n      return {\n        resumeWatermark: must(\n          readChangeLogHead(reconciledDB),\n          'the SQLite change log has no head after reconciliation',\n        ),\n        cookies: readCookies(reconciledDB),\n      };\n    } catch (e) {\n      this.#failSoft('reconciling the SQLite change log', e);\n      return undefined;\n    }\n  }\n\n  /**\n   * Returns the resume point supplied by the log.\n   *\n   * A valid log uses its own head and cookies. A log that must be recreated uses\n   * the replica values returned by `seed`.\n   */\n  #resumeFromLog(\n    db: Database,\n    seed: () => ChangeLogResumePoint,\n  ): ChangeLogResumePoint {\n    if (changeLogWipeReason(db, this.#opts.identity) !== undefined) {\n      return seed();\n    }\n    const head = readChangeLogHead(db);\n    // An empty log has no resume point, so use the replica seed.\n    return head === null\n      ? seed()\n      : {resumeWatermark: head, cookies: readCookies(db)};\n  }\n\n  /**\n   * Writes one stream message, committing at transaction boundaries. Never\n   * throws: a write failure disables the writer instead of failing the stream.\n   */\n  write(change: ChangeStreamData, json: string): void {\n    const writer = this.#writer;\n    const db = this.#db;\n    if (this.#disabled || writer === undefined || db === undefined) {\n      return;\n    }\n    const observer = this.#observer;\n    observer?.messageReceived(change, json);\n    const [type, msg] = change;\n    const startedAt = performance.now();\n    try {\n      let commit: ChangeLogCommit | null = null;\n      switch (type) {\n        case 'begin':\n          writer.begin(change[2].commitWatermark, json);\n          break;\n        case 'commit': {\n          const {watermark} = change[2];\n          const commitStart = performance.now();\n          const stats = writer.commit(watermark, json, this.#now());\n          commit = {\n            watermark,\n            stats,\n            logCommitMs: performance.now() - commitStart,\n          };\n          break;\n        }\n        case 'rollback':\n          writer.rollback();\n          break;\n        default:\n          writer.append(json, msg);\n          break;\n      }\n      observer?.messageProcessed(change, commit, performance.now() - startedAt);\n      if (commit !== null && commit.stats.cookieMutations.length > 0) {\n        // Only re-counted when a schema change actually moved the cookie jar,\n        // which keeps the count off the forward path of every other\n        // transaction. The counts are read after the commit, so they are what\n        // a reader of the file would see.\n        observer?.cookiesMutated(\n          commit.stats.cookieMutations,\n          readCookieRowCounts(db),\n        );\n      }\n      if (commit !== null) {\n        // The head advanced, and it is durable. Releasing the barrier here is\n        // what makes its poll interval a backstop rather than the mechanism.\n        this.#opts.onCommit?.(commit.watermark);\n      }\n    } catch (e) {\n      observer?.messageFailed(change, e, performance.now() - startedAt);\n      if (isConstraintViolation(e)) {\n        // Reconciliation should have truncated whatever this collided with.\n        // Report it as an invariant failure before failing soft, so that a\n        // truncate-above bug cannot look like a lost cache.\n        observer?.invariantFailure(\n          'SQLite change-log insert violated a constraint, which means a ' +\n            'transaction above the resume watermark was not truncated',\n          {tag: msg.tag, error: String(e)},\n        );\n      }\n      this.#failSoft('writing to the SQLite change log', e);\n    }\n  }\n\n  /**\n   * Discards the open transaction, if any. Called when the change stream is\n   * interrupted mid-transaction.\n   */\n  abort(): void {\n    const writer = this.#writer;\n    if (this.#disabled || writer === undefined) {\n      return;\n    }\n    this.#observer?.abort();\n    try {\n      writer.rollback();\n    } catch (e) {\n      this.#failSoft('rolling back the SQLite change log', e);\n    }\n  }\n\n  close(): void {\n    this.#disabled = true;\n    this.#writer = undefined;\n    this.#observer = undefined;\n    const db = this.#db;\n    this.#db = undefined;\n    try {\n      db?.close();\n    } catch (e) {\n      this.#lc.warn?.('error closing the SQLite change log', e);\n    }\n  }\n\n  /**\n   * Logs, disables the writer, deletes the file, and closes the reader, leaving\n   * replication running. Disabling is for the life of the process: a log that\n   * failed a write is not re-opened until the task restarts, so the behavior a\n   * subscriber sees is the same as `sqliteChangeLogMode=off`.\n   */\n  #failSoft(what: string, e: unknown): void {\n    const file = changeLogFileName(this.#opts.replicaFile);\n    if (isSQLiteCorruption(e)) {\n      // Open-time corruption is rebuilt by openChangeLogDBForWriting. Reaching\n      // here means the file went bad under an open handle, which the same\n      // delete-and-reseed answers -- on the next process start rather than now.\n      logSQLiteCorruptionDiagnostics(this.#lc, 'change-log', file, e);\n    }\n    this.#lc.error?.(\n      `error ${what}: disabling the writer and deleting ${file}. ` +\n        `Replication continues; catchup reach is lost until this task restarts.`,\n      e,\n    );\n    this.close();\n    try {\n      deleteChangeLogDB(this.#opts.replicaFile);\n    } catch (deleteError) {\n      this.#lc.error?.(`unable to delete ${file}`, deleteError);\n    }\n    this.#opts.onDisabled?.();\n  }\n}\n\nfunction isConstraintViolation(e: unknown): boolean {\n  return e instanceof SqliteError && e.code.startsWith('SQLITE_CONSTRAINT');\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4FA,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CAEA;CACA;CACA;CACA,YAAY;CAEZ,YAAY,IAAgB,MAAoC;EAC9D,KAAKA,MAAM,GAAG,YAAY,aAAa,0BAA0B;EACjE,KAAKC,QAAQ;EACb,KAAKC,OAAO,KAAK,OAAO,KAAK;CAC/B;;CAGA,IAAI,UAAmB;EACrB,OAAO,CAAC,KAAKC;CACf;;;;;;;;;CAUA,IAAI,aAAmC;EACrC,OAAO,KAAKC;CACd;;CAGA,QAAuD;EACrD,OAAO,KAAKC,WAAW,MAAM;CAC/B;;;;;;;;;;;;;;;CAgBA,UACE,YACkC;EAClC,OAAO,KAAKC,iBAAiB,UAAU;CACzC;;;;;;;CAQA,iBACE,MACkC;EAClC,OAAO,KAAKA,YAAW,OAAM,KAAKC,eAAe,IAAI,IAAI,CAAC;CAC5D;CAEA,WACE,SACkC;EAClC,IAAI,KAAKJ,WACP;EAEF,MAAM,EAAC,aAAa,aAAY,KAAKF;EACrC,MAAM,QAAQ,KAAKC,KAAK;EACxB,MAAM,UAAU,QAAmC;GACjD,GAAG,QAAQ,EAAE;GACb;GACA;EACF;EACA,IAAI;GACF,MAAM,KAAK,KAAKE;GAChB,IAAI,OAAO,KAAA,GAAW;IACpB,MAAM,SAAS,0BAA0B,KAAKJ,KAAK,aAAa,MAAM;IACtE,KAAKI,MAAM,OAAO;IAClB,IAAI,OAAO,cACT,KAAKH,MAAM,YAAY;IAEzB,+BAA+B,KAAKD,KAAK,OAAO,MAAM;IACtD,MAAM,OAAO,uBAAuB,OAAO,EAAE;IAC7C,0BACE,KAAKA,KACL,kBAAkB,WAAW,GAC7B,IACF;IACA,KAAKK,YAAY,IAAI,wBAAwB,KAAKL,KAAK,IAAI;GAC7D,OAAO;IACL,MAAM,WAAW,OAAO,EAAE;IAC1B,IAAI;IACJ,IAAI;KACF,SAAS,mBAAmB,KAAKA,KAAK,IAAI,UAAU,EAClD,+BAA+B,KACjC,CAAC;IACH,SAAS,GAAG;KACV,IAAI,EAAE,aAAa,2BACjB,MAAM;KAIR,KAAKQ,UAAU,KAAA;KACf,KAAKJ,MAAM,KAAA;KACX,GAAG,MAAM;KACT,KAAKH,MAAM,YAAY;KACvB,MAAM,UAAU,6BACd,KAAKD,KACL,aACA,UACA,CACF;KACA,KAAKI,MAAM,QAAQ;KACnB,SAAS,QAAQ;IACnB;IACA,+BAA+B,KAAKJ,KAAK,MAAM;IAC/C,IAAI,OAAO,WAAW,QACpB,KAAKK,WAAW,WACd,uBACE,KAAK,KAAKD,KAAK,mCAAmC,CACpD,CACF;GAEJ;GAGA,MAAM,eAAe,KAAK,KAAKA,KAAK,mCAAmC;GACvE,KAAKI,UAAU,IAAI,sBACjB,IAAI,gBAAgB,YAAY,CAClC;GAGA,OAAO;IACL,iBAAiB,KACf,kBAAkB,YAAY,GAC9B,wDACF;IACA,SAAS,YAAY,YAAY;GACnC;EACF,SAAS,GAAG;GACV,KAAKC,UAAU,qCAAqC,CAAC;GACrD;EACF;CACF;;;;;;;CAQA,eACE,IACA,MACsB;EACtB,IAAI,oBAAoB,IAAI,KAAKR,MAAM,QAAQ,MAAM,KAAA,GACnD,OAAO,KAAK;EAEd,MAAM,OAAO,kBAAkB,EAAE;EAEjC,OAAO,SAAS,OACZ,KAAK,IACL;GAAC,iBAAiB;GAAM,SAAS,YAAY,EAAE;EAAC;CACtD;;;;;CAMA,MAAM,QAA0B,MAAoB;EAClD,MAAM,SAAS,KAAKO;EACpB,MAAM,KAAK,KAAKJ;EAChB,IAAI,KAAKD,aAAa,WAAW,KAAA,KAAa,OAAO,KAAA,GACnD;EAEF,MAAM,WAAW,KAAKE;EACtB,UAAU,gBAAgB,QAAQ,IAAI;EACtC,MAAM,CAAC,MAAM,OAAO;EACpB,MAAM,YAAY,YAAY,IAAI;EAClC,IAAI;GACF,IAAI,SAAiC;GACrC,QAAQ,MAAR;IACE,KAAK;KACH,OAAO,MAAM,OAAO,GAAG,iBAAiB,IAAI;KAC5C;IACF,KAAK,UAAU;KACb,MAAM,EAAC,cAAa,OAAO;KAC3B,MAAM,cAAc,YAAY,IAAI;KAEpC,SAAS;MACP;MACA,OAHY,OAAO,OAAO,WAAW,MAAM,KAAKH,KAAK,CAGrD;MACA,aAAa,YAAY,IAAI,IAAI;KACnC;KACA;IACF;IACA,KAAK;KACH,OAAO,SAAS;KAChB;IACF;KACE,OAAO,OAAO,MAAM,GAAG;KACvB;GACJ;GACA,UAAU,iBAAiB,QAAQ,QAAQ,YAAY,IAAI,IAAI,SAAS;GACxE,IAAI,WAAW,QAAQ,OAAO,MAAM,gBAAgB,SAAS,GAK3D,UAAU,eACR,OAAO,MAAM,iBACb,oBAAoB,EAAE,CACxB;GAEF,IAAI,WAAW,MAGb,KAAKD,MAAM,WAAW,OAAO,SAAS;EAE1C,SAAS,GAAG;GACV,UAAU,cAAc,QAAQ,GAAG,YAAY,IAAI,IAAI,SAAS;GAChE,IAAI,sBAAsB,CAAC,GAIzB,UAAU,iBACR,0HAEA;IAAC,KAAK,IAAI;IAAK,OAAO,OAAO,CAAC;GAAC,CACjC;GAEF,KAAKQ,UAAU,oCAAoC,CAAC;EACtD;CACF;;;;;CAMA,QAAc;EACZ,MAAM,SAAS,KAAKD;EACpB,IAAI,KAAKL,aAAa,WAAW,KAAA,GAC/B;EAEF,KAAKE,WAAW,MAAM;EACtB,IAAI;GACF,OAAO,SAAS;EAClB,SAAS,GAAG;GACV,KAAKI,UAAU,sCAAsC,CAAC;EACxD;CACF;CAEA,QAAc;EACZ,KAAKN,YAAY;EACjB,KAAKK,UAAU,KAAA;EACf,KAAKH,YAAY,KAAA;EACjB,MAAM,KAAK,KAAKD;EAChB,KAAKA,MAAM,KAAA;EACX,IAAI;GACF,IAAI,MAAM;EACZ,SAAS,GAAG;GACV,KAAKJ,IAAI,OAAO,uCAAuC,CAAC;EAC1D;CACF;;;;;;;CAQA,UAAU,MAAc,GAAkB;EACxC,MAAM,OAAO,kBAAkB,KAAKC,MAAM,WAAW;EACrD,IAAI,mBAAmB,CAAC,GAItB,+BAA+B,KAAKD,KAAK,cAAc,MAAM,CAAC;EAEhE,KAAKA,IAAI,QACP,SAAS,KAAK,sCAAsC,KAAK,2EAEzD,CACF;EACA,KAAK,MAAM;EACX,IAAI;GACF,kBAAkB,KAAKC,MAAM,WAAW;EAC1C,SAAS,aAAa;GACpB,KAAKD,IAAI,QAAQ,oBAAoB,QAAQ,WAAW;EAC1D;EACA,KAAKC,MAAM,aAAa;CAC1B;AACF;AAEA,SAAS,sBAAsB,GAAqB;CAClD,OAAO,aAAa,eAAe,EAAE,KAAK,WAAW,mBAAmB;AAC1E"}

package/out/zero-cache/src/services/change-streamer/storer.d.ts

@@ -1,17 +1,21 @@
 import type { LogContext } from '@rocicorp/logger';
+import { type JSONObject } from '../../../../shared/src/bigint-json.ts';
 import { TransactionPool } from '../../db/transaction-pool.ts';
 import { type PostgresDB } from '../../types/pg.ts';
 import { type ShardID } from '../../types/shards.ts';
-import { type BackfillRequest, type Change } from '../change-source/protocol/current.ts';
+import { type BackfillRequest } from '../change-source/protocol/current.ts';
 import { type ChangeStreamData, type Commit } from '../change-source/protocol/current/downstream.ts';
-import type { DownstreamStatusMessage, UpstreamStatusMessage } from '../change-source/protocol/current/status.ts';
+import type { UpstreamStatusMessage } from '../change-source/protocol/current/status.ts';
+import { type CookieSet } from '../replicator/change-log-cookies.ts';
 import type { ReplicatorMode } from '../replicator/replicator.ts';
 import type { Service } from '../service.ts';
+import { type ChangeLogEntry } from './change-log-codec.ts';
 import type { Subscriber } from './subscriber.ts';
 export type TuningOptions = {
     backPressureLimitHeapProportion: number;
     statementTimeoutMs: number;
     changeLogBatchSize: number;
+    drainTimeoutMs?: number | undefined;
 };
 /**
  * Handles the storage of changes and the catchup of subscribers
@@ -46,20 +50,59 @@ export type TuningOptions = {
 export declare class Storer implements Service {
     #private;
     readonly id = "storer";
-    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, db: PostgresDB, replicaVersion: string, onConsumed: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, }: TuningOptions);
+    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, db: PostgresDB, replicaVersion: string, onCommitted: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs, }: TuningOptions);
     assumeOwnership(purgeLock?: PurgeLock | null): Promise<void>;
+    /**
+     * The whole of what a stream connection starts from: where to resume, what to
+     * ask the change source to backfill, and the raw cookie set those requests
+     * were derived from.
+     *
+     * All four statements run in one REPEATABLE READ transaction, so they see one
+     * snapshot. That is invariant 15 — the resume watermark and the cookies must
+     * come from the same store, read at the same position — and it is why the
+     * cookies are read here rather than by a second call: a cookie set paired
+     * with a watermark it was not folded to loses every backfill that completed
+     * in the interval, and those rows are not replayable from the slot.
+     *
+     * `cookies` is not `backfillRequests` in another shape. The request list is
+     * driven off `backfilling`, so it drops the metadata of every table with no
+     * in-flight backfill — correct for starting a stream, lossy as the cookie
+     * snapshot that the SQLite change log continues folding onto.
+     */
     getStartStreamInitializationParameters(): Promise<{
         lastWatermark: string;
         backfillRequests: BackfillRequest[];
+        cookies: CookieSet;
     }>;
     getMinWatermarkForCatchup(): Promise<string | null>;
+    /**
+     * Returns the retained Postgres catchup bounds.
+     * `minWatermark` is the oldest transaction, or `null` for a new replica.
+     * `lastWatermark` is the durable head.
+     */
+    getCatchupBounds(): Promise<{
+        minWatermark: string | null;
+        lastWatermark: string;
+    }>;
+    /**
+     * Lists at most `limit` committed transaction watermarks in
+     * `(afterWatermark, throughWatermark]`, in ascending order.
+     */
+    listCommitWatermarks(afterWatermark: string, throughWatermark: string, limit: number): Promise<string[]>;
+    /**
+     * Reads `(afterWatermark, throughWatermark]` in stream order and bounded batches.
+     * This query matches the subscriber catchup query.
+     *
+     * Both queries fail when an escaped NUL reaches `change->'tag'`.
+     * Postgres cannot convert that value to text.
+     */
+    readCatchupRange(afterWatermark: string, throughWatermark: string, batchRows?: number): AsyncIterable<ChangeLogEntry[]>;
     purgeRecordsBefore(watermark: string): Promise<number>;
     /**
      * @returns The JSON stringified stream message to be sent downstream.
      */
     store(watermark: string, data: ChangeStreamData): string;
     abort(): void;
-    status(s: DownstreamStatusMessage): void;
     catchup(subscriber: Subscriber, mode: ReplicatorMode): void;
     readyForMore(): Promise<void> | undefined;
     /**
@@ -72,16 +115,13 @@ export declare class Storer implements Service {
      * This is only used in tests.
      */
     allProcessed(): Promise<void>;
+    /**
+     * Stops the storer and waits up to a drain timeout for entries to drain,
+     * throwing an exception if it doesn't drain in time, in order to abort
+     * the server when PG (or the connection to it) appears to be wedged.
+     */
     stop(): Promise<void>;
 }
-/**
- * Extracts the stringified change message from the stringified
- * stream message (e.g. the second tuple element). This optimization
- * facilitates stringifying (and sharing the result of) the stream
- * message exactly once, but storing only the change message substring
- * in the changeLog for backwards compatibility.
- */
-export declare function extractChangeSubstring(streamMessageJSON: string, tag: Change['tag'] | undefined): string;
 export declare class PurgeLock {
     #private;
     readonly replicaVersion: string;
@@ -94,4 +134,37 @@ export declare class PurgeLocker {
     constructor(lc: LogContext, shard: ShardID, db: PostgresDB);
     acquire(): Promise<PurgeLock | null>;
 }
+type TaskKey = JSONObject;
+type TaskDoneFn = () => void;
+/**
+ * Periodic monitor for detecting the pathological condition in which a db
+ * query hangs forever because of a connection problem, such as a half-closed
+ * socket.
+ *
+ * Use this for db calls that recur or accumulate progress within a single
+ * logical operation on the main storer loop (e.g. the queue-processing loop,
+ * or a multi-batch catchup read), where "no progress for N ms" is the
+ * meaningful signal. For a one-off db call -- including ones made before
+ * {@link Storer.run} starts this monitor's polling -- use a plain timeout
+ * (see `Storer.#withTimeout`) instead.
+ *
+ * The failure handler should stop the server. The ProgressMonitor will
+ * assume this and stop itself after calling it.
+ *
+ * Internal to the Storer, but exported for testing.
+ */
+export declare class ProgressMonitor {
+    #private;
+    constructor(lc: LogContext, failureThresholdMs: number, onFailure: (err: Error) => void);
+    start(): void;
+    checkTaskProgress(now: Date): void;
+    /**
+     * Starts a task to be tracked.
+     *
+     * @return a callback to invoke when the task is done.
+     */
+    trackTask(task: TaskKey, now?: Date): TaskDoneFn;
+    stop(): void;
+}
+export {};
 //# sourceMappingURL=storer.d.ts.map

package/out/zero-cache/src/services/change-streamer/storer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,EAAC,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAKL,KAAK,eAAe,EACpB,KAAK,MAAM,EAKZ,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EACV,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAS3C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAuChD,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAmBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACvD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,GACnB,EAAE,aAAa;IAgCZ,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI;IA2B5C,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;KACrC,CAAC;IAkCI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAOzD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyCtD;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB;IAkB/C,KAAK;IAIL,MAAM,CAAC,CAAC,EAAE,uBAAuB;IAIjC,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAMpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IAyEzC;;;OAGG;IACG,GAAG;IA0eT;;;OAGG;IACG,YAAY;IAQlB,IAAI;CAOL;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,UAmB/B;AA+BD,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAG5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM;IAUb,OAAO;CAWd;AAED,qBAAa,WAAW;;gBAKV,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU;IAWpD,OAAO;CA4Bd"}
+{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,uCAAuC,CAAC;AAM/C,OAAO,EAAC,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,6CAA6C,CAAC;AACvF,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,EAIL,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAQ/B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAsChD,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAqBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACxD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa;IA+DZ,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI;IA8BlD;;;;;;;;;;;;;;;;OAgBG;IACG,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;QACpC,OAAO,EAAE,SAAS,CAAC;KACpB,CAAC;IAwDI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IASzD;;;;OAIG;IACG,gBAAgB,IAAI,OAAO,CAAC;QAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IAWF;;;OAGG;IACG,oBAAoB,CACxB,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,EAAE,CAAC;IAWpB;;;;;;OAMG;IACH,gBAAgB,CACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,SAAS,SAAO,GACf,aAAa,CAAC,cAAc,EAAE,CAAC;IAkBlC,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyCtD;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB;IAkB/C,KAAK;IAIL,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAMpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IA2EzC;;;OAGG;IACG,GAAG;IA8fT;;;OAGG;IACG,YAAY;IAQlB;;;;OAIG;IACG,IAAI;CAeX;AAED,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAG5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM;IAUb,OAAO;CAWd;AAED,qBAAa,WAAW;;gBAKV,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU;IAWpD,OAAO;CA4Bd;AAKD,KAAK,OAAO,GAAG,UAAU,CAAC;AAE1B,KAAK,UAAU,GAAG,MAAM,IAAI,CAAC;AAE7B;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,eAAe;;gBASxB,EAAE,EAAE,UAAU,EACd,kBAAkB,EAAE,MAAM,EAC1B,SAAS,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI;IAOjC,KAAK;IASL,iBAAiB,CAAC,GAAG,EAAE,IAAI;IAkB3B;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,OAAa,GAAG,UAAU;IAOtD,IAAI;CAIL"}