@rocicorp/zero 1.9.1-canary.1 → 1.10.0-canary.1

package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js

@@ -0,0 +1,413 @@
+import { unreachable } from "../../../../shared/src/asserts.js";
+import { getOrCreateCounter, getOrCreateGauge, getOrCreateLatencyHistogram, getOrCreateValueHistogram } from "../../observability/metrics.js";
+import { readCookieRowCounts } from "./change-log-cookies.js";
+import { versionFromLexi } from "../../types/lexi-version.js";
+import { CHANGE_LOG_STREAM_TABLE, readChangeLogHead, readChangeLogMeta } from "./change-log-db.js";
+import { extractChangeSubstring } from "../change-streamer/change-log-codec.js";
+import { ChangeLogTransactionHasher } from "../change-streamer/change-log-transaction-hash.js";
+import { stat } from "node:fs/promises";
+//#region ../zero-cache/src/services/replicator/sqlite-change-log-observability.ts
+/**
+* Observability for the SQLite change log.
+*
+* Everything here now runs in the **change-streamer** process, which is where
+* the writer lives. The module stays beside the rest of the change-log code
+* under `services/replicator/` for the same reason `change-log-db.ts` and
+* `change-log-stream-writer.ts` do: moving files would obscure the topology
+* diff, and the writer's home is a property of who calls it rather than of
+* where it sits. Metric names are unchanged from 7H — only their emitting
+* process, and therefore their resource attributes, moved.
+*/
+/**
+* Reports the reconciliation of the change-log database against the watermark
+* the stream connection resumes from. This runs once per stream connection, so
+* a reconnect storm is visible here as well as a restart.
+*
+* A wipe is not an error — it is how every abnormal change-log state is
+* collapsed into a correct one — but each wipe costs the retention window, so
+* every reconnecting subscriber below the head gets `too-old`. The `gap` reason
+* in particular means the log neither held the resume watermark nor could be
+* truncated to it, so a nonzero rate in steady state is an alert, not a
+* statistic.
+*/
+function recordSQLiteChangeLogReconcile(lc, result) {
+	switch (result.action) {
+		case "truncated":
+			getOrCreateCounter("replica", "sqlite_change_log.reconcile_truncated_rows", "Phantom change-log rows removed at reconciliation, i.e. rows above the watermark the change stream resumed from.").add(result.rows);
+			break;
+		case "reseeded":
+			getOrCreateCounter("replica", "sqlite_change_log.reconcile_wipes", "Change-log wipes at reconciliation, by reason. Each one resets the retention window, so subscribers below the head get `too-old`.").add(1, { reason: result.reason });
+			break;
+		case "none": break;
+		default: unreachable(result);
+	}
+	if (result.cookiesStale) getOrCreateCounter("replica", "sqlite_change_log.cookie_reseed", "Change-log cookie sets invalidated at reconciliation, by the reconcile action that invalidated them.").add(1, { action: result.action });
+	lc.debug?.("SQLite change-log reconciliation", { sqliteChangeLogReconcile: result });
+}
+/**
+* Reads the meta row and the head. The head is an index-optimized `max()`, so
+* unlike {@link getSQLiteChangeLogInfo} this does not scan the log.
+*
+* Call this after reconciliation, which is the only point at which the head is
+* known to be the resume watermark.
+*/
+function getSQLiteChangeLogStartupInfo(changeLog) {
+	const meta = readChangeLogMeta(changeLog);
+	const head = readChangeLogHead(changeLog);
+	if (head === null) throw new Error("the SQLite change log must contain its seed transaction");
+	return {
+		...meta,
+		headWatermark: head
+	};
+}
+/**
+* Extends {@link getSQLiteChangeLogStartupInfo} with the retained row and
+* estimated byte totals. Computing these scans the entire change-log table, so
+* only call it when the totals are consumed, i.e. when the writer (and thus the
+* observer) is enabled.
+*/
+function getSQLiteChangeLogInfo(changeLog) {
+	const startup = getSQLiteChangeLogStartupInfo(changeLog);
+	const aggregate = changeLog.prepare(`
+      SELECT count(*) AS "rows",
+             coalesce(sum("estimatedBytes"), 0) AS "estimatedBytes"
+        FROM "${CHANGE_LOG_STREAM_TABLE}"
+    `).get();
+	return {
+		...startup,
+		rows: aggregate.rows,
+		estimatedBytes: aggregate.estimatedBytes,
+		cookieRows: readCookieRowCounts(changeLog)
+	};
+}
+function logSQLiteChangeLogStartup(lc, file, info) {
+	lc.info?.("SQLite change-log startup", { sqliteChangeLog: {
+		file,
+		epoch: info.epoch,
+		generation: info.generation,
+		replicaID: info.replicaID,
+		schemaVersion: info.schemaVersion,
+		seedWatermark: info.seedWatermark,
+		seededAtMs: info.seededAtMs,
+		headWatermark: info.headWatermark
+	} });
+}
+/**
+* The on-disk footprint of a SQLite database: its main file plus its `-wal` and
+* `-wal2` sidecars, whichever exist. `-shm` is excluded — it is a
+* shared-memory index rather than retained bytes.
+*
+* A missing file counts as zero rather than warning. Which sidecars exist
+* depends on the journal mode, and the change-log database is absent entirely
+* until the writer creates it.
+*/
+async function sqliteFileBytes(lc, file) {
+	return (await Promise.all([
+		file,
+		`${file}-wal`,
+		`${file}-wal2`
+	].map(async (path) => {
+		try {
+			return (await stat(path)).size;
+		} catch (e) {
+			if (e.code !== "ENOENT") lc.warn?.(`unable to stat ${path} for size metrics`, e);
+			return 0;
+		}
+	}))).reduce((total, size) => total + size, 0);
+}
+var TRANSACTION_ROW_BUCKETS = [
+	2,
+	3,
+	5,
+	10,
+	25,
+	50,
+	100,
+	250,
+	1e3,
+	5e3
+];
+/**
+* Records writer metrics in the change-streamer process. State changes only
+* after the corresponding write succeeds, except receivedHead, which advances
+* when a commit message arrives so that transient lag behind the log is
+* observable.
+*/
+var SQLiteChangeLogObserver = class {
+	#lc;
+	#messageProcessing = getOrCreateLatencyHistogram("replica", "sqlite_change_log.message_processing_duration", "Time to write a replication message to the SQLite change log.");
+	#logCommitDuration = getOrCreateLatencyHistogram("replica", "sqlite_change_log.log_commit_duration", "Time to append the commit row to the SQLite change log and commit its transaction. This is on the forward path: it precedes the forward of that transaction's commit message.");
+	#transactionRows = getOrCreateValueHistogram("replica", "sqlite_change_log.transaction_rows", {
+		description: "Rows stored per committed SQLite change-log transaction.",
+		unit: "{row}",
+		bucketBoundaries: TRANSACTION_ROW_BUCKETS
+	});
+	#rollbackCounter = getOrCreateCounter("replica", "sqlite_change_log.rollbacks", "SQLite change-log transactions rolled back before commit.");
+	#invariantFailureCounter = getOrCreateCounter("replica", "sqlite_change_log.invariant_failures", "Detected SQLite change-log writer invariant failures.");
+	#hashComparisonCounter = getOrCreateCounter("replica", "sqlite_change_log.hash_comparisons", "Ephemeral transaction hash comparisons between the received change stream and SQLite change-log writes.");
+	#hashUnpairedCounter = getOrCreateCounter("replica", "sqlite_change_log.hash_unpaired", "Committed change-log transactions without both ephemeral hashes available for comparison.");
+	#cookieMutationCounter = getOrCreateCounter("replica", "sqlite_change_log.cookie_mutations", "Change-log cookie mutations folded from schema changes, by operation.");
+	#receivedHead;
+	#sqliteHead;
+	#rows;
+	#estimatedBytes;
+	#cookieRows;
+	#cookieMutations = 0;
+	#transactionWatermark;
+	#sourceHasher;
+	#sourcePos = 0;
+	#sourceCommitHash;
+	#rollbacks = 0;
+	#invariantFailures = 0;
+	#hashMatches = 0;
+	#hashMismatches = 0;
+	#hashUnpaired = 0;
+	constructor(lc, info) {
+		this.#lc = lc.withContext("component", "sqlite-change-log-observer");
+		this.#receivedHead = info.headWatermark;
+		this.#sqliteHead = info.headWatermark;
+		this.#rows = info.rows;
+		this.#estimatedBytes = info.estimatedBytes;
+		this.#cookieRows = info.cookieRows;
+		getOrCreateGauge("replica", "sqlite_change_log.rows", "Rows retained in the SQLite change log.").addCallback((result) => result.observe(this.#rows));
+		getOrCreateGauge("replica", "sqlite_change_log.cookie_rows", "Cookie rows retained in the SQLite change log, by cookie table.").addCallback((result) => {
+			result.observe(this.#cookieRows.tableMetadata, { table: "tableMetadata" });
+			result.observe(this.#cookieRows.backfilling, { table: "backfilling" });
+		});
+		getOrCreateGauge("replica", "sqlite_change_log.retained_bytes", {
+			description: "Estimated UTF-8 payload bytes retained in the SQLite change log.",
+			unit: "By"
+		}).addCallback((result) => result.observe(this.#estimatedBytes));
+		getOrCreateGauge("replica", "sqlite_change_log.head", "SQLite change-log head converted from its lexicographic watermark.").addCallback((result) => {
+			const head = watermarkValue(this.#sqliteHead);
+			if (head !== void 0) result.observe(head);
+		});
+		getOrCreateGauge("replica", "sqlite_change_log.head_lag", "Distance from the latest received commit to the SQLite change-log head. The log commits before the forward, so this is zero except within a transaction.").addCallback((result) => {
+			const lag = watermarkDistance(this.#receivedHead, this.#sqliteHead);
+			if (lag !== void 0) result.observe(lag);
+		});
+	}
+	/**
+	* Called as each message arrives, before it is written. Hashes the *received*
+	* stream, which is compared against the hash the writer computes over what it
+	* *stored*.
+	*/
+	messageReceived(data, json) {
+		const tag = data[1].tag;
+		if (data[0] === "begin") {
+			if (this.#sourceHasher !== void 0) this.#invariantFailure("Received change-log begin while a source hash is already open", { receivedWatermark: data[2].commitWatermark });
+			const watermark = data[2].commitWatermark;
+			this.#sourceHasher = new ChangeLogTransactionHasher();
+			this.#sourcePos = 0;
+			this.#sourceCommitHash = void 0;
+			this.#sourceHasher.add({
+				watermark,
+				pos: this.#sourcePos,
+				tag,
+				change: extractChangeSubstring(json, tag),
+				precommit: null
+			});
+			return;
+		}
+		if (tag === "rollback") {
+			this.#resetSourceHash();
+			return;
+		}
+		if (data[0] === "commit") this.#receivedHead = data[2].watermark;
+		const watermark = this.#transactionWatermark;
+		if (this.#sourceHasher === void 0 || watermark === void 0) {
+			this.#invariantFailure("Received change-log message without an observed transaction", { tag });
+			return;
+		}
+		const commitWatermark = data[0] === "commit" ? data[2].watermark : watermark;
+		this.#sourceHasher.add({
+			watermark: commitWatermark,
+			pos: ++this.#sourcePos,
+			tag,
+			change: extractChangeSubstring(json, tag),
+			precommit: data[0] === "commit" ? watermark : null
+		});
+		if (data[0] === "commit") {
+			this.#sourceCommitHash = this.#sourceHasher.digest();
+			this.#sourceHasher = void 0;
+		}
+	}
+	/** Called after the message has been written to the log. */
+	messageProcessed(data, commit, durationMs) {
+		const tag = data[1].tag;
+		this.#messageProcessing.recordMs(durationMs, {
+			tag,
+			outcome: "success"
+		});
+		if (data[0] === "begin") {
+			if (this.#transactionWatermark !== void 0) this.#invariantFailure("SQLite change-log transaction already open", {
+				openWatermark: this.#transactionWatermark,
+				receivedWatermark: data[2].commitWatermark
+			});
+			this.#transactionWatermark = data[2].commitWatermark;
+			return;
+		}
+		if (tag === "rollback") {
+			this.#recordRollback("upstream");
+			this.#transactionWatermark = void 0;
+			this.#resetSourceHash();
+			return;
+		}
+		if (data[0] !== "commit") return;
+		const watermark = data[2].watermark;
+		if (commit !== null) this.#logCommitDuration.recordMs(commit.logCommitMs);
+		if (this.#transactionWatermark !== watermark) this.#invariantFailure("SQLite change-log commit does not match the observed begin", {
+			openWatermark: this.#transactionWatermark,
+			commitWatermark: watermark
+		});
+		if (commit === null) {
+			this.#invariantFailure("SQLite change-log commit result is missing writer statistics", { commitWatermark: watermark });
+			this.#recordUnpaired(watermark, "missing-sqlite-hash");
+		} else {
+			if (commit.watermark !== watermark) this.#invariantFailure("SQLite change-log commit result has an unexpected watermark", {
+				commitWatermark: watermark,
+				resultWatermark: commit.watermark
+			});
+			this.#rows += commit.stats.rows;
+			this.#estimatedBytes += commit.stats.estimatedBytes;
+			this.#transactionRows.record(commit.stats.rows);
+			this.#compareHashes(watermark, this.#sourceCommitHash, commit.stats.hash);
+		}
+		if (watermark < this.#sqliteHead) this.#invariantFailure("SQLite change-log head regressed", {
+			previousHead: this.#sqliteHead,
+			commitWatermark: watermark
+		});
+		this.#sqliteHead = watermark;
+		this.#transactionWatermark = void 0;
+		this.#resetSourceHash();
+	}
+	messageFailed(data, error, durationMs) {
+		const tag = data[1].tag;
+		this.#messageProcessing.recordMs(durationMs, {
+			tag,
+			outcome: "error"
+		});
+		if (data[0] === "commit") this.#recordUnpaired(data[2].watermark, "sqlite-commit-failed");
+		if (error instanceof Error && error.name === "ChangeLogStreamInvariantError") this.#invariantFailure(error.message);
+		if (this.#transactionWatermark !== void 0 || data[0] === "begin") this.#recordRollback("processing-error");
+		this.#transactionWatermark = void 0;
+		this.#resetSourceHash();
+	}
+	/**
+	* A committed transaction folded schema changes into the cookie jar. Reported
+	* separately from {@link messageProcessed} because it costs a read of the
+	* cookie tables, which is only worth taking when they actually moved.
+	*/
+	cookiesMutated(ops, rows) {
+		for (const op of ops) {
+			this.#cookieMutations++;
+			this.#cookieMutationCounter.add(1, { op });
+		}
+		this.#cookieRows = rows;
+	}
+	/**
+	* A missed truncate-above surfaces as a constraint violation on the writer's
+	* plain `INSERT`, which the fail-soft policy would otherwise absorb quietly.
+	* Classifying it as an invariant failure is what keeps a reconciliation bug
+	* from looking like a lost cache.
+	*/
+	invariantFailure(message, details) {
+		this.#invariantFailure(message, details);
+	}
+	/** The source interrupted the stream mid-transaction. */
+	abort() {
+		if (this.#transactionWatermark !== void 0) {
+			this.#recordRollback("source-interruption");
+			this.#transactionWatermark = void 0;
+		}
+		this.#resetSourceHash();
+	}
+	/** Reconciliation moved the head, so the observer's totals are restated. */
+	reconciled(info) {
+		this.#receivedHead = info.headWatermark;
+		this.#sqliteHead = info.headWatermark;
+		this.#rows = info.rows;
+		this.#estimatedBytes = info.estimatedBytes;
+		this.#cookieRows = info.cookieRows;
+		this.#transactionWatermark = void 0;
+		this.#resetSourceHash();
+	}
+	state() {
+		return {
+			receivedHead: this.#receivedHead,
+			sqliteHead: this.#sqliteHead,
+			headLag: watermarkDistance(this.#receivedHead, this.#sqliteHead),
+			rows: this.#rows,
+			estimatedBytes: this.#estimatedBytes,
+			cookieRows: this.#cookieRows,
+			cookieMutations: this.#cookieMutations,
+			rollbacks: this.#rollbacks,
+			invariantFailures: this.#invariantFailures,
+			hashMatches: this.#hashMatches,
+			hashMismatches: this.#hashMismatches,
+			hashUnpaired: this.#hashUnpaired
+		};
+	}
+	#recordRollback(reason) {
+		this.#rollbacks++;
+		this.#rollbackCounter.add(1, { reason });
+	}
+	#invariantFailure(message, details) {
+		this.#invariantFailures++;
+		this.#invariantFailureCounter.add(1);
+		this.#lc.error?.(message, details);
+	}
+	#compareHashes(watermark, sourceHash, sqliteHash) {
+		if (sourceHash === void 0) {
+			this.#recordUnpaired(watermark, "missing-source-hash");
+			return;
+		}
+		if (sourceHash === sqliteHash) {
+			this.#hashMatches++;
+			this.#hashComparisonCounter.add(1, { outcome: "match" });
+			return;
+		}
+		this.#hashMismatches++;
+		this.#hashComparisonCounter.add(1, { outcome: "mismatch" });
+		this.#lc.error?.("SQLite change-log transaction hash mismatch", { sqliteChangeLogHashComparison: {
+			watermark,
+			sourceHash: sourceHash.slice(0, 16),
+			sqliteHash: sqliteHash.slice(0, 16)
+		} });
+	}
+	#recordUnpaired(watermark, reason) {
+		this.#hashUnpaired++;
+		this.#hashUnpairedCounter.add(1, { reason });
+		this.#lc.warn?.("SQLite change-log transaction hash was not compared", { sqliteChangeLogHashComparison: {
+			watermark,
+			reason
+		} });
+	}
+	#resetSourceHash() {
+		this.#sourceHasher = void 0;
+		this.#sourcePos = 0;
+		this.#sourceCommitHash = void 0;
+	}
+};
+function watermarkValue(watermark) {
+	try {
+		const value = Number(versionFromLexi(watermark));
+		return Number.isFinite(value) ? value : void 0;
+	} catch {
+		return;
+	}
+}
+function watermarkDistance(receivedHead, sqliteHead) {
+	if (receivedHead <= sqliteHead) return 0;
+	try {
+		const distance = versionFromLexi(receivedHead) - versionFromLexi(sqliteHead);
+		const value = Number(distance);
+		return Number.isFinite(value) ? value : void 0;
+	} catch {
+		return;
+	}
+}
+//#endregion
+export { SQLiteChangeLogObserver, getSQLiteChangeLogInfo, logSQLiteChangeLogStartup, recordSQLiteChangeLogReconcile, sqliteFileBytes };
+
+//# sourceMappingURL=sqlite-change-log-observability.js.map

package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlite-change-log-observability.js","names":["#lc","#messageProcessing","#logCommitDuration","#transactionRows","#rollbackCounter","#invariantFailureCounter","#hashComparisonCounter","#hashUnpairedCounter","#cookieMutationCounter","#receivedHead","#sqliteHead","#rows","#estimatedBytes","#cookieRows","#sourceHasher","#invariantFailure","#sourcePos","#sourceCommitHash","#resetSourceHash","#transactionWatermark","#recordRollback","#recordUnpaired","#compareHashes","#cookieMutations","#rollbacks","#invariantFailures","#hashMatches","#hashMismatches","#hashUnpaired"],"sources":["../../../../../../zero-cache/src/services/replicator/sqlite-change-log-observability.ts"],"sourcesContent":["/**\n * Observability for the SQLite change log.\n *\n * Everything here now runs in the **change-streamer** process, which is where\n * the writer lives. The module stays beside the rest of the change-log code\n * under `services/replicator/` for the same reason `change-log-db.ts` and\n * `change-log-stream-writer.ts` do: moving files would obscure the topology\n * diff, and the writer's home is a property of who calls it rather than of\n * where it sits. Metric names are unchanged from 7H — only their emitting\n * process, and therefore their resource attributes, moved.\n */\n\nimport {stat} from 'node:fs/promises';\nimport type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n  getOrCreateValueHistogram,\n} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {extractChangeSubstring} from '../change-streamer/change-log-codec.ts';\nimport {ChangeLogTransactionHasher} from '../change-streamer/change-log-transaction-hash.ts';\nimport {\n  readCookieRowCounts,\n  type CookieOpTag,\n  type CookieRowCounts,\n} from './change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  readChangeLogHead,\n  readChangeLogMeta,\n  type ChangeLogMeta,\n  type ReconcileResult,\n} from './change-log-db.ts';\nimport type {ChangeLogStreamTransactionStats} from './change-log-stream-writer.ts';\n\n/**\n * Reports the reconciliation of the change-log database against the watermark\n * the stream connection resumes from. This runs once per stream connection, so\n * a reconnect storm is visible here as well as a restart.\n *\n * A wipe is not an error — it is how every abnormal change-log state is\n * collapsed into a correct one — but each wipe costs the retention window, so\n * every reconnecting subscriber below the head gets `too-old`. The `gap` reason\n * in particular means the log neither held the resume watermark nor could be\n * truncated to it, so a nonzero rate in steady state is an alert, not a\n * statistic.\n */\nexport function recordSQLiteChangeLogReconcile(\n  lc: LogContext,\n  result: ReconcileResult,\n): void {\n  switch (result.action) {\n    case 'truncated':\n      getOrCreateCounter(\n        'replica',\n        'sqlite_change_log.reconcile_truncated_rows',\n        'Phantom change-log rows removed at reconciliation, i.e. rows above ' +\n          'the watermark the change stream resumed from.',\n      ).add(result.rows);\n      break;\n    case 'reseeded':\n      getOrCreateCounter(\n        'replica',\n        'sqlite_change_log.reconcile_wipes',\n        'Change-log wipes at reconciliation, by reason. Each one resets the ' +\n          'retention window, so subscribers below the head get `too-old`.',\n      ).add(1, {reason: result.reason});\n      break;\n    case 'none':\n      break;\n    default:\n      unreachable(result);\n  }\n  if (result.cookiesStale) {\n    // The invariant-17 counter: after any reconcile that moved the head, the\n    // log's cookie set has to be replaced with the one that belongs to the\n    // resume watermark. A nonzero `truncated` series in steady state means\n    // phantom truncation is happening on a shard where it should not.\n    getOrCreateCounter(\n      'replica',\n      'sqlite_change_log.cookie_reseed',\n      'Change-log cookie sets invalidated at reconciliation, by the ' +\n        'reconcile action that invalidated them.',\n    ).add(1, {action: result.action});\n  }\n  lc.debug?.('SQLite change-log reconciliation', {\n    sqliteChangeLogReconcile: result,\n  });\n}\n\n/**\n * What the log says about itself. Nothing here reads the replica: after the\n * writer moved to the change-streamer, the replica's state version is a\n * consequence of this log rather than a fact about it.\n */\nexport type SQLiteChangeLogStartupInfo = ChangeLogMeta & {\n  readonly headWatermark: string;\n};\n\nexport type SQLiteChangeLogInfo = SQLiteChangeLogStartupInfo & {\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly cookieRows: CookieRowCounts;\n};\n\n/**\n * Reads the meta row and the head. The head is an index-optimized `max()`, so\n * unlike {@link getSQLiteChangeLogInfo} this does not scan the log.\n *\n * Call this after reconciliation, which is the only point at which the head is\n * known to be the resume watermark.\n */\nexport function getSQLiteChangeLogStartupInfo(\n  changeLog: Database,\n): SQLiteChangeLogStartupInfo {\n  const meta = readChangeLogMeta(changeLog);\n  const head = readChangeLogHead(changeLog);\n  if (head === null) {\n    throw new Error('the SQLite change log must contain its seed transaction');\n  }\n  return {...meta, headWatermark: head};\n}\n\n/**\n * Extends {@link getSQLiteChangeLogStartupInfo} with the retained row and\n * estimated byte totals. Computing these scans the entire change-log table, so\n * only call it when the totals are consumed, i.e. when the writer (and thus the\n * observer) is enabled.\n */\nexport function getSQLiteChangeLogInfo(\n  changeLog: Database,\n): SQLiteChangeLogInfo {\n  const startup = getSQLiteChangeLogStartupInfo(changeLog);\n  const aggregate = changeLog\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\",\n             coalesce(sum(\"estimatedBytes\"), 0) AS \"estimatedBytes\"\n        FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n    `)\n    .get<{rows: number; estimatedBytes: number}>();\n\n  return {\n    ...startup,\n    rows: aggregate.rows,\n    estimatedBytes: aggregate.estimatedBytes,\n    cookieRows: readCookieRowCounts(changeLog),\n  };\n}\n\nexport function logSQLiteChangeLogStartup(\n  lc: LogContext,\n  file: string,\n  info: SQLiteChangeLogStartupInfo,\n): void {\n  lc.info?.('SQLite change-log startup', {\n    sqliteChangeLog: {\n      file,\n      epoch: info.epoch,\n      generation: info.generation,\n      replicaID: info.replicaID,\n      schemaVersion: info.schemaVersion,\n      seedWatermark: info.seedWatermark,\n      seededAtMs: info.seededAtMs,\n      headWatermark: info.headWatermark,\n    },\n  });\n}\n\n/**\n * The on-disk footprint of a SQLite database: its main file plus its `-wal` and\n * `-wal2` sidecars, whichever exist. `-shm` is excluded — it is a\n * shared-memory index rather than retained bytes.\n *\n * A missing file counts as zero rather than warning. Which sidecars exist\n * depends on the journal mode, and the change-log database is absent entirely\n * until the writer creates it.\n */\nexport async function sqliteFileBytes(\n  lc: LogContext,\n  file: string,\n): Promise<number> {\n  const sizes = await Promise.all(\n    [file, `${file}-wal`, `${file}-wal2`].map(async path => {\n      try {\n        return (await stat(path)).size;\n      } catch (e) {\n        if ((e as NodeJS.ErrnoException).code !== 'ENOENT') {\n          lc.warn?.(`unable to stat ${path} for size metrics`, e);\n        }\n        return 0;\n      }\n    }),\n  );\n  return sizes.reduce((total, size) => total + size, 0);\n}\n\n/** What the writer reports back for a committed transaction. */\nexport type ChangeLogCommit = {\n  readonly watermark: string;\n  readonly stats: ChangeLogStreamTransactionStats;\n  /**\n   * Appending the commit row and committing the log's transaction. This is the\n   * cost that sits on the forward path, once per upstream transaction.\n   */\n  readonly logCommitMs: number;\n};\n\nexport type SQLiteChangeLogObservabilityState = {\n  readonly receivedHead: string;\n  readonly sqliteHead: string;\n  readonly headLag: number | undefined;\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly cookieRows: CookieRowCounts;\n  readonly cookieMutations: number;\n  readonly rollbacks: number;\n  readonly invariantFailures: number;\n  readonly hashMatches: number;\n  readonly hashMismatches: number;\n  readonly hashUnpaired: number;\n};\n\nconst TRANSACTION_ROW_BUCKETS = [2, 3, 5, 10, 25, 50, 100, 250, 1000, 5000];\n\n/**\n * Records writer metrics in the change-streamer process. State changes only\n * after the corresponding write succeeds, except receivedHead, which advances\n * when a commit message arrives so that transient lag behind the log is\n * observable.\n */\nexport class SQLiteChangeLogObserver {\n  readonly #lc: LogContext;\n  readonly #messageProcessing = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.message_processing_duration',\n    'Time to write a replication message to the SQLite change log.',\n  );\n  // The commit that sits on the forward path. There is no second commit to\n  // order against any more: the replica's belongs to a subscriber downstream of\n  // this loop, so `sqlite_change_log.replica_commit_duration` and the phantom\n  // window it measured are gone. A failed commit is still timed, by\n  // `message_processing_duration` keyed `tag=\"commit\"`.\n  readonly #logCommitDuration = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.log_commit_duration',\n    'Time to append the commit row to the SQLite change log and commit its ' +\n      'transaction. This is on the forward path: it precedes the forward of ' +\n      \"that transaction's commit message.\",\n  );\n  readonly #transactionRows = getOrCreateValueHistogram(\n    'replica',\n    'sqlite_change_log.transaction_rows',\n    {\n      description: 'Rows stored per committed SQLite change-log transaction.',\n      unit: '{row}',\n      bucketBoundaries: TRANSACTION_ROW_BUCKETS,\n    },\n  );\n  readonly #rollbackCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.rollbacks',\n    'SQLite change-log transactions rolled back before commit.',\n  );\n  readonly #invariantFailureCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.invariant_failures',\n    'Detected SQLite change-log writer invariant failures.',\n  );\n  readonly #hashComparisonCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.hash_comparisons',\n    'Ephemeral transaction hash comparisons between the received change stream and SQLite change-log writes.',\n  );\n  readonly #hashUnpairedCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.hash_unpaired',\n    'Committed change-log transactions without both ephemeral hashes available for comparison.',\n  );\n  // Near zero at steady state: cookie state only moves on schema changes.\n  readonly #cookieMutationCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.cookie_mutations',\n    'Change-log cookie mutations folded from schema changes, by operation.',\n  );\n\n  #receivedHead: string;\n  #sqliteHead: string;\n  #rows: number;\n  #estimatedBytes: number;\n  #cookieRows: CookieRowCounts;\n  #cookieMutations = 0;\n  #transactionWatermark: string | undefined;\n  #sourceHasher: ChangeLogTransactionHasher | undefined;\n  #sourcePos = 0;\n  #sourceCommitHash: string | undefined;\n  #rollbacks = 0;\n  #invariantFailures = 0;\n  #hashMatches = 0;\n  #hashMismatches = 0;\n  #hashUnpaired = 0;\n\n  constructor(lc: LogContext, info: SQLiteChangeLogInfo) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-observer');\n    // Reconciliation has just put the head at the resume watermark, which is\n    // also the point the stream is about to be received from.\n    this.#receivedHead = info.headWatermark;\n    this.#sqliteHead = info.headWatermark;\n    this.#rows = info.rows;\n    this.#estimatedBytes = info.estimatedBytes;\n    this.#cookieRows = info.cookieRows;\n\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.rows',\n      'Rows retained in the SQLite change log.',\n    ).addCallback(result => result.observe(this.#rows));\n    // Nonzero with no backfill in flight is a leaked cookie, i.e. a backfill\n    // that will be re-requested forever.\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.cookie_rows',\n      'Cookie rows retained in the SQLite change log, by cookie table.',\n    ).addCallback(result => {\n      result.observe(this.#cookieRows.tableMetadata, {table: 'tableMetadata'});\n      result.observe(this.#cookieRows.backfilling, {table: 'backfilling'});\n    });\n    getOrCreateGauge('replica', 'sqlite_change_log.retained_bytes', {\n      description:\n        'Estimated UTF-8 payload bytes retained in the SQLite change log.',\n      unit: 'By',\n    }).addCallback(result => result.observe(this.#estimatedBytes));\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.head',\n      'SQLite change-log head converted from its lexicographic watermark.',\n    ).addCallback(result => {\n      const head = watermarkValue(this.#sqliteHead);\n      if (head !== undefined) {\n        result.observe(head);\n      }\n    });\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.head_lag',\n      'Distance from the latest received commit to the SQLite change-log ' +\n        'head. The log commits before the forward, so this is zero except ' +\n        'within a transaction.',\n    ).addCallback(result => {\n      const lag = watermarkDistance(this.#receivedHead, this.#sqliteHead);\n      if (lag !== undefined) {\n        result.observe(lag);\n      }\n    });\n  }\n\n  /**\n   * Called as each message arrives, before it is written. Hashes the *received*\n   * stream, which is compared against the hash the writer computes over what it\n   * *stored*.\n   */\n  messageReceived(data: ChangeStreamData, json: string): void {\n    const tag = data[1].tag;\n    if (data[0] === 'begin') {\n      if (this.#sourceHasher !== undefined) {\n        this.#invariantFailure(\n          'Received change-log begin while a source hash is already open',\n          {receivedWatermark: data[2].commitWatermark},\n        );\n      }\n      const watermark = data[2].commitWatermark;\n      this.#sourceHasher = new ChangeLogTransactionHasher();\n      this.#sourcePos = 0;\n      this.#sourceCommitHash = undefined;\n      this.#sourceHasher.add({\n        watermark,\n        pos: this.#sourcePos,\n        tag,\n        change: extractChangeSubstring(json, tag),\n        precommit: null,\n      });\n      return;\n    }\n\n    if (tag === 'rollback') {\n      this.#resetSourceHash();\n      return;\n    }\n\n    if (data[0] === 'commit') {\n      this.#receivedHead = data[2].watermark;\n    }\n\n    const watermark = this.#transactionWatermark;\n    if (this.#sourceHasher === undefined || watermark === undefined) {\n      this.#invariantFailure(\n        'Received change-log message without an observed transaction',\n        {tag},\n      );\n      return;\n    }\n\n    const commitWatermark =\n      data[0] === 'commit' ? data[2].watermark : watermark;\n    this.#sourceHasher.add({\n      watermark: commitWatermark,\n      pos: ++this.#sourcePos,\n      tag,\n      change: extractChangeSubstring(json, tag),\n      precommit: data[0] === 'commit' ? watermark : null,\n    });\n    if (data[0] === 'commit') {\n      this.#sourceCommitHash = this.#sourceHasher.digest();\n      this.#sourceHasher = undefined;\n    }\n  }\n\n  /** Called after the message has been written to the log. */\n  messageProcessed(\n    data: ChangeStreamData,\n    commit: ChangeLogCommit | null,\n    durationMs: number,\n  ): void {\n    const tag = data[1].tag;\n    this.#messageProcessing.recordMs(durationMs, {tag, outcome: 'success'});\n\n    if (data[0] === 'begin') {\n      if (this.#transactionWatermark !== undefined) {\n        this.#invariantFailure('SQLite change-log transaction already open', {\n          openWatermark: this.#transactionWatermark,\n          receivedWatermark: data[2].commitWatermark,\n        });\n      }\n      this.#transactionWatermark = data[2].commitWatermark;\n      return;\n    }\n\n    if (tag === 'rollback') {\n      this.#recordRollback('upstream');\n      this.#transactionWatermark = undefined;\n      this.#resetSourceHash();\n      return;\n    }\n\n    if (data[0] !== 'commit') {\n      return;\n    }\n\n    const watermark = data[2].watermark;\n    if (commit !== null) {\n      this.#logCommitDuration.recordMs(commit.logCommitMs);\n    }\n    if (this.#transactionWatermark !== watermark) {\n      this.#invariantFailure(\n        'SQLite change-log commit does not match the observed begin',\n        {openWatermark: this.#transactionWatermark, commitWatermark: watermark},\n      );\n    }\n    if (commit === null) {\n      this.#invariantFailure(\n        'SQLite change-log commit result is missing writer statistics',\n        {commitWatermark: watermark},\n      );\n      this.#recordUnpaired(watermark, 'missing-sqlite-hash');\n    } else {\n      if (commit.watermark !== watermark) {\n        this.#invariantFailure(\n          'SQLite change-log commit result has an unexpected watermark',\n          {commitWatermark: watermark, resultWatermark: commit.watermark},\n        );\n      }\n      this.#rows += commit.stats.rows;\n      this.#estimatedBytes += commit.stats.estimatedBytes;\n      this.#transactionRows.record(commit.stats.rows);\n      this.#compareHashes(watermark, this.#sourceCommitHash, commit.stats.hash);\n    }\n    if (watermark < this.#sqliteHead) {\n      this.#invariantFailure('SQLite change-log head regressed', {\n        previousHead: this.#sqliteHead,\n        commitWatermark: watermark,\n      });\n    }\n    this.#sqliteHead = watermark;\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  messageFailed(\n    data: ChangeStreamData,\n    error: unknown,\n    durationMs: number,\n  ): void {\n    const tag = data[1].tag;\n    this.#messageProcessing.recordMs(durationMs, {tag, outcome: 'error'});\n    if (data[0] === 'commit') {\n      // No commit duration here: a failed commit produces no statistics, so\n      // there is nothing to attribute. The attempt is timed by\n      // `message_processing_duration{tag=\"commit\",outcome=\"error\"}` above.\n      this.#recordUnpaired(data[2].watermark, 'sqlite-commit-failed');\n    }\n    if (\n      error instanceof Error &&\n      error.name === 'ChangeLogStreamInvariantError'\n    ) {\n      this.#invariantFailure(error.message);\n    }\n    if (this.#transactionWatermark !== undefined || data[0] === 'begin') {\n      this.#recordRollback('processing-error');\n    }\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  /**\n   * A committed transaction folded schema changes into the cookie jar. Reported\n   * separately from {@link messageProcessed} because it costs a read of the\n   * cookie tables, which is only worth taking when they actually moved.\n   */\n  cookiesMutated(ops: readonly CookieOpTag[], rows: CookieRowCounts): void {\n    for (const op of ops) {\n      this.#cookieMutations++;\n      this.#cookieMutationCounter.add(1, {op});\n    }\n    this.#cookieRows = rows;\n  }\n\n  /**\n   * A missed truncate-above surfaces as a constraint violation on the writer's\n   * plain `INSERT`, which the fail-soft policy would otherwise absorb quietly.\n   * Classifying it as an invariant failure is what keeps a reconciliation bug\n   * from looking like a lost cache.\n   */\n  invariantFailure(message: string, details?: Record<string, unknown>): void {\n    this.#invariantFailure(message, details);\n  }\n\n  /** The source interrupted the stream mid-transaction. */\n  abort(): void {\n    if (this.#transactionWatermark !== undefined) {\n      this.#recordRollback('source-interruption');\n      this.#transactionWatermark = undefined;\n    }\n    this.#resetSourceHash();\n  }\n\n  /** Reconciliation moved the head, so the observer's totals are restated. */\n  reconciled(info: SQLiteChangeLogInfo): void {\n    this.#receivedHead = info.headWatermark;\n    this.#sqliteHead = info.headWatermark;\n    this.#rows = info.rows;\n    this.#estimatedBytes = info.estimatedBytes;\n    this.#cookieRows = info.cookieRows;\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  state(): SQLiteChangeLogObservabilityState {\n    return {\n      receivedHead: this.#receivedHead,\n      sqliteHead: this.#sqliteHead,\n      headLag: watermarkDistance(this.#receivedHead, this.#sqliteHead),\n      rows: this.#rows,\n      estimatedBytes: this.#estimatedBytes,\n      cookieRows: this.#cookieRows,\n      cookieMutations: this.#cookieMutations,\n      rollbacks: this.#rollbacks,\n      invariantFailures: this.#invariantFailures,\n      hashMatches: this.#hashMatches,\n      hashMismatches: this.#hashMismatches,\n      hashUnpaired: this.#hashUnpaired,\n    };\n  }\n\n  #recordRollback(\n    reason: 'upstream' | 'processing-error' | 'source-interruption',\n  ) {\n    this.#rollbacks++;\n    this.#rollbackCounter.add(1, {reason});\n  }\n\n  #invariantFailure(message: string, details?: Record<string, unknown>) {\n    this.#invariantFailures++;\n    this.#invariantFailureCounter.add(1);\n    this.#lc.error?.(message, details);\n  }\n\n  #compareHashes(\n    watermark: string,\n    sourceHash: string | undefined,\n    sqliteHash: string,\n  ): void {\n    if (sourceHash === undefined) {\n      this.#recordUnpaired(watermark, 'missing-source-hash');\n      return;\n    }\n    if (sourceHash === sqliteHash) {\n      this.#hashMatches++;\n      this.#hashComparisonCounter.add(1, {outcome: 'match'});\n      return;\n    }\n    this.#hashMismatches++;\n    this.#hashComparisonCounter.add(1, {outcome: 'mismatch'});\n    this.#lc.error?.('SQLite change-log transaction hash mismatch', {\n      sqliteChangeLogHashComparison: {\n        watermark,\n        sourceHash: sourceHash.slice(0, 16),\n        sqliteHash: sqliteHash.slice(0, 16),\n      },\n    });\n  }\n\n  #recordUnpaired(\n    watermark: string,\n    reason:\n      | 'missing-source-hash'\n      | 'missing-sqlite-hash'\n      | 'sqlite-commit-failed',\n  ): void {\n    this.#hashUnpaired++;\n    this.#hashUnpairedCounter.add(1, {reason});\n    this.#lc.warn?.('SQLite change-log transaction hash was not compared', {\n      sqliteChangeLogHashComparison: {watermark, reason},\n    });\n  }\n\n  #resetSourceHash(): void {\n    this.#sourceHasher = undefined;\n    this.#sourcePos = 0;\n    this.#sourceCommitHash = undefined;\n  }\n}\n\nfunction watermarkValue(watermark: string): number | undefined {\n  try {\n    const value = Number(versionFromLexi(watermark));\n    return Number.isFinite(value) ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n\nfunction watermarkDistance(\n  receivedHead: string,\n  sqliteHead: string,\n): number | undefined {\n  if (receivedHead <= sqliteHead) {\n    return 0;\n  }\n  try {\n    const distance =\n      versionFromLexi(receivedHead) - versionFromLexi(sqliteHead);\n    const value = Number(distance);\n    return Number.isFinite(value) ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoDA,SAAgB,+BACd,IACA,QACM;CACN,QAAQ,OAAO,QAAf;EACE,KAAK;GACH,mBACE,WACA,8CACA,kHAEF,EAAE,IAAI,OAAO,IAAI;GACjB;EACF,KAAK;GACH,mBACE,WACA,qCACA,mIAEF,EAAE,IAAI,GAAG,EAAC,QAAQ,OAAO,OAAM,CAAC;GAChC;EACF,KAAK,QACH;EACF,SACE,YAAY,MAAM;CACtB;CACA,IAAI,OAAO,cAKT,mBACE,WACA,mCACA,sGAEF,EAAE,IAAI,GAAG,EAAC,QAAQ,OAAO,OAAM,CAAC;CAElC,GAAG,QAAQ,oCAAoC,EAC7C,0BAA0B,OAC5B,CAAC;AACH;;;;;;;;AAwBA,SAAgB,8BACd,WAC4B;CAC5B,MAAM,OAAO,kBAAkB,SAAS;CACxC,MAAM,OAAO,kBAAkB,SAAS;CACxC,IAAI,SAAS,MACX,MAAM,IAAI,MAAM,yDAAyD;CAE3E,OAAO;EAAC,GAAG;EAAM,eAAe;CAAI;AACtC;;;;;;;AAQA,SAAgB,uBACd,WACqB;CACrB,MAAM,UAAU,8BAA8B,SAAS;CACvD,MAAM,YAAY,UACf,QAAgB;;;gBAGL,wBAAwB;KACnC,EACA,IAA4C;CAE/C,OAAO;EACL,GAAG;EACH,MAAM,UAAU;EAChB,gBAAgB,UAAU;EAC1B,YAAY,oBAAoB,SAAS;CAC3C;AACF;AAEA,SAAgB,0BACd,IACA,MACA,MACM;CACN,GAAG,OAAO,6BAA6B,EACrC,iBAAiB;EACf;EACA,OAAO,KAAK;EACZ,YAAY,KAAK;EACjB,WAAW,KAAK;EAChB,eAAe,KAAK;EACpB,eAAe,KAAK;EACpB,YAAY,KAAK;EACjB,eAAe,KAAK;CACtB,EACF,CAAC;AACH;;;;;;;;;;AAWA,eAAsB,gBACpB,IACA,MACiB;CAajB,QAAO,MAZa,QAAQ,IAC1B;EAAC;EAAM,GAAG,KAAK;EAAO,GAAG,KAAK;CAAM,EAAE,IAAI,OAAM,SAAQ;EACtD,IAAI;GACF,QAAQ,MAAM,KAAK,IAAI,GAAG;EAC5B,SAAS,GAAG;GACV,IAAK,EAA4B,SAAS,UACxC,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;GAExD,OAAO;EACT;CACF,CAAC,CACH,GACa,QAAQ,OAAO,SAAS,QAAQ,MAAM,CAAC;AACtD;AA4BA,IAAM,0BAA0B;CAAC;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAM;AAAI;;;;;;;AAQ1E,IAAa,0BAAb,MAAqC;CACnC;CACA,qBAA8B,4BAC5B,WACA,iDACA,+DACF;CAMA,qBAA8B,4BAC5B,WACA,yCACA,+KAGF;CACA,mBAA4B,0BAC1B,WACA,sCACA;EACE,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CACF;CACA,mBAA4B,mBAC1B,WACA,+BACA,2DACF;CACA,2BAAoC,mBAClC,WACA,wCACA,uDACF;CACA,yBAAkC,mBAChC,WACA,sCACA,yGACF;CACA,uBAAgC,mBAC9B,WACA,mCACA,2FACF;CAEA,yBAAkC,mBAChC,WACA,sCACA,uEACF;CAEA;CACA;CACA;CACA;CACA;CACA,mBAAmB;CACnB;CACA;CACA,aAAa;CACb;CACA,aAAa;CACb,qBAAqB;CACrB,eAAe;CACf,kBAAkB;CAClB,gBAAgB;CAEhB,YAAY,IAAgB,MAA2B;EACrD,KAAKA,MAAM,GAAG,YAAY,aAAa,4BAA4B;EAGnE,KAAKS,gBAAgB,KAAK;EAC1B,KAAKC,cAAc,KAAK;EACxB,KAAKC,QAAQ,KAAK;EAClB,KAAKC,kBAAkB,KAAK;EAC5B,KAAKC,cAAc,KAAK;EAExB,iBACE,WACA,0BACA,yCACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKF,KAAK,CAAC;EAGlD,iBACE,WACA,iCACA,iEACF,EAAE,aAAY,WAAU;GACtB,OAAO,QAAQ,KAAKE,YAAY,eAAe,EAAC,OAAO,gBAAe,CAAC;GACvE,OAAO,QAAQ,KAAKA,YAAY,aAAa,EAAC,OAAO,cAAa,CAAC;EACrE,CAAC;EACD,iBAAiB,WAAW,oCAAoC;GAC9D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKD,eAAe,CAAC;EAC7D,iBACE,WACA,0BACA,oEACF,EAAE,aAAY,WAAU;GACtB,MAAM,OAAO,eAAe,KAAKF,WAAW;GAC5C,IAAI,SAAS,KAAA,GACX,OAAO,QAAQ,IAAI;EAEvB,CAAC;EACD,iBACE,WACA,8BACA,0JAGF,EAAE,aAAY,WAAU;GACtB,MAAM,MAAM,kBAAkB,KAAKD,eAAe,KAAKC,WAAW;GAClE,IAAI,QAAQ,KAAA,GACV,OAAO,QAAQ,GAAG;EAEtB,CAAC;CACH;;;;;;CAOA,gBAAgB,MAAwB,MAAoB;EAC1D,MAAM,MAAM,KAAK,GAAG;EACpB,IAAI,KAAK,OAAO,SAAS;GACvB,IAAI,KAAKI,kBAAkB,KAAA,GACzB,KAAKC,kBACH,iEACA,EAAC,mBAAmB,KAAK,GAAG,gBAAe,CAC7C;GAEF,MAAM,YAAY,KAAK,GAAG;GAC1B,KAAKD,gBAAgB,IAAI,2BAA2B;GACpD,KAAKE,aAAa;GAClB,KAAKC,oBAAoB,KAAA;GACzB,KAAKH,cAAc,IAAI;IACrB;IACA,KAAK,KAAKE;IACV;IACA,QAAQ,uBAAuB,MAAM,GAAG;IACxC,WAAW;GACb,CAAC;GACD;EACF;EAEA,IAAI,QAAQ,YAAY;GACtB,KAAKE,iBAAiB;GACtB;EACF;EAEA,IAAI,KAAK,OAAO,UACd,KAAKT,gBAAgB,KAAK,GAAG;EAG/B,MAAM,YAAY,KAAKU;EACvB,IAAI,KAAKL,kBAAkB,KAAA,KAAa,cAAc,KAAA,GAAW;GAC/D,KAAKC,kBACH,+DACA,EAAC,IAAG,CACN;GACA;EACF;EAEA,MAAM,kBACJ,KAAK,OAAO,WAAW,KAAK,GAAG,YAAY;EAC7C,KAAKD,cAAc,IAAI;GACrB,WAAW;GACX,KAAK,EAAE,KAAKE;GACZ;GACA,QAAQ,uBAAuB,MAAM,GAAG;GACxC,WAAW,KAAK,OAAO,WAAW,YAAY;EAChD,CAAC;EACD,IAAI,KAAK,OAAO,UAAU;GACxB,KAAKC,oBAAoB,KAAKH,cAAc,OAAO;GACnD,KAAKA,gBAAgB,KAAA;EACvB;CACF;;CAGA,iBACE,MACA,QACA,YACM;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,KAAKb,mBAAmB,SAAS,YAAY;GAAC;GAAK,SAAS;EAAS,CAAC;EAEtE,IAAI,KAAK,OAAO,SAAS;GACvB,IAAI,KAAKkB,0BAA0B,KAAA,GACjC,KAAKJ,kBAAkB,8CAA8C;IACnE,eAAe,KAAKI;IACpB,mBAAmB,KAAK,GAAG;GAC7B,CAAC;GAEH,KAAKA,wBAAwB,KAAK,GAAG;GACrC;EACF;EAEA,IAAI,QAAQ,YAAY;GACtB,KAAKC,gBAAgB,UAAU;GAC/B,KAAKD,wBAAwB,KAAA;GAC7B,KAAKD,iBAAiB;GACtB;EACF;EAEA,IAAI,KAAK,OAAO,UACd;EAGF,MAAM,YAAY,KAAK,GAAG;EAC1B,IAAI,WAAW,MACb,KAAKhB,mBAAmB,SAAS,OAAO,WAAW;EAErD,IAAI,KAAKiB,0BAA0B,WACjC,KAAKJ,kBACH,8DACA;GAAC,eAAe,KAAKI;GAAuB,iBAAiB;EAAS,CACxE;EAEF,IAAI,WAAW,MAAM;GACnB,KAAKJ,kBACH,gEACA,EAAC,iBAAiB,UAAS,CAC7B;GACA,KAAKM,gBAAgB,WAAW,qBAAqB;EACvD,OAAO;GACL,IAAI,OAAO,cAAc,WACvB,KAAKN,kBACH,+DACA;IAAC,iBAAiB;IAAW,iBAAiB,OAAO;GAAS,CAChE;GAEF,KAAKJ,SAAS,OAAO,MAAM;GAC3B,KAAKC,mBAAmB,OAAO,MAAM;GACrC,KAAKT,iBAAiB,OAAO,OAAO,MAAM,IAAI;GAC9C,KAAKmB,eAAe,WAAW,KAAKL,mBAAmB,OAAO,MAAM,IAAI;EAC1E;EACA,IAAI,YAAY,KAAKP,aACnB,KAAKK,kBAAkB,oCAAoC;GACzD,cAAc,KAAKL;GACnB,iBAAiB;EACnB,CAAC;EAEH,KAAKA,cAAc;EACnB,KAAKS,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;CAEA,cACE,MACA,OACA,YACM;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,KAAKjB,mBAAmB,SAAS,YAAY;GAAC;GAAK,SAAS;EAAO,CAAC;EACpE,IAAI,KAAK,OAAO,UAId,KAAKoB,gBAAgB,KAAK,GAAG,WAAW,sBAAsB;EAEhE,IACE,iBAAiB,SACjB,MAAM,SAAS,iCAEf,KAAKN,kBAAkB,MAAM,OAAO;EAEtC,IAAI,KAAKI,0BAA0B,KAAA,KAAa,KAAK,OAAO,SAC1D,KAAKC,gBAAgB,kBAAkB;EAEzC,KAAKD,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;;;;;;CAOA,eAAe,KAA6B,MAA6B;EACvE,KAAK,MAAM,MAAM,KAAK;GACpB,KAAKK;GACL,KAAKf,uBAAuB,IAAI,GAAG,EAAC,GAAE,CAAC;EACzC;EACA,KAAKK,cAAc;CACrB;;;;;;;CAQA,iBAAiB,SAAiB,SAAyC;EACzE,KAAKE,kBAAkB,SAAS,OAAO;CACzC;;CAGA,QAAc;EACZ,IAAI,KAAKI,0BAA0B,KAAA,GAAW;GAC5C,KAAKC,gBAAgB,qBAAqB;GAC1C,KAAKD,wBAAwB,KAAA;EAC/B;EACA,KAAKD,iBAAiB;CACxB;;CAGA,WAAW,MAAiC;EAC1C,KAAKT,gBAAgB,KAAK;EAC1B,KAAKC,cAAc,KAAK;EACxB,KAAKC,QAAQ,KAAK;EAClB,KAAKC,kBAAkB,KAAK;EAC5B,KAAKC,cAAc,KAAK;EACxB,KAAKM,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;CAEA,QAA2C;EACzC,OAAO;GACL,cAAc,KAAKT;GACnB,YAAY,KAAKC;GACjB,SAAS,kBAAkB,KAAKD,eAAe,KAAKC,WAAW;GAC/D,MAAM,KAAKC;GACX,gBAAgB,KAAKC;GACrB,YAAY,KAAKC;GACjB,iBAAiB,KAAKU;GACtB,WAAW,KAAKC;GAChB,mBAAmB,KAAKC;GACxB,aAAa,KAAKC;GAClB,gBAAgB,KAAKC;GACrB,cAAc,KAAKC;EACrB;CACF;CAEA,gBACE,QACA;EACA,KAAKJ;EACL,KAAKpB,iBAAiB,IAAI,GAAG,EAAC,OAAM,CAAC;CACvC;CAEA,kBAAkB,SAAiB,SAAmC;EACpE,KAAKqB;EACL,KAAKpB,yBAAyB,IAAI,CAAC;EACnC,KAAKL,IAAI,QAAQ,SAAS,OAAO;CACnC;CAEA,eACE,WACA,YACA,YACM;EACN,IAAI,eAAe,KAAA,GAAW;GAC5B,KAAKqB,gBAAgB,WAAW,qBAAqB;GACrD;EACF;EACA,IAAI,eAAe,YAAY;GAC7B,KAAKK;GACL,KAAKpB,uBAAuB,IAAI,GAAG,EAAC,SAAS,QAAO,CAAC;GACrD;EACF;EACA,KAAKqB;EACL,KAAKrB,uBAAuB,IAAI,GAAG,EAAC,SAAS,WAAU,CAAC;EACxD,KAAKN,IAAI,QAAQ,+CAA+C,EAC9D,+BAA+B;GAC7B;GACA,YAAY,WAAW,MAAM,GAAG,EAAE;GAClC,YAAY,WAAW,MAAM,GAAG,EAAE;EACpC,EACF,CAAC;CACH;CAEA,gBACE,WACA,QAIM;EACN,KAAK4B;EACL,KAAKrB,qBAAqB,IAAI,GAAG,EAAC,OAAM,CAAC;EACzC,KAAKP,IAAI,OAAO,uDAAuD,EACrE,+BAA+B;GAAC;GAAW;EAAM,EACnD,CAAC;CACH;CAEA,mBAAyB;EACvB,KAAKc,gBAAgB,KAAA;EACrB,KAAKE,aAAa;EAClB,KAAKC,oBAAoB,KAAA;CAC3B;AACF;AAEA,SAAS,eAAe,WAAuC;CAC7D,IAAI;EACF,MAAM,QAAQ,OAAO,gBAAgB,SAAS,CAAC;EAC/C,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;CAC1C,QAAQ;EACN;CACF;AACF;AAEA,SAAS,kBACP,cACA,YACoB;CACpB,IAAI,gBAAgB,YAClB,OAAO;CAET,IAAI;EACF,MAAM,WACJ,gBAAgB,YAAY,IAAI,gBAAgB,UAAU;EAC5D,MAAM,QAAQ,OAAO,QAAQ;EAC7B,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;CAC1C,QAAQ;EACN;CACF;AACF"}

package/out/zero-cache/src/services/replicator/sqlite-change-log-purger.d.ts

@@ -0,0 +1,120 @@
+/**
+ * Deletes aged-out transactions from the change-log database, one bounded write
+ * transaction at a time.
+ *
+ * The purger is a pure function of one database: it takes the floors its caller
+ * has already gated on and narrows them, never widens them, and it reads no
+ * replication state. That is what makes it testable without a scheduler and
+ * what lets slice 9 run it from the change-streamer's stream loop, between the
+ * writer's commits, on the writer's own connection.
+ */
+import type { Database } from '../../../../zqlite/src/db.ts';
+/**
+ * The eligible set, as three ANDed bounds. Only the `externalFloor` is a safety
+ * property; the other two make the log serviceable and make purge conservative.
+ *
+ * `"watermark" < @externalFloor` is deliberately strict. A follower that lost
+ * its replica restores it from S3 at watermark `W` and then asks this log for
+ * everything after `W`, which requires the `commit` row *at* `W` to still be
+ * here (`SQLITE_CHANGE_LOG_BOUNDARY_SQL`). Deleting it returns `too-old` to the
+ * very subscriber the floor exists to protect.
+ *
+ * The head cap reads `max("watermark")` from the log's own database, which
+ * keeps the purger a pure function of one file. The cap is inert in normal
+ * operation: the floor is derived from a backup of a replica that consumes this
+ * loop's output, and the log commits before that output is forwarded, so
+ * `externalFloor <= head` holds by construction. It engages only on a *stale*
+ * log — writer stopped, mode flipped to `off`, another task took over — where
+ * deleting to a floor that has run past a frozen log would empty the file. Note
+ * that head can be a phantom — committed here but not yet re-deliverable by a
+ * resumed stream — for as long as it takes the watermark the stream would
+ * resume from to advance past it. That is harmless precisely because head is
+ * never a purge candidate, and it is why the purger asserts nothing about
+ * anything outside its own file.
+ *
+ * Ordered by `("writeTimeMs", "watermark")` so it rides the partial index,
+ * which carries one entry per transaction rather than one per change. The
+ * `LIMIT` keeps a drain from re-scanning the remaining history on every batch,
+ * which would be quadratic in the number of eligible transactions and would do
+ * it inside the `BEGIN IMMEDIATE`, holding the write lock alongside the delete.
+ *
+ * Exported so the focused query-plan test covers the production query.
+ */
+export declare const SQLITE_CHANGE_LOG_ELIGIBLE_COMMITS_SQL = "\n  SELECT \"watermark\"\n  FROM \"_zero.changeLogStream\"\n  WHERE \"writeTimeMs\" IS NOT NULL\n    AND \"writeTimeMs\" < @retentionCutoffMs\n    AND \"watermark\" < @externalFloor\n    AND \"watermark\" < (SELECT max(\"watermark\") FROM \"_zero.changeLogStream\")\n  ORDER BY \"writeTimeMs\", \"watermark\"\n  LIMIT @limit\n";
+/**
+ * Counts the rows at `@watermark`, capped at `@limit`. The cap is what keeps
+ * batch sizing O(budget): this runs per candidate inside the
+ * `BEGIN IMMEDIATE`, and the transaction being sized can be arbitrarily
+ * large — an uncapped count of a multi-million-row backfill would rescan
+ * every remaining row on every tear batch, holding the write lock (and the
+ * appending path behind it) for the duration, quadratically over the drain.
+ * The caller only needs to know whether the transaction fits its remaining
+ * budget, so it passes `remaining + 1` and reads a count above `remaining`
+ * as "does not fit".
+ *
+ * Exported so the focused tests cover the production query.
+ */
+export declare const SQLITE_CHANGE_LOG_COUNT_TRANSACTION_SQL = "\n  SELECT count(*) AS \"rows\" FROM (\n    SELECT 1\n    FROM \"_zero.changeLogStream\"\n    WHERE \"watermark\" = @watermark\n    LIMIT @limit\n  )\n";
+export type PurgeBatchResult = {
+    readonly deletedRows: number;
+    /**
+     * The watermark through which whole transactions were removed. `undefined`
+     * when the batch only tore rows off the minimum transaction without
+     * completing it — in that case `deletedRows > 0` and `moreEligible` is true.
+     */
+    readonly deletedThrough: string | undefined;
+    /** True while a tear is in progress, as well as while whole transactions remain eligible. */
+    readonly moreEligible: boolean;
+};
+export type PurgeBatchOptions = {
+    /**
+     * The floor the caller has already gated on subscriber ACKs and snapshot
+     * reservations — normally the confirmed S3 backup watermark. Transactions at
+     * or above it are NOT durable anywhere else: the change log is excluded from
+     * the litestream backup, so this is the boundary a follower restores to and
+     * then catches up from. Compared strictly (`<`), so the transaction *at* the
+     * floor survives as that follower's catchup boundary. The purger narrows this
+     * input and never widens it.
+     */
+    readonly externalFloor: string;
+    /**
+     * A *minimum* retention, ANDed on top of `externalFloor` rather than an
+     * alternative to it. Which of the two binds depends on the backup interval.
+     */
+    readonly retentionCutoffMs: number;
+    /** Starting chunk size. Not the bound — `maxDurationMs` is. */
+    readonly maxRows: number;
+    /**
+     * The statement duration target, enforced by feedback rather than by
+     * interruption: SQLite cannot yield mid-statement, so the purger measures
+     * each delete and shrinks the next batch's row budget when one overruns
+     * (see {@link SQLiteChangeLogPurger.budgetRows}) — which means the first
+     * delete after a restart, sized by `maxRows` alone, can still overrun it.
+     * The target is this consequential because purge shares the writer's
+     * connection: there is no lock to arbitrate and no `busy_timeout` to bound
+     * the wait — a long statement blocks the appending path, and the forward
+     * behind it, directly and for its full duration. That is why this is a
+     * precondition of purging from the stream loop rather than a tuning knob.
+     */
+    readonly maxDurationMs: number;
+};
+export declare class SQLiteChangeLogPurger {
+    #private;
+    /**
+     * @param db a writable connection to a change log the writer has already
+     *     created and reconciled — in slice 9, the writer's own connection.
+     *     Statements are prepared eagerly, so a caller that may run before the
+     *     log exists opens the connection lazily and treats construction failure
+     *     as a skipped cycle.
+     */
+    constructor(db: Database, now?: () => number);
+    /** The row budget the next batch would use. Exposed for tests and metrics. */
+    get budgetRows(): number | undefined;
+    /**
+     * Removes one bounded batch of aged-out history, in one `BEGIN IMMEDIATE`
+     * transaction. Callers loop while `moreEligible`, releasing whatever guards
+     * they hold between batches.
+     */
+    purgeBatch(opts: PurgeBatchOptions): PurgeBatchResult;
+}
+//# sourceMappingURL=sqlite-change-log-purger.d.ts.map

package/out/zero-cache/src/services/replicator/sqlite-change-log-purger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlite-change-log-purger.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/sqlite-change-log-purger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,8BAA8B,CAAC;AAGtE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,eAAO,MAAM,sCAAsC,2UASlD,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,uCAAuC,4JAOnD,CAAC;AAuCF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5C,6FAA6F;IAC7F,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;;;;;;;OAQG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;;;OAGG;IACH,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,+DAA+D;IAC/D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC,CAAC;AAWF,qBAAa,qBAAqB;;IAahC;;;;;;OAMG;gBAED,EAAE,EAAE,QAAQ,EACZ,GAAG,GAAE,MAAM,MAA0C;IAcvD,8EAA8E;IAC9E,IAAI,UAAU,IAAI,MAAM,GAAG,SAAS,CAEnC;IAED;;;;OAIG;IACH,UAAU,CAAC,IAAI,EAAE,iBAAiB,GAAG,gBAAgB;CAwMtD"}