@rocicorp/zero 1.9.1-canary.1 → 1.10.0-canary.1

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts

@@ -2,36 +2,81 @@ import type { LogContext } from '@rocicorp/logger';
 import type { PostgresDB } from '../../types/pg.ts';
 import type { ShardID } from '../../types/shards.ts';
 import type { ChangeSource } from '../change-source/change-source.ts';
-import { type ChangeStreamData } from '../change-source/protocol/current/downstream.ts';
+import type { LitestreamVersion } from '../litestream/metrics.ts';
 import { type ReplicationStatusPublisher } from '../replicator/replication-status.ts';
 import type { SubscriptionState } from '../replicator/schema/replication-state.ts';
-import { type ChangeStreamerService } from './change-streamer.ts';
+import { type ChangeStreamerService, type SubscriberContext } from './change-streamer.ts';
+import { type SQLiteChangeLogCleanupGuard } from './sqlite-change-log-catchup.ts';
+import { type SQLiteChangeLogPurgeSchedulerOptions } from './sqlite-change-log-purge-scheduler.ts';
+import { type SQLiteChangeLogWriterOptions } from './sqlite-change-log-writer.ts';
 import { type PurgeLock, type TuningOptions as StorerOptions } from './storer.ts';
+export type BackupConfig = {
+    backupURL: string;
+    litestreamVersion: LitestreamVersion;
+};
+export type SQLiteCatchupOptions = {
+    /** The change-log database, i.e. `changeLogFileName(replicaFile)`. */
+    changeLogFile: string;
+    readBatchRows: number;
+    barrierTimeoutMs: number;
+    /**
+     * Backstop for the change-log writer's ACK, which is what normally releases
+     * the barrier. Defaults to the catchup coordinator's interval.
+     */
+    barrierPollIntervalMs?: number | undefined;
+    /**
+     * Slice 11 supplies the production canary selector.
+     *
+     * It does not need to exclude backup subscribers or the change-log writer:
+     * eligibility checks reject both before invoking this selector. Serving a
+     * writer from SQLite would make it wait on its own ACK.
+     */
+    shouldUse?: ((ctx: SubscriberContext) => boolean) | undefined;
+    /**
+     * Overrides the purge scheduler's guard, for tests. When absent, the
+     * service supplies the scheduler's real writer-serialized guard, or the
+     * catchup's no-op default when purging is not configured.
+     */
+    cleanupGuard?: SQLiteChangeLogCleanupGuard | undefined;
+    /**
+     * How long the change log may be unavailable before declining to serve from
+     * it is reported as a warning rather than a debug line. Defaults to
+     * {@link DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS}.
+     */
+    notReadyWarnThresholdMs?: number | undefined;
+};
 export type TuningOptions = StorerOptions & {
-    flowControlConsensusPaddingSeconds: number;
+    flowControlConsensusTimeoutProportion: number;
+    flowControlSlowSubscriberGracePeriodMs?: number | undefined;
+    sqliteCatchup?: SQLiteCatchupOptions | undefined;
+    /**
+     * Supplied when `sqliteChangeLogMode != off`, i.e. this is the gate on the
+     * change-log writer. Absent, nothing writes the log.
+     */
+    sqliteChangeLogWriter?: Omit<SQLiteChangeLogWriterOptions, 'onCommit' | 'onDisabled' | 'onRebuilt'> | undefined;
+    /**
+     * Also supplied when `sqliteChangeLogMode != off`, and deliberately not
+     * gated on the read path: in `write` mode no reader ever opens, and this is
+     * the configuration the purge scheduler actually ships in. The scheduler
+     * runs on the writer's own connection, so with the writer absent (mode
+     * `off`) it is never constructed, and a leftover file on disk is left to
+     * the replicator's cleanup.
+     */
+    sqliteChangeLogPurge?: SQLiteChangeLogPurgeSchedulerOptions | undefined;
+    /**
+     * Supplied at `sqliteChangeLogMode >= compare`, and the gate on deriving the
+     * stream's initialization parameters from the replica as well as from
+     * Postgres. Postgres stays authoritative; the two are compared and the
+     * result is reported. Not a new configuration option -- the mode ladder
+     * carries it, because a state in which the log's buffer and its cookies were
+     * at different rollout stages is what invariant 15 exists to forbid.
+     */
+    sqliteChangeLogCompare?: {
+        replicaFile: string;
+    } | undefined;
 };
 /**
  * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.
  */
-export declare function initializeStreamer(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, changeDB: PostgresDB, changeSource: ChangeSource, replicationStatusPublisher: ReplicationStatusPublisher, subscriptionState: SubscriptionState, purgeLock: PurgeLock | null, autoReset: boolean, opts: TuningOptions, setTimeoutFn?: typeof setTimeout): Promise<ChangeStreamerService>;
-export type ChangeTag = ChangeStreamData[1]['tag'];
-/**
- * Internally all Downstream messages (not just commits) are given a watermark.
- * These are used for internal ordering for:
- * 1. Replaying new changes in the Storer
- * 2. Filtering old changes in the Subscriber
- *
- * However, only the watermark for `Commit` messages are exposed to
- * subscribers, as that is the only semantically correct watermark to
- * use for tracking a position in a replication stream.
- *
- * Additionally, the ChangeStreamData is eagerly stringified once, after which
- * the string is passed to the changeLog and all subscribers, eliminating
- * redundant stringification and reducing GC churn.
- */
-export type WatermarkedChange = [
-    watermark: string,
-    tag: ChangeTag,
-    json: string
-];
+export declare function initializeStreamer(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, changeDB: PostgresDB, changeSource: ChangeSource, replicationStatusPublisher: ReplicationStatusPublisher, subscriptionState: SubscriptionState, backupConfig: BackupConfig | null, purgeLock: PurgeLock | null, autoReset: boolean, opts: TuningOptions, setTimeoutFn?: typeof setTimeout): Promise<ChangeStreamerService>;
 //# sourceMappingURL=change-streamer-service.d.ts.map

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EAEtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAGrB,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC;AAID,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC"}
+{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAM3C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAUjF,OAAO,EACL,KAAK,qBAAqB,EAE1B,KAAK,iBAAiB,EAEvB,MAAM,sBAAsB,CAAC;AAU9B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAGL,KAAK,oCAAoC,EAC1C,MAAM,wCAAwC,CAAC;AAEhD,OAAO,EAEL,KAAK,4BAA4B,EAClC,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAIrB,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,iBAAiB,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;IAC9D;;;;OAIG;IACH,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5D,aAAa,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACjD;;;OAGG;IACH,qBAAqB,CAAC,EAClB,IAAI,CACF,4BAA4B,EAC5B,UAAU,GAAG,YAAY,GAAG,WAAW,CACxC,GACD,SAAS,CAAC;IACd;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,oCAAoC,GAAG,SAAS,CAAC;IACxE;;;;;;;OAOG;IACH,sBAAsB,CAAC,EAAE;QAAC,WAAW,EAAE,MAAM,CAAA;KAAC,GAAG,SAAS,CAAC;CAC5D,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -1,31 +1,38 @@
-import { unreachable } from "../../../../shared/src/asserts.js";
+import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { must } from "../../../../shared/src/must.js";
 import "../change-source/protocol/current/downstream.js";
-import { getOrCreateCounter } from "../../observability/metrics.js";
+import { getOrCreateCounter, getOrCreateLatencyHistogram } from "../../observability/metrics.js";
 import { DEFAULT_MAX_RETRY_DELAY_MS, RunningState, UnrecoverableError } from "../running-state.js";
 import { min } from "../../types/lexi-version.js";
 import { publishCriticalEvent } from "../../observability/events.js";
-import { AutoResetSignal, ensureReplicationConfig, markResetRequired } from "./schema/tables.js";
 import { Subscription } from "../../types/subscription.js";
+import { AutoResetSignal, ensureReplicationConfig, markResetRequired } from "./schema/tables.js";
 import { publishReplicationError, replicationStatusError } from "../replicator/replication-status.js";
 import "./change-streamer.js";
+import { SQLiteChangeLogReader } from "./sqlite-change-log-reader.js";
+import { ChangeLogInitializer, replicaInitializationSource } from "./change-log-initializer.js";
 import { Forwarder } from "./forwarder.js";
-import { initChangeStreamerSchema } from "./schema/init.js";
+import { SnapshotReservations } from "./snapshot-reservations.js";
+import { SQLiteChangeLogCatchup } from "./sqlite-change-log-catchup.js";
+import { SQLiteChangeLogPurgeScheduler } from "./sqlite-change-log-purge-scheduler.js";
+import { SQLiteChangeLogWriter } from "./sqlite-change-log-writer.js";
 import { Storer } from "./storer.js";
 import { Subscriber } from "./subscriber.js";
+import { UpstreamAcker } from "./upstream-acker.js";
 import { resolver } from "@rocicorp/resolver";
 import { getDefaultHighWaterMark } from "node:stream";
 //#region ../zero-cache/src/services/change-streamer/change-streamer-service.ts
 /**
 * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.
 */
-async function initializeStreamer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, changeSource, replicationStatusPublisher, subscriptionState, purgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
-	await initChangeStreamerSchema(lc, changeDB, shard);
-	await ensureReplicationConfig(lc, changeDB, subscriptionState, shard, autoReset, setTimeoutFn);
+async function initializeStreamer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, changeSource, replicationStatusPublisher, subscriptionState, backupConfig, purgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
+	await ensureReplicationConfig(lc, changeDB, subscriptionState, shard, autoReset, purgeLock ?? void 0, setTimeoutFn);
 	const { replicaVersion } = subscriptionState;
-	return new ChangeStreamerImpl(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, changeSource, replicationStatusPublisher, purgeLock, autoReset, opts, setTimeoutFn);
+	return new ChangeStreamerImpl(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, changeSource, replicationStatusPublisher, backupConfig, purgeLock, autoReset, opts, setTimeoutFn);
 }
 var REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5e3;
+var DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS = 6e4;
 /**
 * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a
 * {@link Forwarder} and {@link Storer} to execute the forward-store-ack
@@ -173,30 +180,84 @@ var ChangeStreamerImpl = class {
 	#source;
 	#storer;
 	#forwarder;
+	#reservations;
 	#replicationStatusPublisher;
+	#sqliteCatchupOptions;
+	#changeLogWriter;
+	#purgeScheduler;
+	#acker;
+	#initializer;
 	#autoReset;
 	#state;
-	#initialWatermarks = /* @__PURE__ */ new Set();
 	#serving = resolver();
 	#txCounter = getOrCreateCounter("replication", "transactions", "Count of replicated transactions");
 	#changeCounter = getOrCreateCounter("replication", "changes", "Count of replicated changes (DML or DDL statements)");
+	#transactionForwardDuration = getOrCreateLatencyHistogram("replication", "transaction_forward_duration", "Time from receiving a transaction's `begin` to forwarding its `commit`, i.e. the change-streamer half of forward-to-subscriber latency.");
 	#latestStatus;
+	#latestLagReportCommitTimeMs = 0;
+	#backupWatermark;
+	#pgPurgedWatermark = "";
+	/**
+	* The floor last logged as held back by a laggard. The level-triggered
+	* retry re-evaluates every {@link CLEANUP_DELAY_MS}; logging only when the
+	* blocking floor moves keeps a stuck subscriber from emitting the same
+	* line indefinitely.
+	*/
+	#loggedBehindWatermark;
+	#sqlitePurgeContinuation;
 	#purgeLock;
+	#pgPurgeScheduled = false;
+	#pgPurgeRunning = false;
+	#sqlitePurgeScheduled = false;
+	#sqlitePurgeRunning = false;
 	#stream;
-	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, source, replicationStatusPublisher, initialPurgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
+	#sqliteCatchup;
+	#changeLogUnavailableSince;
+	#lastForwardedCommitWatermark;
+	#transactionForwardStartedAt;
+	#currentTransactionCompletion;
+	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, source, replicationStatusPublisher, backupConfig, initialPurgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
 		this.id = `change-streamer`;
 		this.#lc = lc.withContext("component", "change-streamer");
 		this.#shard = shard;
 		this.#changeDB = changeDB;
 		this.#replicaVersion = replicaVersion;
 		this.#source = source;
-		this.#storer = new Storer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, (consumed) => this.#stream?.acks.push([
-			"status",
-			consumed[1],
-			consumed[2]
-		]), (err) => this.stop(err), opts);
-		this.#forwarder = new Forwarder(lc, { flowControlConsensusPaddingSeconds: opts.flowControlConsensusPaddingSeconds });
+		this.#storer = new Storer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, (consumed) => this.#acker.trackPgChangeLog(consumed[2].watermark), (err) => this.stop(err), opts);
+		this.#forwarder = new Forwarder(lc, {
+			flowControlConsensusTimeoutProportion: opts.flowControlConsensusTimeoutProportion,
+			flowControlSlowSubscriberGracePeriodMs: opts.flowControlSlowSubscriberGracePeriodMs
+		});
+		this.#reservations = backupConfig ? new SnapshotReservations(lc, backupConfig, (taskID) => this.#purgeScheduler?.resume(taskID)) : void 0;
 		this.#replicationStatusPublisher = replicationStatusPublisher;
+		this.#changeLogWriter = opts.sqliteChangeLogWriter ? new SQLiteChangeLogWriter(lc, {
+			...opts.sqliteChangeLogWriter,
+			onCommit: (watermark) => {
+				this.#sqliteCatchup?.onChangeLogCommit(watermark);
+				this.#purgeScheduler?.onWriterIdle();
+			},
+			onDisabled: () => this.#closeSQLiteCatchup(),
+			onRebuilt: () => this.#closeSQLiteCatchup()
+		}) : void 0;
+		this.#purgeScheduler = opts.sqliteChangeLogPurge && this.#changeLogWriter ? new SQLiteChangeLogPurgeScheduler(lc, () => this.#changeLogWriter?.connection, () => this.#forwarder.getAcks(), opts.sqliteChangeLogPurge) : void 0;
+		this.#acker = new UpstreamAcker({
+			trackPgChangeLog: true,
+			trackBackup: backupConfig?.litestreamVersion === "v5"
+		});
+		const replicaSource = opts.sqliteChangeLogCompare ? replicaInitializationSource(lc, opts.sqliteChangeLogCompare.replicaFile) : void 0;
+		this.#initializer = new ChangeLogInitializer(lc, {
+			initFromPgChangeLog: true,
+			initFromReplica: replicaSource !== void 0
+		}, {
+			pgChangeLog: () => this.#storer.getStartStreamInitializationParameters(),
+			replica: () => must(replicaSource)(),
+			reconcileChangeLog: (resumeFrom, seed) => resumeFrom ? this.#changeLogWriter?.reconcile(resumeFrom) : this.#changeLogWriter?.reconcileFromLog(seed),
+			changeLog: () => this.#changeLogWriter?.connection
+		});
+		this.#sqliteCatchupOptions = opts.sqliteCatchup ? {
+			...opts.sqliteCatchup,
+			cleanupGuard: opts.sqliteCatchup.cleanupGuard ?? this.#purgeScheduler?.cleanupGuard
+		} : void 0;
 		this.#purgeLock = initialPurgeLock;
 		this.#autoReset = autoReset;
 		this.#state = new RunningState(this.id, void 0, setTimeoutFn);
@@ -205,8 +266,11 @@ var ChangeStreamerImpl = class {
 	async run() {
 		this.#lc.info?.("starting change stream");
 		this.#forwarder.startProgressMonitor();
-		const lagReport = await this.#source.startLagReporter();
-		if (lagReport) this.#latestStatus.lagReport = lagReport;
+		const lagReportInit = await this.#source.startLagReporter();
+		if (lagReportInit) {
+			this.#latestStatus.lagReport = { nextSendTimeMs: lagReportInit.nextSendTimeMs };
+			this.#latestLagReportCommitTimeMs = lagReportInit.firstCommitTimeMs;
+		}
 		await this.#storer.assumeOwnership(this.#purgeLock);
 		this.#purgeLock = null;
 		const flushBytesThreshold = getDefaultHighWaterMark(false);
@@ -215,19 +279,22 @@ var ChangeStreamerImpl = class {
 			let watermark = null;
 			let unflushedBytes = 0;
 			try {
-				const { lastWatermark, backfillRequests } = await this.#storer.getStartStreamInitializationParameters();
+				const { lastWatermark, backfillRequests } = await this.#initializer.initialize();
+				this.#lastForwardedCommitWatermark = lastWatermark;
 				const stream = await this.#source.startStream(lastWatermark, backfillRequests);
 				this.#storer.run().catch((e) => stream.changes.cancel(e));
 				this.#stream = stream;
 				if (this.#state.resetBackoff() > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS) this.#replicationStatusPublisher.publish(this.#lc, "Replicating", `Replicating from ${lastWatermark}`);
 				watermark = null;
+				this.#acker.reset(stream.acks);
 				for await (const change of stream.changes) {
+					this.#acker.trackDownstream(change);
 					const [type, msg] = change;
 					switch (type) {
 						case "status":
-							if (msg.ack) this.#storer.status(change);
-							if (msg.lagReport) {
+							if (msg.lagReport && msg.lagReport.lastTimings.commitTimeMs >= this.#latestLagReportCommitTimeMs) {
 								this.#latestStatus.lagReport = msg.lagReport;
+								this.#latestLagReportCommitTimeMs = msg.lagReport.lastTimings.commitTimeMs;
 								this.#forwarder.sendStatus(this.#latestStatus);
 							}
 							continue;
@@ -247,15 +314,19 @@ var ChangeStreamerImpl = class {
 							break;
 					}
 					const json = this.#storer.store(watermark, change);
+					this.#changeLogWriter?.write(change, json);
 					const entry = [
 						watermark,
 						change[1].tag,
 						json
 					];
 					unflushedBytes += json.length;
-					if (unflushedBytes < flushBytesThreshold) this.#forwarder.forward(entry);
-					else {
+					if (unflushedBytes < flushBytesThreshold) {
+						this.#forwarder.forward(entry);
+						this.#recordForwardedTransactionBoundary(type, entry[0]);
+					} else {
 						const forwarded = this.#forwarder.forwardWithFlowControl(entry);
+						this.#recordForwardedTransactionBoundary(type, entry[0]);
 						await stream.changes.doneOr(forwarded);
 						unflushedBytes = 0;
 					}
@@ -272,11 +343,14 @@ var ChangeStreamerImpl = class {
 			if (watermark) {
 				this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);
 				this.#storer.abort();
+				this.#changeLogWriter?.abort();
+				this.#purgeScheduler?.onWriterIdle();
 				this.#forwarder.forward([
 					watermark,
 					"rollback",
 					ROLLBACK_JSON
 				]);
+				this.#recordForwardedTransactionBoundary("rollback", watermark);
 			}
 			await Promise.all([
 				this.#storer.stop(),
@@ -302,27 +376,88 @@ var ChangeStreamerImpl = class {
 			default: unreachable(tag);
 		}
 	}
-	subscribe(ctx) {
+	async subscribe(ctx) {
 		const { protocolVersion, id, mode, replicaVersion, watermark } = ctx;
 		if (mode === "serving") this.#serving.resolve();
-		const downstream = Subscription.create({ cleanup: () => this.#forwarder.remove(subscriber) });
-		const subscriber = new Subscriber(protocolVersion, id, watermark, downstream, () => this.#latestStatus);
+		let cleanupSubscriber = () => {};
+		const downstream = Subscription.create({ cleanup: () => cleanupSubscriber() });
+		const subscriber = new Subscriber(protocolVersion, id, watermark, downstream, () => this.#latestStatus, {});
+		const lc = this.#lc.withContext("subscriber", subscriber.id);
+		cleanupSubscriber = () => {
+			lc.info?.(`removing subscriber ${subscriber.id}`);
+			this.#forwarder.remove(subscriber);
+		};
 		if (replicaVersion !== this.#replicaVersion) {
-			this.#lc.warn?.(`rejecting subscriber at replica version ${replicaVersion}`);
+			lc.warn?.(`rejecting subscriber at replica version ${replicaVersion}`);
 			subscriber.close(1, `current replica version is ${this.#replicaVersion} (requested ${replicaVersion})`);
 		} else {
-			this.#lc.debug?.(`adding subscriber ${subscriber.id}`);
-			this.#forwarder.add(subscriber);
-			this.#storer.catchup(subscriber, mode);
+			lc.info?.(`adding subscriber ${subscriber.id}`);
+			const sqliteCatchup = this.#selectSQLiteCatchup(ctx);
+			if (sqliteCatchup) {
+				cleanupSubscriber = () => sqliteCatchup.remove(subscriber);
+				await sqliteCatchup.catchup(subscriber, () => this.#captureRequiredHead());
+			} else {
+				this.#forwarder.add(subscriber);
+				this.#storer.catchup(subscriber, mode);
+			}
+		}
+		this.#reservations?.close(ctx.taskID);
+		return downstream;
+	}
+	async startSnapshotReservation(taskID) {
+		if (!this.#reservations) throw new Error("backups are not configured");
+		const downstream = this.#reservations.open(taskID);
+		try {
+			await (this.#purgeScheduler?.pause(taskID) ?? promiseVoid);
+			await this.#confirmReservations();
+			return downstream;
+		} catch (e) {
+			downstream.cancel();
+			throw e;
+		}
+	}
+	trackBackupWatermark(watermark) {
+		this.#backupWatermark = watermark;
+		this.#acker.trackBackup(watermark);
+		this.#requestSQLitePurge("deferred");
+		this.#maybeSchedulePGPurge();
+		this.#maybeScheduleSQLitePurge();
+		this.#confirmReservations().catch((e) => this.#lc.warn?.(`error confirming snapshot reservation`, e));
+	}
+	async #confirmReservations() {
+		if (this.#backupWatermark && this.#reservations?.confirmationsRequired()) {
+			const { replicaVersion, minWatermark } = await this.#getChangeLogState();
+			if (minWatermark <= this.#backupWatermark) this.#reservations?.confirm(replicaVersion, this.#backupWatermark);
+			else this.#lc.error?.(`change-log minWatermark ${minWatermark} is later than backupWatermark ${this.#backupWatermark}. Delaying confirmation of snapshot reservation until next backup.`);
 		}
-		return Promise.resolve(downstream);
 	}
-	scheduleCleanup(watermark) {
-		const origSize = this.#initialWatermarks.size;
-		this.#initialWatermarks.add(watermark);
-		if (origSize === 0) this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);
+	#maybeSchedulePGPurge() {
+		const backupWatermark = this.#backupWatermark;
+		if (this.#pgPurgeScheduled || this.#pgPurgeRunning || backupWatermark === void 0 || this.#pgPurgedWatermark >= backupWatermark) return;
+		this.#pgPurgeScheduled = true;
+		this.#state.setTimeout(() => {
+			this.#pgPurgeScheduled = false;
+			this.#pgPurgeRunning = true;
+			return this.#purgePGChangeLog().finally(() => {
+				this.#pgPurgeRunning = false;
+				this.#maybeSchedulePGPurge();
+			});
+		}, CLEANUP_DELAY_MS);
 	}
-	async getChangeLogState() {
+	#maybeScheduleSQLitePurge() {
+		if (this.#sqlitePurgeScheduled || this.#sqlitePurgeRunning || this.#backupWatermark === void 0 || this.#sqlitePurgeContinuation === void 0) return;
+		const delay = this.#sqlitePurgeContinuation === "immediate" ? 0 : CLEANUP_DELAY_MS;
+		this.#sqlitePurgeScheduled = true;
+		this.#state.setTimeout(() => {
+			this.#sqlitePurgeScheduled = false;
+			this.#sqlitePurgeRunning = true;
+			return this.#purgeSQLiteChangeLog().finally(() => {
+				this.#sqlitePurgeRunning = false;
+				this.#maybeScheduleSQLitePurge();
+			});
+		}, delay);
+	}
+	async #getChangeLogState() {
 		const minWatermark = await this.#storer.getMinWatermarkForCatchup();
 		if (!minWatermark) this.#lc.warn?.(`Unexpected empty changeLog. Resync if "Local replica watermark" errors arise`);
 		return {
@@ -330,44 +465,176 @@ var ChangeStreamerImpl = class {
 			minWatermark: minWatermark ?? this.#replicaVersion
 		};
 	}
+	#getCleanupFloor() {
+		const backupWatermark = this.#backupWatermark;
+		assert(backupWatermark !== void 0, "cleanup cannot run without a backup watermark");
+		const current = [...this.#forwarder.getAcks(), ...this.#reservations?.getReservedWatermarks() ?? []];
+		return {
+			backupWatermark,
+			purgeWatermark: min(backupWatermark, ...current),
+			current
+		};
+	}
+	async #purgePGChangeLog() {
+		try {
+			const { backupWatermark, purgeWatermark, current } = this.#getCleanupFloor();
+			if (purgeWatermark < backupWatermark) {
+				if (this.#loggedBehindWatermark !== purgeWatermark) {
+					this.#loggedBehindWatermark = purgeWatermark;
+					this.#lc.info?.(`At least one client is behind backup ${backupWatermark}`, { watermarks: current });
+				}
+			} else this.#loggedBehindWatermark = void 0;
+			if (purgeWatermark <= this.#pgPurgedWatermark) return;
+			this.#lc.info?.(`Purging PG changes before ${purgeWatermark} ...`);
+			const start = performance.now();
+			const deleted = await this.#storer.purgeRecordsBefore(purgeWatermark);
+			const elapsed = (performance.now() - start).toFixed(2);
+			this.#lc.info?.(`Purged ${deleted} PG changes before ${purgeWatermark} (${elapsed} ms)`);
+			this.#pgPurgedWatermark = purgeWatermark;
+		} catch (e) {
+			this.#lc.warn?.(`error purging the PG change log`, e);
+		}
+	}
+	async #purgeSQLiteChangeLog() {
+		const scheduler = this.#purgeScheduler;
+		if (!scheduler) return;
+		try {
+			const { backupWatermark, purgeWatermark } = this.#getCleanupFloor();
+			this.#sqlitePurgeContinuation = void 0;
+			if (purgeWatermark < backupWatermark) this.#requestSQLitePurge("deferred");
+			const result = await scheduler.purge(purgeWatermark);
+			this.#requestSQLitePurge(result.continuation);
+		} catch (e) {
+			this.#requestSQLitePurge("deferred");
+			this.#lc.warn?.(`error purging the SQLite change log`, e);
+		}
+	}
+	#requestSQLitePurge(continuation) {
+		if (!this.#purgeScheduler || continuation === void 0) return;
+		if (continuation === "immediate" || this.#sqlitePurgeContinuation === void 0) this.#sqlitePurgeContinuation = continuation;
+	}
+	async stop(err) {
+		this.#state.stop(this.#lc, err);
+		this.#stream?.changes.cancel();
+		this.#purgeScheduler?.stop();
+		this.#sqliteCatchup?.close();
+		this.#changeLogWriter?.close();
+		await Promise.allSettled([this.#storer.stop(), this.#source.stop()]);
+	}
+	#recordForwardedTransactionBoundary(type, watermark) {
+		switch (type) {
+			case "begin":
+				assert(this.#currentTransactionCompletion === void 0, "forwarded begin while a transaction is already in progress");
+				this.#currentTransactionCompletion = resolver();
+				this.#transactionForwardStartedAt = performance.now();
+				break;
+			case "commit": {
+				const completion = this.#currentTransactionCompletion;
+				assert(completion, "forwarded commit without a pending transaction");
+				this.#recordTransactionForwardDuration();
+				this.#lastForwardedCommitWatermark = watermark;
+				this.#currentTransactionCompletion = void 0;
+				completion.resolve({
+					kind: "committed",
+					watermark
+				});
+				break;
+			}
+			case "rollback": {
+				const completion = this.#currentTransactionCompletion;
+				assert(completion, "forwarded rollback without a pending transaction");
+				const committed = this.#lastForwardedCommitWatermark;
+				assert(committed, "last forwarded commit watermark is not initialized");
+				this.#transactionForwardStartedAt = void 0;
+				this.#currentTransactionCompletion = void 0;
+				completion.resolve({
+					kind: "rolled-back",
+					watermark: committed
+				});
+				break;
+			}
+		}
+	}
+	#recordTransactionForwardDuration() {
+		const startedAt = this.#transactionForwardStartedAt;
+		this.#transactionForwardStartedAt = void 0;
+		if (startedAt !== void 0) this.#transactionForwardDuration.recordMs(performance.now() - startedAt, { sqlite_change_log: this.#changeLogWriter?.enabled ? "on" : "off" });
+	}
 	/**
-	* Makes a best effort to purge the change log. In the event of a database
-	* error, exceptions will be logged and swallowed, so this method is safe
-	* to run in a timeout.
+	* Closes and forgets the cached catchup coordinator, so that a later
+	* subscription re-opens the log from scratch rather than reading a handle
+	* whose file is gone.
 	*/
-	async #purgeOldChanges() {
-		const initial = [...this.#initialWatermarks];
-		if (initial.length === 0) {
-			this.#lc.warn?.("No initial watermarks to check for cleanup");
+	#closeSQLiteCatchup() {
+		this.#sqliteCatchup?.close();
+		this.#sqliteCatchup = void 0;
+	}
+	#captureRequiredHead() {
+		const committed = this.#lastForwardedCommitWatermark;
+		assert(committed, "last forwarded commit watermark is not initialized");
+		return this.#currentTransactionCompletion?.promise.then((completion) => completion.watermark) ?? committed;
+	}
+	#selectSQLiteCatchup(ctx) {
+		const opts = this.#sqliteCatchupOptions;
+		if (this.#lastForwardedCommitWatermark === void 0 || !opts) return;
+		if (ctx.mode !== "serving") {
+			this.#lc.info?.(`not serving backup subscriber ${ctx.id} from SQLite catchup`);
 			return;
 		}
-		const current = [...this.#forwarder.getAcks()];
-		if (current.length === 0) {
-			this.#lc.warn?.("No subscribers to confirm cleanup");
+		if (ctx.logsChangeStream) {
+			this.#lc.warn?.(`not serving legacy change-log writer ${ctx.id} from SQLite catchup`);
 			return;
 		}
+		if (!opts.shouldUse?.(ctx)) return;
+		return this.#sqliteCatchup ?? this.#openSQLiteCatchup(opts, ctx);
+	}
+	/**
+	* Opens the change log and, if it can serve, wraps it in the coordinator that
+	* subsequent subscriptions reuse.
+	*
+	* The log may not exist yet -- the writer creates it at the stream loop's
+	* first reconcile, and deletes it when it fails soft -- or may exist without
+	* content, or may be unreadable. None of those can be allowed to fail a
+	* subscription:
+	* this is the last point at which PG catchup is still available -- past
+	* `Forwarder.add()` the subscriber is committed to SQLite -- so each declines
+	* here instead. Neither the failure nor the reader is retained, so a later
+	* subscription retries from scratch.
+	*/
+	#openSQLiteCatchup(opts, ctx) {
+		let reader;
 		try {
-			const earliestInitial = min(...initial);
-			const earliestCurrent = min(...current);
-			if (earliestCurrent < earliestInitial) this.#lc.info?.(`At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`);
-			else {
-				this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);
-				const start = performance.now();
-				const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);
-				const elapsed = (performance.now() - start).toFixed(2);
-				this.#lc.info?.(`Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`);
-				this.#initialWatermarks.delete(earliestInitial);
+			reader = new SQLiteChangeLogReader(this.#lc, opts.changeLogFile);
+			if (reader.plan(ctx.watermark).kind === "not-ready") {
+				reader.close();
+				this.#declineSQLiteCatchup(ctx, opts, `${opts.changeLogFile} has no changes to serve yet`);
+				return;
 			}
 		} catch (e) {
-			this.#lc.warn?.(`error purging change log`, e);
-		} finally {
-			if (this.#initialWatermarks.size) this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);
+			reader?.close();
+			this.#declineSQLiteCatchup(ctx, opts, `cannot read ${opts.changeLogFile}`, e);
+			return;
 		}
+		this.#changeLogUnavailableSince = void 0;
+		this.#sqliteCatchup = new SQLiteChangeLogCatchup(this.#lc, this.#forwarder, reader, {
+			batchSize: opts.readBatchRows,
+			barrierTimeoutMs: opts.barrierTimeoutMs,
+			barrierPollIntervalMs: opts.barrierPollIntervalMs,
+			cleanupGuard: opts.cleanupGuard
+		});
+		return this.#sqliteCatchup;
 	}
-	async stop(err) {
-		this.#state.stop(this.#lc, err);
-		this.#stream?.changes.cancel();
-		await Promise.allSettled([this.#storer.stop(), this.#source.stop()]);
+	/**
+	* Reports falling back to PG catchup, tracking how long the log has been
+	* unavailable so that a subscription that merely arrived before the writer's
+	* first reconcile does not look like an incident.
+	*/
+	#declineSQLiteCatchup(ctx, opts, reason, error) {
+		const now = Date.now();
+		this.#changeLogUnavailableSince ??= now;
+		const unavailableMs = now - this.#changeLogUnavailableSince;
+		const threshold = opts.notReadyWarnThresholdMs ?? DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS;
+		this.#lc[unavailableMs >= threshold ? "warn" : "debug"]?.(`serving ${ctx.id} from PG catchup: ${reason} (unavailable for ${unavailableMs} ms)`, ...error === void 0 ? [] : [error]);
 	}
 };
 var CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;