@rocicorp/zero 1.12.0-canary.0 → 1.12.0-canary.10

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -1,5 +1,6 @@
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { resolver } from "../../../../shared/src/resolver.js";
 import { must } from "../../../../shared/src/must.js";
 import { promiseOrAbort } from "../../../../shared/src/promise-race.js";
 import "../change-source/protocol/current/downstream.js";
@@ -27,7 +28,6 @@ import { SQLiteChangeLogWriter } from "./sqlite-change-log-writer.js";
 import { Storer } from "./storer.js";
 import { Subscriber } from "./subscriber.js";
 import { UpstreamAcker } from "./upstream-acker.js";
-import { resolver } from "@rocicorp/resolver";
 import { defu } from "defu";
 import postgres from "postgres";
 import { getDefaultHighWaterMark } from "node:stream";
@@ -36,12 +36,11 @@ import { getDefaultHighWaterMark } from "node:stream";
 * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.
 */
 async function initializeStreamer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, changeSource, replicationStatusPublisher, subscriptionState, backupConfig, purgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
-	if (opts.pgChangeLogEnabled) await checkReplicationConfig(lc, changeDB, subscriptionState, shard, autoReset, opts.pgChangeLogBehindSlot);
-	const changeDBProvider = (applicationName, max) => postgres(defu({
+	if (changeDB) await checkReplicationConfig(lc, changeDB, subscriptionState, shard, autoReset, opts.pgChangeLogBehindSlot);
+	return new ChangeStreamerImpl(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB ? (applicationName, max) => postgres(defu({
 		max,
 		connection: { ["application_name"]: applicationName }
-	}, changeDB.options));
-	return new ChangeStreamerImpl(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, subscriptionState, changeSource, replicationStatusPublisher, backupConfig, purgeLock, autoReset, opts, setTimeoutFn);
+	}, changeDB.options)) : null, subscriptionState, changeSource, replicationStatusPublisher, backupConfig, purgeLock, autoReset, opts, setTimeoutFn);
 }
 var REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5e3;
 var DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS = 6e4;
@@ -190,8 +189,8 @@ var ChangeStreamerImpl = class {
 	#changeDBProvider;
 	#replicaVersion;
 	#source;
+	/** The PG change log's storer, or `null` when the PG change log is disabled. */
 	#storer;
-	#pgChangeLogEnabled;
 	#pgChangeLogBehindSlot;
 	#forwarder;
 	#reservations;
@@ -203,6 +202,7 @@ var ChangeStreamerImpl = class {
 	#purgeScheduler;
 	#comparator;
 	#acker;
+	#backupMonitor;
 	#initializer;
 	#readRouter;
 	#autoReset;
@@ -217,7 +217,7 @@ var ChangeStreamerImpl = class {
 	#reservationDemotions = getOrCreateCounter("replication", "sqlite_change_log.reservation_demotions", "Snapshot reservations demoted from SQLite to PG because the change log could not cover the backup being restored. With PG retired these followers have no fallback, so this is the rate at which one would instead have to wait for a later backup.");
 	#reservationConfirmDelays = getOrCreateCounter("replication", "sqlite_change_log.reservation_confirm_delays", "Snapshot reservations whose confirmation was deferred because the selected source's change-log minimum was later than the backup watermark. Counted once per reservation, by that source.");
 	#latestLagReportCommitTimeMs = 0;
-	#lagReportingDisabled = resolver();
+	#lagReports = resolver();
 	#backupWatermark;
 	#pgPurgedWatermark = "";
 	/**
@@ -248,14 +248,13 @@ var ChangeStreamerImpl = class {
 		const { replicaVersion } = subscriptionState;
 		this.#replicaVersion = replicaVersion;
 		this.#source = source;
-		this.#pgChangeLogEnabled = opts.pgChangeLogEnabled;
 		this.#pgChangeLogBehindSlot = opts.pgChangeLogBehindSlot;
+		this.#storer = changeDBProvider ? new Storer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, subscriptionState, (consumed) => this.#acker.trackPgChangeLog(consumed[2].watermark), (err) => this.stop(err), opts) : null;
 		if (!this.#pgChangeLogEnabled) {
-			assert(opts.sqliteChangeLogWriter && opts.sqliteChangeLogPurge && opts.sqliteCatchup && opts.sqliteChangeLogServe?.readPercent === 100 && opts.sqliteChangeLogServe.coldReadPercent === 100 && backupConfig?.litestreamVersion === "v5", "disabling the PG change log requires SQLite serving at 100 percent and a v5 backup");
+			assert(opts.sqliteChangeLogWriter && opts.sqliteChangeLogPurge && opts.sqliteCatchup && opts.sqliteChangeLogServe?.readPercent === 100 && opts.sqliteChangeLogServe.coldReadPercent === 100, "disabling the PG change log requires SQLite serving at 100 percent and a v5 backup");
 			assert(opts.sqliteChangeLogCompare === void 0, "SQLite change-log comparison requires the PG change log");
 		}
 		assert(!(this.#pgChangeLogEnabled && opts.pgChangeLogBehindSlot && !backupConfig?.replicaFile), "the PG change log is behind the replication slot, but there is no restored replica to re-initialize it from");
-		this.#storer = new Storer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, subscriptionState, (consumed) => this.#acker.trackPgChangeLog(consumed[2].watermark), (err) => this.stop(err), opts);
 		this.#forwarder = new Forwarder(lc, {
 			flowControlConsensusTimeoutProportion: opts.flowControlConsensusTimeoutProportion,
 			flowControlSlowSubscriberGracePeriodMs: opts.flowControlSlowSubscriberGracePeriodMs
@@ -305,7 +304,7 @@ var ChangeStreamerImpl = class {
 		this.#purgeScheduler = opts.sqliteChangeLogPurge && this.#changeLogWriter ? new SQLiteChangeLogPurgeScheduler(lc, () => this.#changeLogWriter?.connection, () => this.#forwarder.getAcks(), opts.sqliteChangeLogPurge) : void 0;
 		this.#acker = new UpstreamAcker({
 			trackPgChangeLog: this.#pgChangeLogEnabled,
-			trackBackup: backupConfig?.litestreamVersion === "v5"
+			trackBackup: backupConfig ? backupConfig.litestreamVersion === "v5" : !this.#pgChangeLogEnabled
 		});
 		const replicaFileForInitialization = opts.sqliteChangeLogCompare?.replicaFile ?? (!this.#pgChangeLogEnabled ? opts.sqliteChangeLogWriter?.replicaFile : void 0);
 		const replicaFileForReinitialization = this.#pgChangeLogEnabled && opts.pgChangeLogBehindSlot ? backupConfig?.replicaFile : void 0;
@@ -315,39 +314,52 @@ var ChangeStreamerImpl = class {
 			initFromPgChangeLog: this.#pgChangeLogEnabled,
 			initFromReplica: replicaFileForInitialization !== void 0
 		}, {
-			pgChangeLog: () => this.#storer.getStartStreamInitializationParameters(),
+			pgChangeLog: () => must(this.#storer).getStartStreamInitializationParameters(),
 			replica: () => must(replicaSource)(),
 			reconcileChangeLog: (resumeFrom, seed) => resumeFrom ? this.#changeLogWriter?.reconcile(resumeFrom) : this.#changeLogWriter?.reconcileFromLog(seed),
 			changeLog: () => this.#changeLogWriter?.connection,
-			reinitializePgChangeLog: replicaFileForReinitialization ? (params) => this.#storer.reinitializeFromReplica(params) : void 0
+			reinitializePgChangeLog: replicaFileForReinitialization ? (params) => must(this.#storer).reinitializeFromReplica(params) : void 0
 		});
 		this.#sqliteCatchupOptions = opts.sqliteCatchup ? {
 			...opts.sqliteCatchup,
 			cleanupGuard: opts.sqliteCatchup.cleanupGuard ?? this.#purgeScheduler?.cleanupGuard
 		} : void 0;
-		this.#comparator = this.#pgChangeLogEnabled && opts.sqliteChangeLogCompare && opts.sqliteChangeLogWriter && opts.sqliteCatchup ? new SQLiteChangeLogComparator(lc, shard, opts.sqliteCatchup.changeLogFile, opts.sqliteChangeLogWriter.identity, this.#storer, {
+		this.#comparator = this.#storer && opts.sqliteChangeLogCompare && opts.sqliteChangeLogWriter && opts.sqliteCatchup ? new SQLiteChangeLogComparator(lc, shard, opts.sqliteCatchup.changeLogFile, opts.sqliteChangeLogWriter.identity, this.#storer, {
 			setTimeoutFn,
 			...opts.sqliteChangeLogCompare
 		}) : void 0;
 		this.#purgeLock = initialPurgeLock;
 		this.#autoReset = autoReset;
 		this.#state = new RunningState(this.id, void 0, setTimeoutFn);
+		this.#backupMonitor = opts.backupMonitor;
+		if (this.#backupMonitor) this.#trackBackups(this.#backupMonitor);
+	}
+	async #trackBackups(monitor) {
+		for await (const { watermark } of monitor.backups()) this.#trackBackupWatermark(watermark);
+	}
+	/**
+	* Whether the legacy Postgres change log is enabled, which is determined
+	* solely by the presence of its {@link Storer} (i.e. of a change DB).
+	*/
+	get #pgChangeLogEnabled() {
+		return this.#storer !== null;
 	}
 	async run() {
 		this.#lc.info?.("starting change stream");
 		this.#forwarder.startProgressMonitor();
 		const lagReportInit = await this.#source.startLagReporter();
-		this.#lagReportingDisabled.resolve(!lagReportInit);
+		this.#lagReports.resolve(lagReportInit ? { request: () => this.#source.requestLagReport?.() } : void 0);
 		if (lagReportInit) this.#latestLagReportCommitTimeMs = lagReportInit.firstCommitTimeMs;
-		let takeoverPending = this.#pgChangeLogEnabled;
+		const storer = this.#storer;
+		let takeoverPending = storer !== null;
 		const flushBytesThreshold = getDefaultHighWaterMark(false);
 		while (this.#state.shouldRun()) {
 			let err;
 			let watermark = null;
 			let unflushedBytes = 0;
 			try {
-				if (takeoverPending) {
-					await this.#storer.assumeOwnershipAtHead({
+				if (storer && takeoverPending) {
+					await storer.assumeOwnershipAtHead({
 						purgeLock: this.#purgeLock,
 						requestRestart: (reason) => this.#backfills.requestRestart(reason),
 						autoReset: this.#autoReset,
@@ -362,17 +374,20 @@ var ChangeStreamerImpl = class {
 				const stream = await this.#source.startStream(lastWatermark, backfillRequests);
 				this.#stream = stream;
 				if (!this.#state.shouldRun()) break;
-				if (this.#pgChangeLogEnabled) this.#storer.run().catch((e) => stream.changes.cancel(e));
+				storer?.run().catch((e) => stream.changes.cancel(e));
 				if (this.#state.resetBackoff() > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS) this.#replicationStatusPublisher.publish(this.#lc, "Replicating", `Replicating from ${lastWatermark}`);
 				watermark = null;
 				this.#acker.reset(stream.acks, lastWatermark);
-				for await (const change of stream.changes) {
+				streamLoop: for await (const change of stream.changes) {
 					this.#acker.trackDownstream(change);
 					const [type, msg] = change;
-					if (type === "status" && watermark === null && this.#mayRestart()) break;
 					switch (type) {
 						case "status":
-							if (watermark === null) this.#storer.status(change[2].watermark);
+							if (msg.syncBackupThrough) this.#backupMonitor?.requestSync(msg.syncBackupThrough);
+							if (watermark === null) {
+								if (this.#mayRestart()) break streamLoop;
+								storer?.status(change[2].watermark);
+							}
 							if (msg.lagReport && msg.lagReport.lastTimings.commitTimeMs >= this.#latestLagReportCommitTimeMs) {
 								this.#latestLagReportCommitTimeMs = msg.lagReport.lastTimings.commitTimeMs;
 								this.#forwarder.sendStatus({
@@ -394,12 +409,11 @@ var ChangeStreamerImpl = class {
 						default:
 							if (type === "data") this.#changeCounter.add(1);
 							if (watermark === null) throw new UnrecoverableError(`${type} change (${msg.tag}) received before 'begin' message`);
-							break;
 					}
 					this.#backfills.track(change);
 					const serialized = serializeChangeStreamDataWithChange(change);
 					const { json } = serialized;
-					if (this.#pgChangeLogEnabled) this.#storer.store(watermark, change, serialized);
+					storer?.store(watermark, change, serialized);
 					this.#changeLogWriter?.write(change, json, serialized.change);
 					const entry = [
 						watermark,
@@ -416,7 +430,7 @@ var ChangeStreamerImpl = class {
 						await promiseOrAbort(flowControl, stream.changes.signal, this.#state.signal);
 						unflushedBytes = 0;
 					}
-					const readyForMore = this.#pgChangeLogEnabled ? this.#storer.readyForMore() : void 0;
+					const readyForMore = storer?.readyForMore();
 					if (readyForMore) await promiseOrAbort(readyForMore, stream.changes.signal, this.#state.signal);
 					if (type === "commit" && this.#mayRestart()) break;
 				}
@@ -428,7 +442,7 @@ var ChangeStreamerImpl = class {
 			}
 			if (watermark) {
 				this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);
-				if (this.#pgChangeLogEnabled) this.#storer.abort();
+				storer?.abort();
 				this.#changeLogWriter?.abort();
 				this.#purgeScheduler?.onWriterIdle();
 				this.#forwarder.forward([
@@ -441,9 +455,9 @@ var ChangeStreamerImpl = class {
 			}
 			if (this.#backfills.restartReason !== null && err === void 0) {
 				this.#backfills.recordRestart();
-				await this.#storer.stop();
+				await storer?.stop();
 			} else await Promise.all([
-				this.#storer.stop(),
+				storer?.stop(),
 				this.#state.backoff(this.#lc, err),
 				this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS ? publishCriticalEvent(this.#lc, replicationStatusError(this.#lc, "Replicating", err)) : promiseVoid
 			]);
@@ -456,7 +470,7 @@ var ChangeStreamerImpl = class {
 		const { tag } = msg;
 		switch (tag) {
 			case "reset-required":
-				if (this.#pgChangeLogEnabled) await markResetRequired(this.#changeDBProvider("change-streamer-reset", 1), this.#shard);
+				if (this.#changeDBProvider) await markResetRequired(this.#changeDBProvider("change-streamer-reset", 1), this.#shard);
 				await publishReplicationError(this.#lc, "Replicating", msg.message ?? "Resync required", msg.errorDetails);
 				if (this.#autoReset) {
 					this.#lc.warn?.("shutting down for auto-reset");
@@ -473,7 +487,7 @@ var ChangeStreamerImpl = class {
 		const lc = this.#lc.withContext("subscriber", id);
 		const subscriber = new Subscriber(protocolVersion, id, mode, watermark, downstream, {
 			wsBatched,
-			lagReportingDisabled: this.#lagReportingDisabled.promise,
+			lagReports: this.#lagReports.promise,
 			drainable: mode === "serving" && ctx.drainable,
 			...this.#backfills.subscriberOptions(lc, ctx)
 		});
@@ -482,27 +496,28 @@ var ChangeStreamerImpl = class {
 			if (this.#drainTracker.draining) {
 				lc.warn?.(`rejecting subscriber ${id}: this replication-manager is draining`);
 				subscriber.close();
-			} else if (replicaVersion !== this.#replicaVersion) if (ctx.switchover && mode === "serving" && this.#restoreAdmission && replicaVersion < this.#replicaVersion) {
-				lc.info?.(`holding switchover subscriber ${subscriber.id} at replica version ${replicaVersion} for permission to restore ${this.#replicaVersion}`);
-				const ticket = this.#restoreAdmission.request({
-					onQueued: (position) => subscriber.sendControlStatus({
-						tag: "status",
-						restore: {
-							state: "queued",
-							position
-						}
-					}),
-					onGranted: () => subscriber.sendControlStatus({
-						tag: "status",
-						restore: { state: "granted" }
-					})
-				});
-				cleanupSubscriber = () => ticket.release();
+			} else if (replicaVersion !== this.#replicaVersion) {
+				if (ctx.switchover && mode === "serving" && this.#restoreAdmission && replicaVersion < this.#replicaVersion) {
+					lc.info?.(`holding switchover subscriber ${subscriber.id} at replica version ${replicaVersion} for permission to restore ${this.#replicaVersion}`);
+					const ticket = this.#restoreAdmission.request({
+						onQueued: (position) => subscriber.sendControlStatus({
+							tag: "status",
+							restore: {
+								state: "queued",
+								position
+							}
+						}),
+						onGranted: () => subscriber.sendControlStatus({
+							tag: "status",
+							restore: { state: "granted" }
+						})
+					});
+					cleanupSubscriber = () => ticket.release();
+				} else {
+					lc.warn?.(`rejecting subscriber at replica version ${replicaVersion}`);
+					subscriber.close(1, `current replica version is ${this.#replicaVersion} (requested ${replicaVersion})`, { replicaVersion: this.#replicaVersion });
+				}
 			} else {
-				lc.warn?.(`rejecting subscriber at replica version ${replicaVersion}`);
-				subscriber.close(1, `current replica version is ${this.#replicaVersion} (requested ${replicaVersion})`, { replicaVersion: this.#replicaVersion });
-			}
-			else {
 				lc.info?.(`adding subscriber ${subscriber.id}`);
 				this.#drainTracker.register(subscriber, downstream);
 				const replacement = this.#restoreAdmission?.reservationSubscribed(ctx.taskID);
@@ -533,10 +548,10 @@ var ChangeStreamerImpl = class {
 			this.#forwarder.remove(subscriber);
 		};
 		const catchupFromPG = () => {
-			assert(this.#pgChangeLogEnabled, "cannot catch up from a disabled PG change log");
+			const storer = must(this.#storer, "cannot catch up from a disabled PG change log");
 			try {
 				this.#forwarder.add(subscriber);
-				this.#storer.catchup(subscriber, ctx.mode);
+				storer.catchup(subscriber, ctx.mode);
 			} catch (e) {
 				removeFromForwarder();
 				throw e;
@@ -610,7 +625,17 @@ var ChangeStreamerImpl = class {
 	drain() {
 		return this.#drainTracker.drain();
 	}
-	trackBackupWatermark(watermark) {
+	/**
+	* Tracks the watermark up to which the replica has been backed up. This
+	* serves two purposes:
+	* - It serves as the maximum watermark up to which the change-log can be
+	*   purged. Note that the change-streamer also takes into account the
+	*   positions of its subscribers and snapshot reservations when purging
+	*   changes in the change-log.
+	* - In RMv2, this ACK's the upstream change-source to allow its buffer of
+	*   changes (e.g. the replication_slot) to advance.
+	*/
+	#trackBackupWatermark(watermark) {
 		this.#backupWatermark = watermark;
 		this.#acker.trackBackup(watermark);
 		this.#requestSQLitePurge("deferred");
@@ -683,7 +708,7 @@ var ChangeStreamerImpl = class {
 		}, delay);
 	}
 	async #getChangeLogState() {
-		const minWatermark = await this.#storer.getMinWatermarkForCatchup();
+		const minWatermark = await must(this.#storer, "the PG change log is disabled").getMinWatermarkForCatchup();
 		if (!minWatermark) this.#lc.warn?.(`Unexpected empty changeLog. Resync if "Local replica watermark" errors arise`);
 		return {
 			replicaVersion: this.#replicaVersion,
@@ -701,7 +726,8 @@ var ChangeStreamerImpl = class {
 		};
 	}
 	async #purgePGChangeLog() {
-		if (this.#storer.ownershipPending) return;
+		const storer = this.#storer;
+		if (!storer || storer.ownershipPending) return;
 		try {
 			const { backupWatermark, purgeWatermark, current } = this.#getCleanupFloor();
 			if (purgeWatermark < backupWatermark) {
@@ -713,7 +739,7 @@ var ChangeStreamerImpl = class {
 			if (purgeWatermark <= this.#pgPurgedWatermark) return;
 			this.#lc.info?.(`Purging PG changes before ${purgeWatermark} ...`);
 			const start = performance.now();
-			const deleted = await this.#storer.purgeRecordsBefore(purgeWatermark);
+			const deleted = await storer.purgeRecordsBefore(purgeWatermark);
 			const elapsed = (performance.now() - start).toFixed(2);
 			this.#lc.info?.(`Purged ${deleted} PG changes before ${purgeWatermark} (${elapsed} ms)`);
 			this.#pgPurgedWatermark = purgeWatermark;
@@ -747,18 +773,18 @@ var ChangeStreamerImpl = class {
 	* anyway (see Storer.assumeOwnershipAtHead()).
 	*/
 	#mayRestart() {
-		return this.#backfills.restartReason !== null && !this.#storer.ownershipPending;
+		return this.#backfills.restartReason !== null && !this.#storer?.ownershipPending;
 	}
 	async stop(err) {
 		this.#state.stop(this.#lc, err);
 		this.#drainTracker.stop();
-		this.#lagReportingDisabled.resolve(false);
+		this.#lagReports.resolve({ request: () => {} });
 		this.#stream?.changes.cancel();
 		this.#purgeScheduler?.stop();
 		this.#comparator?.stop();
 		this.#sqliteCatchup?.close();
 		this.#changeLogWriter?.close();
-		await Promise.allSettled([this.#storer.stop(), this.#source.stop()]);
+		await Promise.allSettled([this.#storer?.stop(), this.#source.stop()]);
 	}
 	#recordForwardedTransactionBoundary(type, watermark) {
 		switch (type) {