@rocicorp/zero 1.12.0-canary.0 → 1.12.0-canary.1

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -36,12 +36,11 @@ import { getDefaultHighWaterMark } from "node:stream";
 * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.
 */
 async function initializeStreamer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, changeSource, replicationStatusPublisher, subscriptionState, backupConfig, purgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
-	if (opts.pgChangeLogEnabled) await checkReplicationConfig(lc, changeDB, subscriptionState, shard, autoReset, opts.pgChangeLogBehindSlot);
-	const changeDBProvider = (applicationName, max) => postgres(defu({
+	if (changeDB) await checkReplicationConfig(lc, changeDB, subscriptionState, shard, autoReset, opts.pgChangeLogBehindSlot);
+	return new ChangeStreamerImpl(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB ? (applicationName, max) => postgres(defu({
 		max,
 		connection: { ["application_name"]: applicationName }
-	}, changeDB.options));
-	return new ChangeStreamerImpl(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, subscriptionState, changeSource, replicationStatusPublisher, backupConfig, purgeLock, autoReset, opts, setTimeoutFn);
+	}, changeDB.options)) : null, subscriptionState, changeSource, replicationStatusPublisher, backupConfig, purgeLock, autoReset, opts, setTimeoutFn);
 }
 var REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5e3;
 var DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS = 6e4;
@@ -190,8 +189,8 @@ var ChangeStreamerImpl = class {
 	#changeDBProvider;
 	#replicaVersion;
 	#source;
+	/** The PG change log's storer, or `null` when the PG change log is disabled. */
 	#storer;
-	#pgChangeLogEnabled;
 	#pgChangeLogBehindSlot;
 	#forwarder;
 	#reservations;
@@ -248,14 +247,13 @@ var ChangeStreamerImpl = class {
 		const { replicaVersion } = subscriptionState;
 		this.#replicaVersion = replicaVersion;
 		this.#source = source;
-		this.#pgChangeLogEnabled = opts.pgChangeLogEnabled;
 		this.#pgChangeLogBehindSlot = opts.pgChangeLogBehindSlot;
+		this.#storer = changeDBProvider ? new Storer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, subscriptionState, (consumed) => this.#acker.trackPgChangeLog(consumed[2].watermark), (err) => this.stop(err), opts) : null;
 		if (!this.#pgChangeLogEnabled) {
-			assert(opts.sqliteChangeLogWriter && opts.sqliteChangeLogPurge && opts.sqliteCatchup && opts.sqliteChangeLogServe?.readPercent === 100 && opts.sqliteChangeLogServe.coldReadPercent === 100 && backupConfig?.litestreamVersion === "v5", "disabling the PG change log requires SQLite serving at 100 percent and a v5 backup");
+			assert(opts.sqliteChangeLogWriter && opts.sqliteChangeLogPurge && opts.sqliteCatchup && opts.sqliteChangeLogServe?.readPercent === 100 && opts.sqliteChangeLogServe.coldReadPercent === 100, "disabling the PG change log requires SQLite serving at 100 percent and a v5 backup");
 			assert(opts.sqliteChangeLogCompare === void 0, "SQLite change-log comparison requires the PG change log");
 		}
 		assert(!(this.#pgChangeLogEnabled && opts.pgChangeLogBehindSlot && !backupConfig?.replicaFile), "the PG change log is behind the replication slot, but there is no restored replica to re-initialize it from");
-		this.#storer = new Storer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, subscriptionState, (consumed) => this.#acker.trackPgChangeLog(consumed[2].watermark), (err) => this.stop(err), opts);
 		this.#forwarder = new Forwarder(lc, {
 			flowControlConsensusTimeoutProportion: opts.flowControlConsensusTimeoutProportion,
 			flowControlSlowSubscriberGracePeriodMs: opts.flowControlSlowSubscriberGracePeriodMs
@@ -305,7 +303,7 @@ var ChangeStreamerImpl = class {
 		this.#purgeScheduler = opts.sqliteChangeLogPurge && this.#changeLogWriter ? new SQLiteChangeLogPurgeScheduler(lc, () => this.#changeLogWriter?.connection, () => this.#forwarder.getAcks(), opts.sqliteChangeLogPurge) : void 0;
 		this.#acker = new UpstreamAcker({
 			trackPgChangeLog: this.#pgChangeLogEnabled,
-			trackBackup: backupConfig?.litestreamVersion === "v5"
+			trackBackup: backupConfig ? backupConfig.litestreamVersion === "v5" : !this.#pgChangeLogEnabled
 		});
 		const replicaFileForInitialization = opts.sqliteChangeLogCompare?.replicaFile ?? (!this.#pgChangeLogEnabled ? opts.sqliteChangeLogWriter?.replicaFile : void 0);
 		const replicaFileForReinitialization = this.#pgChangeLogEnabled && opts.pgChangeLogBehindSlot ? backupConfig?.replicaFile : void 0;
@@ -315,17 +313,17 @@ var ChangeStreamerImpl = class {
 			initFromPgChangeLog: this.#pgChangeLogEnabled,
 			initFromReplica: replicaFileForInitialization !== void 0
 		}, {
-			pgChangeLog: () => this.#storer.getStartStreamInitializationParameters(),
+			pgChangeLog: () => must(this.#storer).getStartStreamInitializationParameters(),
 			replica: () => must(replicaSource)(),
 			reconcileChangeLog: (resumeFrom, seed) => resumeFrom ? this.#changeLogWriter?.reconcile(resumeFrom) : this.#changeLogWriter?.reconcileFromLog(seed),
 			changeLog: () => this.#changeLogWriter?.connection,
-			reinitializePgChangeLog: replicaFileForReinitialization ? (params) => this.#storer.reinitializeFromReplica(params) : void 0
+			reinitializePgChangeLog: replicaFileForReinitialization ? (params) => must(this.#storer).reinitializeFromReplica(params) : void 0
 		});
 		this.#sqliteCatchupOptions = opts.sqliteCatchup ? {
 			...opts.sqliteCatchup,
 			cleanupGuard: opts.sqliteCatchup.cleanupGuard ?? this.#purgeScheduler?.cleanupGuard
 		} : void 0;
-		this.#comparator = this.#pgChangeLogEnabled && opts.sqliteChangeLogCompare && opts.sqliteChangeLogWriter && opts.sqliteCatchup ? new SQLiteChangeLogComparator(lc, shard, opts.sqliteCatchup.changeLogFile, opts.sqliteChangeLogWriter.identity, this.#storer, {
+		this.#comparator = this.#storer && opts.sqliteChangeLogCompare && opts.sqliteChangeLogWriter && opts.sqliteCatchup ? new SQLiteChangeLogComparator(lc, shard, opts.sqliteCatchup.changeLogFile, opts.sqliteChangeLogWriter.identity, this.#storer, {
 			setTimeoutFn,
 			...opts.sqliteChangeLogCompare
 		}) : void 0;
@@ -333,21 +331,29 @@ var ChangeStreamerImpl = class {
 		this.#autoReset = autoReset;
 		this.#state = new RunningState(this.id, void 0, setTimeoutFn);
 	}
+	/**
+	* Whether the legacy Postgres change log is enabled, which is determined
+	* solely by the presence of its {@link Storer} (i.e. of a change DB).
+	*/
+	get #pgChangeLogEnabled() {
+		return this.#storer !== null;
+	}
 	async run() {
 		this.#lc.info?.("starting change stream");
 		this.#forwarder.startProgressMonitor();
 		const lagReportInit = await this.#source.startLagReporter();
 		this.#lagReportingDisabled.resolve(!lagReportInit);
 		if (lagReportInit) this.#latestLagReportCommitTimeMs = lagReportInit.firstCommitTimeMs;
-		let takeoverPending = this.#pgChangeLogEnabled;
+		const storer = this.#storer;
+		let takeoverPending = storer !== null;
 		const flushBytesThreshold = getDefaultHighWaterMark(false);
 		while (this.#state.shouldRun()) {
 			let err;
 			let watermark = null;
 			let unflushedBytes = 0;
 			try {
-				if (takeoverPending) {
-					await this.#storer.assumeOwnershipAtHead({
+				if (storer && takeoverPending) {
+					await storer.assumeOwnershipAtHead({
 						purgeLock: this.#purgeLock,
 						requestRestart: (reason) => this.#backfills.requestRestart(reason),
 						autoReset: this.#autoReset,
@@ -362,7 +368,7 @@ var ChangeStreamerImpl = class {
 				const stream = await this.#source.startStream(lastWatermark, backfillRequests);
 				this.#stream = stream;
 				if (!this.#state.shouldRun()) break;
-				if (this.#pgChangeLogEnabled) this.#storer.run().catch((e) => stream.changes.cancel(e));
+				storer?.run().catch((e) => stream.changes.cancel(e));
 				if (this.#state.resetBackoff() > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS) this.#replicationStatusPublisher.publish(this.#lc, "Replicating", `Replicating from ${lastWatermark}`);
 				watermark = null;
 				this.#acker.reset(stream.acks, lastWatermark);
@@ -372,7 +378,7 @@ var ChangeStreamerImpl = class {
 					if (type === "status" && watermark === null && this.#mayRestart()) break;
 					switch (type) {
 						case "status":
-							if (watermark === null) this.#storer.status(change[2].watermark);
+							if (watermark === null) storer?.status(change[2].watermark);
 							if (msg.lagReport && msg.lagReport.lastTimings.commitTimeMs >= this.#latestLagReportCommitTimeMs) {
 								this.#latestLagReportCommitTimeMs = msg.lagReport.lastTimings.commitTimeMs;
 								this.#forwarder.sendStatus({
@@ -399,7 +405,7 @@ var ChangeStreamerImpl = class {
 					this.#backfills.track(change);
 					const serialized = serializeChangeStreamDataWithChange(change);
 					const { json } = serialized;
-					if (this.#pgChangeLogEnabled) this.#storer.store(watermark, change, serialized);
+					storer?.store(watermark, change, serialized);
 					this.#changeLogWriter?.write(change, json, serialized.change);
 					const entry = [
 						watermark,
@@ -416,7 +422,7 @@ var ChangeStreamerImpl = class {
 						await promiseOrAbort(flowControl, stream.changes.signal, this.#state.signal);
 						unflushedBytes = 0;
 					}
-					const readyForMore = this.#pgChangeLogEnabled ? this.#storer.readyForMore() : void 0;
+					const readyForMore = storer?.readyForMore();
 					if (readyForMore) await promiseOrAbort(readyForMore, stream.changes.signal, this.#state.signal);
 					if (type === "commit" && this.#mayRestart()) break;
 				}
@@ -428,7 +434,7 @@ var ChangeStreamerImpl = class {
 			}
 			if (watermark) {
 				this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);
-				if (this.#pgChangeLogEnabled) this.#storer.abort();
+				storer?.abort();
 				this.#changeLogWriter?.abort();
 				this.#purgeScheduler?.onWriterIdle();
 				this.#forwarder.forward([
@@ -441,9 +447,9 @@ var ChangeStreamerImpl = class {
 			}
 			if (this.#backfills.restartReason !== null && err === void 0) {
 				this.#backfills.recordRestart();
-				await this.#storer.stop();
+				await storer?.stop();
 			} else await Promise.all([
-				this.#storer.stop(),
+				storer?.stop(),
 				this.#state.backoff(this.#lc, err),
 				this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS ? publishCriticalEvent(this.#lc, replicationStatusError(this.#lc, "Replicating", err)) : promiseVoid
 			]);
@@ -456,7 +462,7 @@ var ChangeStreamerImpl = class {
 		const { tag } = msg;
 		switch (tag) {
 			case "reset-required":
-				if (this.#pgChangeLogEnabled) await markResetRequired(this.#changeDBProvider("change-streamer-reset", 1), this.#shard);
+				if (this.#changeDBProvider) await markResetRequired(this.#changeDBProvider("change-streamer-reset", 1), this.#shard);
 				await publishReplicationError(this.#lc, "Replicating", msg.message ?? "Resync required", msg.errorDetails);
 				if (this.#autoReset) {
 					this.#lc.warn?.("shutting down for auto-reset");
@@ -533,10 +539,10 @@ var ChangeStreamerImpl = class {
 			this.#forwarder.remove(subscriber);
 		};
 		const catchupFromPG = () => {
-			assert(this.#pgChangeLogEnabled, "cannot catch up from a disabled PG change log");
+			const storer = must(this.#storer, "cannot catch up from a disabled PG change log");
 			try {
 				this.#forwarder.add(subscriber);
-				this.#storer.catchup(subscriber, ctx.mode);
+				storer.catchup(subscriber, ctx.mode);
 			} catch (e) {
 				removeFromForwarder();
 				throw e;
@@ -683,7 +689,7 @@ var ChangeStreamerImpl = class {
 		}, delay);
 	}
 	async #getChangeLogState() {
-		const minWatermark = await this.#storer.getMinWatermarkForCatchup();
+		const minWatermark = await must(this.#storer, "the PG change log is disabled").getMinWatermarkForCatchup();
 		if (!minWatermark) this.#lc.warn?.(`Unexpected empty changeLog. Resync if "Local replica watermark" errors arise`);
 		return {
 			replicaVersion: this.#replicaVersion,
@@ -701,7 +707,8 @@ var ChangeStreamerImpl = class {
 		};
 	}
 	async #purgePGChangeLog() {
-		if (this.#storer.ownershipPending) return;
+		const storer = this.#storer;
+		if (!storer || storer.ownershipPending) return;
 		try {
 			const { backupWatermark, purgeWatermark, current } = this.#getCleanupFloor();
 			if (purgeWatermark < backupWatermark) {
@@ -713,7 +720,7 @@ var ChangeStreamerImpl = class {
 			if (purgeWatermark <= this.#pgPurgedWatermark) return;
 			this.#lc.info?.(`Purging PG changes before ${purgeWatermark} ...`);
 			const start = performance.now();
-			const deleted = await this.#storer.purgeRecordsBefore(purgeWatermark);
+			const deleted = await storer.purgeRecordsBefore(purgeWatermark);
 			const elapsed = (performance.now() - start).toFixed(2);
 			this.#lc.info?.(`Purged ${deleted} PG changes before ${purgeWatermark} (${elapsed} ms)`);
 			this.#pgPurgedWatermark = purgeWatermark;
@@ -747,7 +754,7 @@ var ChangeStreamerImpl = class {
 	* anyway (see Storer.assumeOwnershipAtHead()).
 	*/
 	#mayRestart() {
-		return this.#backfills.restartReason !== null && !this.#storer.ownershipPending;
+		return this.#backfills.restartReason !== null && !this.#storer?.ownershipPending;
 	}
 	async stop(err) {
 		this.#state.stop(this.#lc, err);
@@ -758,7 +765,7 @@ var ChangeStreamerImpl = class {
 		this.#comparator?.stop();
 		this.#sqliteCatchup?.close();
 		this.#changeLogWriter?.close();
-		await Promise.allSettled([this.#storer.stop(), this.#source.stop()]);
+		await Promise.allSettled([this.#storer?.stop(), this.#source.stop()]);
 	}
 	#recordForwardedTransactionBoundary(type, watermark) {
 		switch (type) {