@rocicorp/zero 1.11.0-canary.5 → 1.11.0-canary.7

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -11,6 +11,7 @@ import { Subscription } from "../../types/subscription.js";
 import { AutoResetSignal, ensureReplicationConfig, markResetRequired } from "./schema/tables.js";
 import { publishReplicationError, replicationStatusError } from "../replicator/replication-status.js";
 import "./change-streamer.js";
+import { serializeChangeStreamDataWithChange } from "./change-log-codec.js";
 import { SQLiteChangeLogReader } from "./sqlite-change-log-reader.js";
 import { ChangeLogInitializer, replicaInitializationSource } from "./change-log-initializer.js";
 import { Forwarder } from "./forwarder.js";
@@ -32,7 +33,7 @@ import { getDefaultHighWaterMark } from "node:stream";
 * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.
 */
 async function initializeStreamer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, changeSource, replicationStatusPublisher, subscriptionState, backupConfig, purgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
-	await ensureReplicationConfig(lc, changeDB, subscriptionState, shard, autoReset, purgeLock ?? void 0, setTimeoutFn);
+	await ensureReplicationConfig(lc, changeDB, subscriptionState, shard, autoReset, purgeLock ?? void 0, setTimeoutFn, opts.pgChangeLogEnabled);
 	const changeDBProvider = (applicationName, max) => postgres(defu({
 		max,
 		connection: { ["application_name"]: applicationName }
@@ -188,6 +189,7 @@ var ChangeStreamerImpl = class {
 	#replicaVersion;
 	#source;
 	#storer;
+	#pgChangeLogEnabled;
 	#forwarder;
 	#reservations;
 	#replicationStatusPublisher;
@@ -237,6 +239,11 @@ var ChangeStreamerImpl = class {
 		this.#changeDBProvider = changeDBProvider;
 		this.#replicaVersion = replicaVersion;
 		this.#source = source;
+		this.#pgChangeLogEnabled = opts.pgChangeLogEnabled;
+		if (!this.#pgChangeLogEnabled) {
+			assert(opts.sqliteChangeLogWriter && opts.sqliteChangeLogPurge && opts.sqliteCatchup && opts.sqliteChangeLogServe?.readPercent === 100 && opts.sqliteChangeLogServe.coldReadPercent === 100 && backupConfig?.litestreamVersion === "v5", "disabling the PG change log requires SQLite serving at 100 percent and a v5 backup");
+			assert(opts.sqliteChangeLogCompare === void 0, "SQLite change-log comparison requires the PG change log");
+		}
 		this.#storer = new Storer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, replicaVersion, (consumed) => this.#acker.trackPgChangeLog(consumed[2].watermark), (err) => this.stop(err), opts);
 		this.#forwarder = new Forwarder(lc, {
 			flowControlConsensusTimeoutProportion: opts.flowControlConsensusTimeoutProportion,
@@ -277,12 +284,13 @@ var ChangeStreamerImpl = class {
 		}) : void 0;
 		this.#purgeScheduler = opts.sqliteChangeLogPurge && this.#changeLogWriter ? new SQLiteChangeLogPurgeScheduler(lc, () => this.#changeLogWriter?.connection, () => this.#forwarder.getAcks(), opts.sqliteChangeLogPurge) : void 0;
 		this.#acker = new UpstreamAcker({
-			trackPgChangeLog: true,
+			trackPgChangeLog: this.#pgChangeLogEnabled,
 			trackBackup: backupConfig?.litestreamVersion === "v5"
 		});
-		const replicaSource = opts.sqliteChangeLogCompare ? replicaInitializationSource(lc, opts.sqliteChangeLogCompare.replicaFile) : void 0;
+		const replicaFileForInitialization = opts.sqliteChangeLogCompare?.replicaFile ?? (!this.#pgChangeLogEnabled ? opts.sqliteChangeLogWriter?.replicaFile : void 0);
+		const replicaSource = replicaFileForInitialization ? replicaInitializationSource(lc, replicaFileForInitialization) : void 0;
 		this.#initializer = new ChangeLogInitializer(lc, {
-			initFromPgChangeLog: true,
+			initFromPgChangeLog: this.#pgChangeLogEnabled,
 			initFromReplica: replicaSource !== void 0
 		}, {
 			pgChangeLog: () => this.#storer.getStartStreamInitializationParameters(),
@@ -294,7 +302,7 @@ var ChangeStreamerImpl = class {
 			...opts.sqliteCatchup,
 			cleanupGuard: opts.sqliteCatchup.cleanupGuard ?? this.#purgeScheduler?.cleanupGuard
 		} : void 0;
-		this.#comparator = opts.sqliteChangeLogCompare && opts.sqliteChangeLogWriter && opts.sqliteCatchup ? new SQLiteChangeLogComparator(lc, shard, opts.sqliteCatchup.changeLogFile, opts.sqliteChangeLogWriter.identity, this.#storer, {
+		this.#comparator = this.#pgChangeLogEnabled && opts.sqliteChangeLogCompare && opts.sqliteChangeLogWriter && opts.sqliteCatchup ? new SQLiteChangeLogComparator(lc, shard, opts.sqliteCatchup.changeLogFile, opts.sqliteChangeLogWriter.identity, this.#storer, {
 			setTimeoutFn,
 			...opts.sqliteChangeLogCompare
 		}) : void 0;
@@ -322,7 +330,7 @@ var ChangeStreamerImpl = class {
 				const { lastWatermark, backfillRequests } = await this.#initializer.initialize();
 				this.#lastForwardedCommitWatermark = lastWatermark;
 				const stream = await this.#source.startStream(lastWatermark, backfillRequests);
-				this.#storer.run().catch((e) => stream.changes.cancel(e));
+				if (this.#pgChangeLogEnabled) this.#storer.run().catch((e) => stream.changes.cancel(e));
 				this.#stream = stream;
 				if (this.#state.resetBackoff() > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS) this.#replicationStatusPublisher.publish(this.#lc, "Replicating", `Replicating from ${lastWatermark}`);
 				watermark = null;
@@ -353,8 +361,10 @@ var ChangeStreamerImpl = class {
 							if (watermark === null) throw new UnrecoverableError(`${type} change (${msg.tag}) received before 'begin' message`);
 							break;
 					}
-					const json = this.#storer.store(watermark, change);
-					this.#changeLogWriter?.write(change, json);
+					const serialized = serializeChangeStreamDataWithChange(change);
+					const { json } = serialized;
+					if (this.#pgChangeLogEnabled) this.#storer.store(watermark, change, serialized);
+					this.#changeLogWriter?.write(change, json, serialized.change);
 					const entry = [
 						watermark,
 						change[1].tag,
@@ -371,7 +381,7 @@ var ChangeStreamerImpl = class {
 						unflushedBytes = 0;
 					}
 					if (type === "commit" || type === "rollback") watermark = null;
-					const readyForMore = this.#storer.readyForMore();
+					const readyForMore = this.#pgChangeLogEnabled ? this.#storer.readyForMore() : void 0;
 					if (readyForMore) await promiseOrAbort(readyForMore, stream.changes.signal, this.#state.signal);
 				}
 			} catch (e) {
@@ -382,7 +392,7 @@ var ChangeStreamerImpl = class {
 			}
 			if (watermark) {
 				this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);
-				this.#storer.abort();
+				if (this.#pgChangeLogEnabled) this.#storer.abort();
 				this.#changeLogWriter?.abort();
 				this.#purgeScheduler?.onWriterIdle();
 				this.#forwarder.forward([
@@ -434,14 +444,19 @@ var ChangeStreamerImpl = class {
 		} else {
 			lc.info?.(`adding subscriber ${subscriber.id}`);
 			const catchupFromPG = () => {
+				assert(this.#pgChangeLogEnabled, "cannot catch up from a disabled PG change log");
 				cleanupSubscriber = removeFromForwarder;
 				this.#forwarder.add(subscriber);
 				this.#storer.catchup(subscriber, mode);
 			};
-			const sqliteSelection = this.#selectSQLiteCatchup(lc, ctx);
-			if (!sqliteSelection) catchupFromPG();
-			else {
-				const { catchup, reason, coverage, logWarm } = sqliteSelection;
+			const sqliteDecision = this.#selectSQLiteCatchup(lc, ctx);
+			if (!sqliteDecision) catchupFromPG();
+			else if (sqliteDecision.kind === "rejected") {
+				lc.warn?.(`${sqliteDecision.terminal ? "rejecting" : "ending"} subscription for ${ctx.id}: ${sqliteDecision.message}`);
+				if (sqliteDecision.terminal) subscriber.close(2, sqliteDecision.message);
+				else subscriber.close();
+			} else {
+				const { catchup, reason, coverage, logWarm } = sqliteDecision;
 				cleanupSubscriber = () => catchup.remove(subscriber);
 				const registration = await catchup.catchup(subscriber, () => this.#captureRequiredHead(), { logWarm });
 				switch (registration.kind) {
@@ -450,14 +465,27 @@ var ChangeStreamerImpl = class {
 						this.#recordCatchupRoute("sqlite", reason);
 						break;
 					case "uncovered":
-						lc.info?.(`serving ${ctx.id} from PG catchup: subscriber watermark ${ctx.watermark} is below the SQLite change-log minimum ` + registration.minWatermark);
-						this.#recordCatchupRoute("pg", "watermark-uncovered");
-						catchupFromPG();
+						if (this.#pgChangeLogEnabled) {
+							lc.info?.(`serving ${ctx.id} from PG catchup: subscriber watermark ${ctx.watermark} is below the SQLite change-log minimum ` + registration.minWatermark);
+							this.#recordCatchupRoute("pg", "watermark-uncovered");
+							catchupFromPG();
+						} else {
+							const message = `earliest supported watermark is ${registration.minWatermark} (requested ${ctx.watermark})`;
+							lc.warn?.(`rejecting subscriber at watermark ${ctx.watermark}: the SQLite change log starts at ${registration.minWatermark} and the PG change log is disabled`);
+							this.#recordCatchupRoute("none", "watermark-uncovered");
+							subscriber.close(2, message);
+						}
 						break;
 					case "declined":
-						lc.error?.(`serving ${ctx.id} from PG catchup: SQLite catchup registration failed`, registration.error);
-						this.#recordCatchupRoute("pg", "registration-failed");
-						catchupFromPG();
+						if (this.#pgChangeLogEnabled) {
+							lc.error?.(`serving ${ctx.id} from PG catchup: SQLite catchup registration failed`, registration.error);
+							this.#recordCatchupRoute("pg", "registration-failed");
+							catchupFromPG();
+						} else {
+							lc.error?.(`ending subscription for ${ctx.id} to retry SQLite catchup: registration failed and the PG change log is disabled`, registration.error);
+							this.#recordCatchupRoute("none", "registration-failed");
+							subscriber.fail(registration.error);
+						}
 						break;
 					case "handled":
 						this.#recordCatchupRoute("sqlite", "registration-handled");
@@ -495,13 +523,22 @@ var ChangeStreamerImpl = class {
 		const backupWatermark = this.#backupWatermark;
 		const reservations = this.#reservations;
 		if (backupWatermark === void 0 || !reservations?.confirmationsRequired()) return;
-		const pgState = await this.#getChangeLogState();
+		const pgState = this.#pgChangeLogEnabled ? await this.#getChangeLogState() : void 0;
 		for (const taskID of reservations.unconfirmedTaskIDs()) {
 			let route = this.#readRouter?.peek(taskID);
 			if (this.#readRouter && route === void 0) continue;
+			if (!this.#pgChangeLogEnabled && route?.source === "pg") {
+				this.#readRouter?.release(taskID);
+				route = this.#readRouter?.pin(taskID);
+				if (route?.source === "pg") {
+					if (reservations.noteConfirmationDelayed(taskID)) this.#reservationConfirmDelays.add(1);
+					this.#lc.warn?.(`delaying snapshot reservation for ${taskID}: SQLite change log is unavailable and the PG change log is disabled`);
+					continue;
+				}
+			}
 			if (route?.source === "sqlite") {
 				const coverage = must(route.coverage, "a pinned SQLite route must carry its covered range");
-				if (coverage.minWatermark > backupWatermark) {
+				if (this.#pgChangeLogEnabled && coverage.minWatermark > backupWatermark) {
 					this.#lc.info?.(`demoting ${taskID} to PG catchup: SQLite change-log minimum ${coverage.minWatermark} is later than backupWatermark ` + backupWatermark);
 					this.#reservationDemotions.add(1);
 					route = must(this.#readRouter).demote(taskID);
@@ -510,17 +547,17 @@ var ChangeStreamerImpl = class {
 			const source = route?.source ?? "pg";
 			let minWatermark;
 			if (route?.source === "sqlite") minWatermark = must(route.coverage, "a pinned SQLite route must carry its covered range").minWatermark;
-			else ({minWatermark} = pgState);
-			if (minWatermark <= backupWatermark) reservations.confirmFor(taskID, pgState.replicaVersion, backupWatermark, source);
+			else minWatermark = must(pgState, "a PG reservation route requires the PG change log").minWatermark;
+			if (minWatermark <= backupWatermark) reservations.confirmFor(taskID, this.#replicaVersion, backupWatermark, source);
 			else {
 				if (reservations.noteConfirmationDelayed(taskID)) this.#reservationConfirmDelays.add(1);
-				this.#lc.error?.(`pg change-log minWatermark ${minWatermark} is later than backupWatermark ${backupWatermark}. Delaying confirmation of snapshot reservation until next backup.`);
+				this.#lc.error?.(`${source} change-log minWatermark ${minWatermark} is later than backupWatermark ${backupWatermark}. Delaying confirmation of snapshot reservation until next backup.`);
 			}
 		}
 	}
 	#maybeSchedulePGPurge() {
 		const backupWatermark = this.#backupWatermark;
-		if (this.#pgPurgeScheduled || this.#pgPurgeRunning || backupWatermark === void 0 || this.#pgPurgedWatermark >= backupWatermark) return;
+		if (!this.#pgChangeLogEnabled || this.#pgPurgeScheduled || this.#pgPurgeRunning || backupWatermark === void 0 || this.#pgPurgedWatermark >= backupWatermark) return;
 		this.#pgPurgeScheduled = true;
 		this.#state.setTimeout(() => {
 			this.#pgPurgeScheduled = false;
@@ -664,14 +701,10 @@ var ChangeStreamerImpl = class {
 	}
 	#selectSQLiteCatchup(lc, ctx) {
 		const opts = this.#sqliteCatchupOptions;
-		if (this.#lastForwardedCommitWatermark === void 0 || !opts) {
-			this.#recordCatchupRoute("pg", "not-ready");
-			return;
-		}
-		if (ctx.mode !== "serving") {
+		if (this.#lastForwardedCommitWatermark === void 0 || !opts) return this.#fallbackOrReject("not-ready", "SQLite catchup is not ready and the PG change log is disabled");
+		if (ctx.mode === "backup" && this.#pgChangeLogEnabled) {
 			lc.info?.(`not serving backup subscriber ${ctx.id} from SQLite catchup`);
-			this.#recordCatchupRoute("pg", "ineligible-mode");
-			return;
+			return this.#fallbackOrReject("ineligible-mode", "backup subscribers retain the PG change-log recovery policy");
 		}
 		let route;
 		if (this.#readRouter) {
@@ -680,33 +713,34 @@ var ChangeStreamerImpl = class {
 				if (route.reason === "cold-log") lc.info?.(`serving ${ctx.id} from PG catchup: SQLite change log is still warming`, { sqliteChangeLogCoverage: route.coverage });
 				else if (route.reason === "log-unavailable") this.#declineSQLiteCatchup(lc, ctx, opts, `${opts.changeLogFile} is unavailable or incompatible`);
 				else lc.debug?.(`serving ${ctx.id} from PG catchup: SQLite route ${route.reason}`, ...route.coverage ? [{ sqliteChangeLogCoverage: route.coverage }] : []);
-				this.#recordCatchupRoute("pg", route.reason);
-				return;
+				return this.#fallbackOrReject(route.reason, `SQLite catchup route ${route.reason} is unavailable and the PG change log is disabled`);
 			}
 			const coverage = route.coverage;
 			assert(coverage, "a SQLite route must carry its covered range");
 			if (ctx.watermark < coverage.minWatermark) {
 				lc.info?.(`serving ${ctx.id} from PG catchup: subscriber watermark ${ctx.watermark} is below the SQLite change-log minimum ` + coverage.minWatermark, { sqliteChangeLogCoverage: coverage });
-				this.#recordCatchupRoute("pg", "watermark-uncovered");
-				return;
+				return this.#fallbackOrReject("watermark-uncovered", `earliest supported watermark is ${coverage.minWatermark} (requested ${ctx.watermark})`, true);
 			}
 		}
-		if (!this.#readRouter && !opts.shouldUse?.(ctx)) {
-			this.#recordCatchupRoute("pg", "selector");
-			return;
-		}
-		if (this.#readRouter && opts.shouldUse && !opts.shouldUse(ctx)) {
-			this.#recordCatchupRoute("pg", "selector");
-			return;
-		}
+		if (!this.#readRouter && !opts.shouldUse?.(ctx)) return this.#fallbackOrReject("selector", "SQLite catchup was not selected and the PG change log is disabled");
+		if (this.#readRouter && opts.shouldUse && !opts.shouldUse(ctx)) return this.#fallbackOrReject("selector", "SQLite catchup was not selected and the PG change log is disabled");
 		const catchup = this.#sqliteCatchup ?? this.#openSQLiteCatchup(lc, opts, ctx);
 		if (catchup) return {
+			kind: "selected",
 			catchup,
 			reason: route?.reason ?? "selector",
 			coverage: route?.coverage,
 			logWarm: route === void 0 ? void 0 : route.reason !== "selected-cold"
 		};
-		else this.#recordCatchupRoute("pg", "log-unavailable");
+		else return this.#fallbackOrReject("log-unavailable", "SQLite change log is unavailable and the PG change log is disabled");
+	}
+	#fallbackOrReject(reason, message, terminal = false) {
+		this.#recordCatchupRoute(this.#pgChangeLogEnabled ? "pg" : "none", reason);
+		return this.#pgChangeLogEnabled ? void 0 : {
+			kind: "rejected",
+			message,
+			terminal
+		};
 	}
 	#recordCatchupRoute(source, reason) {
 		this.#catchupRoutes.add(1, {
@@ -722,10 +756,11 @@ var ChangeStreamerImpl = class {
 	* first reconcile, and deletes it when it fails soft -- or may exist without
 	* content, or may be unreadable. None of those can be allowed to fail a
 	* subscription:
-	* this is the last point at which PG catchup is still available -- past
-	* `Forwarder.add()` the subscriber is committed to SQLite -- so each declines
-	* here instead. Neither the failure nor the reader is retained, so a later
-	* subscription retries from scratch.
+	* this is the last point at which PG catchup can be selected when it is
+	* enabled -- past `Forwarder.add()` the subscriber is committed to SQLite --
+	* so each declines here instead. With PG disabled, the caller ends the
+	* subscription so that it retries. Neither the failure nor the reader is
+	* retained, so a later subscription retries from scratch.
 	*/
 	#openSQLiteCatchup(lc, opts, ctx) {
 		let reader;
@@ -763,7 +798,7 @@ var ChangeStreamerImpl = class {
 		const now = Date.now();
 		this.#changeLogUnavailableSince ??= now;
 		const unavailableMs = now - this.#changeLogUnavailableSince;
-		lc[unavailableMs >= (opts.notReadyWarnThresholdMs ?? DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS) ? "warn" : "debug"]?.(`serving ${ctx.id} from PG catchup: ${reason} (unavailable for ${unavailableMs} ms)`, ...error === void 0 ? [] : [error]);
+		lc[unavailableMs >= (opts.notReadyWarnThresholdMs ?? DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS) ? "warn" : "debug"]?.((this.#pgChangeLogEnabled ? `serving ${ctx.id} from PG catchup: ` : `cannot serve ${ctx.id} from SQLite catchup: `) + `${reason} (unavailable for ${unavailableMs} ms)`, ...error === void 0 ? [] : [error]);
 	}
 };
 var CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;