@rocicorp/zero 1.10.0-canary.2 → 1.10.0-canary.21

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -1,6 +1,7 @@
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
 import { must } from "../../../../shared/src/must.js";
+import { promiseOrAbort } from "../../../../shared/src/promise-race.js";
 import "../change-source/protocol/current/downstream.js";
 import { getOrCreateCounter, getOrCreateLatencyHistogram } from "../../observability/metrics.js";
 import { DEFAULT_MAX_RETRY_DELAY_MS, RunningState, UnrecoverableError } from "../running-state.js";
@@ -15,12 +16,16 @@ import { ChangeLogInitializer, replicaInitializationSource } from "./change-log-
 import { Forwarder } from "./forwarder.js";
 import { SnapshotReservations } from "./snapshot-reservations.js";
 import { SQLiteChangeLogCatchup } from "./sqlite-change-log-catchup.js";
+import { SQLiteChangeLogComparator } from "./sqlite-change-log-comparator.js";
 import { SQLiteChangeLogPurgeScheduler } from "./sqlite-change-log-purge-scheduler.js";
+import { SQLiteChangeLogReadRouter, inspectSQLiteChangeLog } from "./sqlite-change-log-read-router.js";
 import { SQLiteChangeLogWriter } from "./sqlite-change-log-writer.js";
 import { Storer } from "./storer.js";
 import { Subscriber } from "./subscriber.js";
 import { UpstreamAcker } from "./upstream-acker.js";
 import { resolver } from "@rocicorp/resolver";
+import { defu } from "defu";
+import postgres from "postgres";
 import { getDefaultHighWaterMark } from "node:stream";
 //#region ../zero-cache/src/services/change-streamer/change-streamer-service.ts
 /**
@@ -28,8 +33,12 @@ import { getDefaultHighWaterMark } from "node:stream";
 */
 async function initializeStreamer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, changeSource, replicationStatusPublisher, subscriptionState, backupConfig, purgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
 	await ensureReplicationConfig(lc, changeDB, subscriptionState, shard, autoReset, purgeLock ?? void 0, setTimeoutFn);
+	const changeDBProvider = (applicationName, max) => postgres(defu({
+		max,
+		connection: { ["application_name"]: applicationName }
+	}, changeDB.options));
 	const { replicaVersion } = subscriptionState;
-	return new ChangeStreamerImpl(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, changeSource, replicationStatusPublisher, backupConfig, purgeLock, autoReset, opts, setTimeoutFn);
+	return new ChangeStreamerImpl(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, replicaVersion, changeSource, replicationStatusPublisher, backupConfig, purgeLock, autoReset, opts, setTimeoutFn);
 }
 var REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5e3;
 var DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS = 6e4;
@@ -175,7 +184,7 @@ var ChangeStreamerImpl = class {
 	id;
 	#lc;
 	#shard;
-	#changeDB;
+	#changeDBProvider;
 	#replicaVersion;
 	#source;
 	#storer;
@@ -185,14 +194,19 @@ var ChangeStreamerImpl = class {
 	#sqliteCatchupOptions;
 	#changeLogWriter;
 	#purgeScheduler;
+	#comparator;
 	#acker;
 	#initializer;
+	#readRouter;
 	#autoReset;
 	#state;
 	#serving = resolver();
 	#txCounter = getOrCreateCounter("replication", "transactions", "Count of replicated transactions");
 	#changeCounter = getOrCreateCounter("replication", "changes", "Count of replicated changes (DML or DDL statements)");
 	#transactionForwardDuration = getOrCreateLatencyHistogram("replication", "transaction_forward_duration", "Time from receiving a transaction's `begin` to forwarding its `commit`, i.e. the change-streamer half of forward-to-subscriber latency.");
+	#catchupRoutes = getOrCreateCounter("replication", "sqlite_change_log.catchup_routes", "Catchup subscriptions by selected source and low-cardinality reason.");
+	#reservationDemotions = getOrCreateCounter("replication", "sqlite_change_log.reservation_demotions", "Snapshot reservations demoted from SQLite to PG because the change log could not cover the backup being restored. With PG retired these followers have no fallback, so this is the rate at which one would instead have to wait for a later backup.");
+	#reservationConfirmDelays = getOrCreateCounter("replication", "sqlite_change_log.reservation_confirm_delays", "Snapshot reservations whose confirmation was deferred because the selected source's change-log minimum was later than the backup watermark. Counted once per reservation, by that source.");
 	#latestStatus;
 	#latestLagReportCommitTimeMs = 0;
 	#backupWatermark;
@@ -216,19 +230,34 @@ var ChangeStreamerImpl = class {
 	#lastForwardedCommitWatermark;
 	#transactionForwardStartedAt;
 	#currentTransactionCompletion;
-	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, source, replicationStatusPublisher, backupConfig, initialPurgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
+	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, replicaVersion, source, replicationStatusPublisher, backupConfig, initialPurgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
 		this.id = `change-streamer`;
 		this.#lc = lc.withContext("component", "change-streamer");
 		this.#shard = shard;
-		this.#changeDB = changeDB;
+		this.#changeDBProvider = changeDBProvider;
 		this.#replicaVersion = replicaVersion;
 		this.#source = source;
-		this.#storer = new Storer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, (consumed) => this.#acker.trackPgChangeLog(consumed[2].watermark), (err) => this.stop(err), opts);
+		this.#storer = new Storer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, replicaVersion, (consumed) => this.#acker.trackPgChangeLog(consumed[2].watermark), (err) => this.stop(err), opts);
 		this.#forwarder = new Forwarder(lc, {
 			flowControlConsensusTimeoutProportion: opts.flowControlConsensusTimeoutProportion,
 			flowControlSlowSubscriberGracePeriodMs: opts.flowControlSlowSubscriberGracePeriodMs
 		});
-		this.#reservations = backupConfig ? new SnapshotReservations(lc, backupConfig, (taskID) => this.#purgeScheduler?.resume(taskID)) : void 0;
+		const serveOptions = opts.sqliteChangeLogServe;
+		const writerOptions = opts.sqliteChangeLogWriter;
+		const catchupOptions = opts.sqliteCatchup;
+		this.#readRouter = serveOptions && writerOptions && catchupOptions ? new SQLiteChangeLogReadRouter({
+			shard,
+			readPercent: serveOptions.readPercent,
+			coldReadPercent: serveOptions.coldReadPercent,
+			retentionMs: serveOptions.retentionMs,
+			failureCooldownMs: serveOptions.failureCooldownMs,
+			now: serveOptions.now,
+			inspect: serveOptions.inspect ?? (() => inspectSQLiteChangeLog(lc, catchupOptions.changeLogFile, writerOptions.identity))
+		}) : void 0;
+		this.#reservations = backupConfig ? new SnapshotReservations(lc, backupConfig, (taskID) => {
+			this.#readRouter?.release(taskID);
+			this.#purgeScheduler?.resume(taskID);
+		}) : void 0;
 		this.#replicationStatusPublisher = replicationStatusPublisher;
 		this.#changeLogWriter = opts.sqliteChangeLogWriter ? new SQLiteChangeLogWriter(lc, {
 			...opts.sqliteChangeLogWriter,
@@ -236,8 +265,15 @@ var ChangeStreamerImpl = class {
 				this.#sqliteCatchup?.onChangeLogCommit(watermark);
 				this.#purgeScheduler?.onWriterIdle();
 			},
-			onDisabled: () => this.#closeSQLiteCatchup(),
-			onRebuilt: () => this.#closeSQLiteCatchup()
+			onDisabled: () => {
+				this.#readRouter?.trip(true);
+				this.#closeSQLiteCatchup();
+				this.#comparator?.stop();
+			},
+			onRebuilt: () => {
+				this.#closeSQLiteCatchup();
+				this.#comparator?.invalidate();
+			}
 		}) : void 0;
 		this.#purgeScheduler = opts.sqliteChangeLogPurge && this.#changeLogWriter ? new SQLiteChangeLogPurgeScheduler(lc, () => this.#changeLogWriter?.connection, () => this.#forwarder.getAcks(), opts.sqliteChangeLogPurge) : void 0;
 		this.#acker = new UpstreamAcker({
@@ -258,6 +294,10 @@ var ChangeStreamerImpl = class {
 			...opts.sqliteCatchup,
 			cleanupGuard: opts.sqliteCatchup.cleanupGuard ?? this.#purgeScheduler?.cleanupGuard
 		} : void 0;
+		this.#comparator = opts.sqliteChangeLogCompare && opts.sqliteChangeLogWriter && opts.sqliteCatchup ? new SQLiteChangeLogComparator(lc, shard, opts.sqliteCatchup.changeLogFile, opts.sqliteChangeLogWriter.identity, this.#storer, {
+			setTimeoutFn,
+			...opts.sqliteChangeLogCompare
+		}) : void 0;
 		this.#purgeLock = initialPurgeLock;
 		this.#autoReset = autoReset;
 		this.#state = new RunningState(this.id, void 0, setTimeoutFn);
@@ -327,12 +367,12 @@ var ChangeStreamerImpl = class {
 					} else {
 						const forwarded = this.#forwarder.forwardWithFlowControl(entry);
 						this.#recordForwardedTransactionBoundary(type, entry[0]);
-						await stream.changes.doneOr(forwarded);
+						await promiseOrAbort(forwarded, stream.changes.signal, this.#state.signal);
 						unflushedBytes = 0;
 					}
 					if (type === "commit" || type === "rollback") watermark = null;
 					const readyForMore = this.#storer.readyForMore();
-					if (readyForMore) await stream.changes.doneOr(readyForMore);
+					if (readyForMore) await promiseOrAbort(readyForMore, stream.changes.signal, this.#state.signal);
 				}
 			} catch (e) {
 				err = e;
@@ -366,7 +406,7 @@ var ChangeStreamerImpl = class {
 		const { tag } = msg;
 		switch (tag) {
 			case "reset-required":
-				await markResetRequired(this.#changeDB, this.#shard);
+				await markResetRequired(this.#changeDBProvider("change-streamer-reset", 1), this.#shard);
 				await publishReplicationError(this.#lc, "Replicating", msg.message ?? "Resync required", msg.errorDetails);
 				if (this.#autoReset) {
 					this.#lc.warn?.("shutting down for auto-reset");
@@ -381,24 +421,49 @@ var ChangeStreamerImpl = class {
 		if (mode === "serving") this.#serving.resolve();
 		let cleanupSubscriber = () => {};
 		const downstream = Subscription.create({ cleanup: () => cleanupSubscriber() });
-		const subscriber = new Subscriber(protocolVersion, id, watermark, downstream, () => this.#latestStatus, {});
+		const subscriber = new Subscriber(protocolVersion, id, mode, watermark, downstream, () => this.#latestStatus, {});
 		const lc = this.#lc.withContext("subscriber", subscriber.id);
-		cleanupSubscriber = () => {
+		const removeFromForwarder = () => {
 			lc.info?.(`removing subscriber ${subscriber.id}`);
 			this.#forwarder.remove(subscriber);
 		};
+		cleanupSubscriber = removeFromForwarder;
 		if (replicaVersion !== this.#replicaVersion) {
 			lc.warn?.(`rejecting subscriber at replica version ${replicaVersion}`);
 			subscriber.close(1, `current replica version is ${this.#replicaVersion} (requested ${replicaVersion})`);
 		} else {
 			lc.info?.(`adding subscriber ${subscriber.id}`);
-			const sqliteCatchup = this.#selectSQLiteCatchup(ctx);
-			if (sqliteCatchup) {
-				cleanupSubscriber = () => sqliteCatchup.remove(subscriber);
-				await sqliteCatchup.catchup(subscriber, () => this.#captureRequiredHead());
-			} else {
+			const catchupFromPG = () => {
+				cleanupSubscriber = removeFromForwarder;
 				this.#forwarder.add(subscriber);
 				this.#storer.catchup(subscriber, mode);
+			};
+			const sqliteSelection = this.#selectSQLiteCatchup(lc, ctx);
+			if (!sqliteSelection) catchupFromPG();
+			else {
+				const { catchup, reason, coverage, logWarm } = sqliteSelection;
+				cleanupSubscriber = () => catchup.remove(subscriber);
+				const registration = await catchup.catchup(subscriber, () => this.#captureRequiredHead(), { logWarm });
+				switch (registration.kind) {
+					case "registered":
+						lc.debug?.(`serving ${ctx.id} from SQLite catchup`, ...coverage ? [{ sqliteChangeLogCoverage: coverage }] : []);
+						this.#recordCatchupRoute("sqlite", reason);
+						break;
+					case "uncovered":
+						lc.info?.(`serving ${ctx.id} from PG catchup: subscriber watermark ${ctx.watermark} is below the SQLite change-log minimum ` + registration.minWatermark);
+						this.#recordCatchupRoute("pg", "watermark-uncovered");
+						catchupFromPG();
+						break;
+					case "declined":
+						lc.error?.(`serving ${ctx.id} from PG catchup: SQLite catchup registration failed`, registration.error);
+						this.#recordCatchupRoute("pg", "registration-failed");
+						catchupFromPG();
+						break;
+					case "handled":
+						this.#recordCatchupRoute("sqlite", "registration-handled");
+						break;
+					default: unreachable(registration);
+				}
 			}
 		}
 		this.#reservations?.close(ctx.taskID);
@@ -409,6 +474,8 @@ var ChangeStreamerImpl = class {
 		const downstream = this.#reservations.open(taskID);
 		try {
 			await (this.#purgeScheduler?.pause(taskID) ?? promiseVoid);
+			if (!this.#reservations.isCurrent(taskID, downstream)) return downstream;
+			this.#readRouter?.pin(taskID);
 			await this.#confirmReservations();
 			return downstream;
 		} catch (e) {
@@ -425,10 +492,30 @@ var ChangeStreamerImpl = class {
 		this.#confirmReservations().catch((e) => this.#lc.warn?.(`error confirming snapshot reservation`, e));
 	}
 	async #confirmReservations() {
-		if (this.#backupWatermark && this.#reservations?.confirmationsRequired()) {
-			const { replicaVersion, minWatermark } = await this.#getChangeLogState();
-			if (minWatermark <= this.#backupWatermark) this.#reservations?.confirm(replicaVersion, this.#backupWatermark);
-			else this.#lc.error?.(`change-log minWatermark ${minWatermark} is later than backupWatermark ${this.#backupWatermark}. Delaying confirmation of snapshot reservation until next backup.`);
+		const backupWatermark = this.#backupWatermark;
+		const reservations = this.#reservations;
+		if (backupWatermark === void 0 || !reservations?.confirmationsRequired()) return;
+		const pgState = await this.#getChangeLogState();
+		for (const taskID of reservations.unconfirmedTaskIDs()) {
+			let route = this.#readRouter?.peek(taskID);
+			if (this.#readRouter && route === void 0) continue;
+			if (route?.source === "sqlite") {
+				const coverage = must(route.coverage, "a pinned SQLite route must carry its covered range");
+				if (coverage.minWatermark > backupWatermark) {
+					this.#lc.info?.(`demoting ${taskID} to PG catchup: SQLite change-log minimum ${coverage.minWatermark} is later than backupWatermark ` + backupWatermark);
+					this.#reservationDemotions.add(1);
+					route = must(this.#readRouter).demote(taskID);
+				}
+			}
+			const source = route?.source ?? "pg";
+			let minWatermark;
+			if (route?.source === "sqlite") minWatermark = must(route.coverage, "a pinned SQLite route must carry its covered range").minWatermark;
+			else ({minWatermark} = pgState);
+			if (minWatermark <= backupWatermark) reservations.confirmFor(taskID, pgState.replicaVersion, backupWatermark, source);
+			else {
+				if (reservations.noteConfirmationDelayed(taskID)) this.#reservationConfirmDelays.add(1);
+				this.#lc.error?.(`pg change-log minWatermark ${minWatermark} is later than backupWatermark ${backupWatermark}. Delaying confirmation of snapshot reservation until next backup.`);
+			}
 		}
 	}
 	#maybeSchedulePGPurge() {
@@ -517,6 +604,7 @@ var ChangeStreamerImpl = class {
 		this.#state.stop(this.#lc, err);
 		this.#stream?.changes.cancel();
 		this.#purgeScheduler?.stop();
+		this.#comparator?.stop();
 		this.#sqliteCatchup?.close();
 		this.#changeLogWriter?.close();
 		await Promise.allSettled([this.#storer.stop(), this.#source.stop()]);
@@ -574,19 +662,57 @@ var ChangeStreamerImpl = class {
 		assert(committed, "last forwarded commit watermark is not initialized");
 		return this.#currentTransactionCompletion?.promise.then((completion) => completion.watermark) ?? committed;
 	}
-	#selectSQLiteCatchup(ctx) {
+	#selectSQLiteCatchup(lc, ctx) {
 		const opts = this.#sqliteCatchupOptions;
-		if (this.#lastForwardedCommitWatermark === void 0 || !opts) return;
+		if (this.#lastForwardedCommitWatermark === void 0 || !opts) {
+			this.#recordCatchupRoute("pg", "not-ready");
+			return;
+		}
 		if (ctx.mode !== "serving") {
-			this.#lc.info?.(`not serving backup subscriber ${ctx.id} from SQLite catchup`);
+			lc.info?.(`not serving backup subscriber ${ctx.id} from SQLite catchup`);
+			this.#recordCatchupRoute("pg", "ineligible-mode");
 			return;
 		}
-		if (ctx.logsChangeStream) {
-			this.#lc.warn?.(`not serving legacy change-log writer ${ctx.id} from SQLite catchup`);
+		let route;
+		if (this.#readRouter) {
+			route = this.#readRouter.consume(ctx.taskID);
+			if (route.source === "pg") {
+				if (route.reason === "cold-log") lc.info?.(`serving ${ctx.id} from PG catchup: SQLite change log is still warming`, { sqliteChangeLogCoverage: route.coverage });
+				else if (route.reason === "log-unavailable") this.#declineSQLiteCatchup(lc, ctx, opts, `${opts.changeLogFile} is unavailable or incompatible`);
+				else lc.debug?.(`serving ${ctx.id} from PG catchup: SQLite route ${route.reason}`, ...route.coverage ? [{ sqliteChangeLogCoverage: route.coverage }] : []);
+				this.#recordCatchupRoute("pg", route.reason);
+				return;
+			}
+			const coverage = route.coverage;
+			assert(coverage, "a SQLite route must carry its covered range");
+			if (ctx.watermark < coverage.minWatermark) {
+				lc.info?.(`serving ${ctx.id} from PG catchup: subscriber watermark ${ctx.watermark} is below the SQLite change-log minimum ` + coverage.minWatermark, { sqliteChangeLogCoverage: coverage });
+				this.#recordCatchupRoute("pg", "watermark-uncovered");
+				return;
+			}
+		}
+		if (!this.#readRouter && !opts.shouldUse?.(ctx)) {
+			this.#recordCatchupRoute("pg", "selector");
 			return;
 		}
-		if (!opts.shouldUse?.(ctx)) return;
-		return this.#sqliteCatchup ?? this.#openSQLiteCatchup(opts, ctx);
+		if (this.#readRouter && opts.shouldUse && !opts.shouldUse(ctx)) {
+			this.#recordCatchupRoute("pg", "selector");
+			return;
+		}
+		const catchup = this.#sqliteCatchup ?? this.#openSQLiteCatchup(lc, opts, ctx);
+		if (catchup) return {
+			catchup,
+			reason: route?.reason ?? "selector",
+			coverage: route?.coverage,
+			logWarm: route === void 0 ? void 0 : route.reason !== "selected-cold"
+		};
+		else this.#recordCatchupRoute("pg", "log-unavailable");
+	}
+	#recordCatchupRoute(source, reason) {
+		this.#catchupRoutes.add(1, {
+			source,
+			reason
+		});
 	}
 	/**
 	* Opens the change log and, if it can serve, wraps it in the coordinator that
@@ -601,18 +727,18 @@ var ChangeStreamerImpl = class {
 	* here instead. Neither the failure nor the reader is retained, so a later
 	* subscription retries from scratch.
 	*/
-	#openSQLiteCatchup(opts, ctx) {
+	#openSQLiteCatchup(lc, opts, ctx) {
 		let reader;
 		try {
 			reader = new SQLiteChangeLogReader(this.#lc, opts.changeLogFile);
 			if (reader.plan(ctx.watermark).kind === "not-ready") {
 				reader.close();
-				this.#declineSQLiteCatchup(ctx, opts, `${opts.changeLogFile} has no changes to serve yet`);
+				this.#declineSQLiteCatchup(lc, ctx, opts, `${opts.changeLogFile} has no changes to serve yet`);
 				return;
 			}
 		} catch (e) {
 			reader?.close();
-			this.#declineSQLiteCatchup(ctx, opts, `cannot read ${opts.changeLogFile}`, e);
+			this.#declineSQLiteCatchup(lc, ctx, opts, `cannot read ${opts.changeLogFile}`, e);
 			return;
 		}
 		this.#changeLogUnavailableSince = void 0;
@@ -620,7 +746,11 @@ var ChangeStreamerImpl = class {
 			batchSize: opts.readBatchRows,
 			barrierTimeoutMs: opts.barrierTimeoutMs,
 			barrierPollIntervalMs: opts.barrierPollIntervalMs,
-			cleanupGuard: opts.cleanupGuard
+			cleanupGuard: opts.cleanupGuard,
+			onFailure: (failure) => {
+				this.#lc.warn?.(`temporarily disabling SQLite catchup after ${failure}`);
+				this.#readRouter?.trip();
+			}
 		});
 		return this.#sqliteCatchup;
 	}
@@ -629,12 +759,11 @@ var ChangeStreamerImpl = class {
 	* unavailable so that a subscription that merely arrived before the writer's
 	* first reconcile does not look like an incident.
 	*/
-	#declineSQLiteCatchup(ctx, opts, reason, error) {
+	#declineSQLiteCatchup(lc, ctx, opts, reason, error) {
 		const now = Date.now();
 		this.#changeLogUnavailableSince ??= now;
 		const unavailableMs = now - this.#changeLogUnavailableSince;
-		const threshold = opts.notReadyWarnThresholdMs ?? DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS;
-		this.#lc[unavailableMs >= threshold ? "warn" : "debug"]?.(`serving ${ctx.id} from PG catchup: ${reason} (unavailable for ${unavailableMs} ms)`, ...error === void 0 ? [] : [error]);
+		lc[unavailableMs >= (opts.notReadyWarnThresholdMs ?? DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS) ? "warn" : "debug"]?.(`serving ${ctx.id} from PG catchup: ${reason} (unavailable for ${unavailableMs} ms)`, ...error === void 0 ? [] : [error]);
 	}
 };
 var CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;