@rocicorp/zero 1.10.0-canary.2 → 1.10.0-canary.20

package/out/zero-cache/src/services/change-streamer/storer.js

@@ -2,6 +2,7 @@ import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
 import { AbortError } from "../../../../shared/src/abort-error.js";
 import { parse, valita_exports } from "../../../../shared/src/valita.js";
+import { BigIntJSON } from "../../../../shared/src/bigint-json.js";
 import { cdcSchema } from "../../types/shards.js";
 import { isDataChange, isSchemaChange } from "../change-source/protocol/current/data.js";
 import "../change-source/protocol/current/downstream.js";
@@ -13,7 +14,7 @@ import { READONLY, READ_COMMITTED } from "../../db/mode-enum.js";
 import { runTx } from "../../db/run-transaction.js";
 import { Queue } from "../../../../shared/src/queue.js";
 import { AutoResetSignal, markResetRequired } from "./schema/tables.js";
-import { TransactionPool } from "../../db/transaction-pool.js";
+import { TransactionPool, sharedSnapshot } from "../../db/transaction-pool.js";
 import { orTimeout } from "../../types/timeout.js";
 import { extractChangeSubstring, reconstructWatermarkedChange, serializeChangeStreamData } from "./change-log-codec.js";
 import { resolver } from "@rocicorp/resolver";
@@ -58,7 +59,10 @@ var Storer = class {
 	#taskID;
 	#discoveryAddress;
 	#discoveryProtocol;
+	#makeConnectionPool;
 	#db;
+	#inserter;
+	#purger;
 	#replicaVersion;
 	#onCommitted;
 	#onFatal;
@@ -67,21 +71,26 @@ var Storer = class {
 	#statementTimeoutMs;
 	#changeLogBatchSize;
 	#drainTimeoutMs;
+	#progressMonitor;
 	#approximateQueuedBytes = 0;
 	#running = false;
-	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onCommitted, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs = 3e4 }) {
+	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, dbProvider, replicaVersion, onCommitted, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs = 3e4 }) {
 		this.#lc = lc.withContext("component", "change-log");
 		this.#shard = shard;
 		this.#taskID = taskID;
 		this.#discoveryAddress = discoveryAddress;
 		this.#discoveryProtocol = discoveryProtocol;
-		this.#db = db;
+		this.#makeConnectionPool = dbProvider;
+		this.#db = dbProvider("change-stream-init", 3);
+		this.#inserter = dbProvider("change-log-inserter", 1);
+		this.#purger = dbProvider("change-log-purger", 1);
 		this.#replicaVersion = replicaVersion;
 		this.#onCommitted = onCommitted;
 		this.#onFatal = onFatal;
 		this.#statementTimeoutMs = statementTimeoutMs;
 		this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);
 		this.#drainTimeoutMs = drainTimeoutMs;
+		this.#progressMonitor = new ProgressMonitor(lc, statementTimeoutMs, onFatal);
 		const heapStats = getHeapStatistics();
 		this.#backPressureThresholdBytes = (heapStats.heap_size_limit - heapStats.used_heap_size) * backPressureLimitHeapProportion;
 		this.#lc.info?.(`Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of --max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) to absorb upstream spikes`, { heapStats });
@@ -89,6 +98,28 @@ var Storer = class {
 	#cdc(table) {
 		return this.#db(`${cdcSchema(this.#shard)}.${table}`);
 	}
+	/**
+	* Bounds a one-off db call (i.e. not part of the main storer loop or the
+	* background catchup read, which are tracked by the ProgressMonitor
+	* instead) with a plain timeout. This covers calls like
+	* {@link assumeOwnership} and {@link getStartStreamInitializationParameters}
+	* that are made by the caller *before* {@link run()} -- and thus before the
+	* ProgressMonitor's polling starts -- as well as ones made well after,
+	* where a continuously-polling watchdog would be overkill for a single
+	* bounded db round trip.
+	*
+	* Does not cancel `promise` on timeout (there is no cancellation mechanism
+	* for a postgres.js query); it just stops waiting for it, and swallows a
+	* later rejection so it doesn't surface as an unhandled rejection.
+	*/
+	async #withTimeout(name, promise) {
+		const result = await orTimeout(promise, this.#statementTimeoutMs);
+		if (result === "timed-out") {
+			promise.catch(() => {});
+			throw new AbortError(`${name} did not complete within ${this.#statementTimeoutMs}ms`);
+		}
+		return result;
+	}
 	async assumeOwnership(purgeLock) {
 		const db = this.#db;
 		const owner = this.#taskID;
@@ -97,10 +128,10 @@ var Storer = class {
 		const addressWithProtocol = ownerProtocol === "ws" ? ownerAddress : `${ownerProtocol}://${ownerAddress}`;
 		this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);
 		const start = performance.now();
-		await db`UPDATE ${this.#cdc("replicationState")} SET ${db({
+		await this.#withTimeout("assume-ownership", db`UPDATE ${this.#cdc("replicationState")} SET ${db({
 			owner,
 			ownerAddress: addressWithProtocol
-		})}`;
+		})}`);
 		const elapsed = (performance.now() - start).toFixed(2);
 		this.#lc.info?.(`assumed ownership at ${addressWithProtocol} (${elapsed} ms)`);
 		if (purgeLock) purgeLock.release();
@@ -123,7 +154,7 @@ var Storer = class {
 	* snapshot that the SQLite change log continues folding onto.
 	*/
 	async getStartStreamInitializationParameters() {
-		const [[{ lastWatermark }], result, tableMetadata, backfilling] = await runTx(this.#db, (sql) => [
+		const [[{ lastWatermark }], result, tableMetadata, backfilling] = await this.#withTimeout("get-stream-params", runTx(this.#db, (sql) => [
 			sql`
         SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}`,
 			sql`
@@ -146,7 +177,7 @@ var Storer = class {
 			sql`
         SELECT "schema", "table", "column", "backfill" FROM ${this.#cdc("backfilling")}
           ORDER BY "schema" COLLATE "C", "table" COLLATE "C", "column" COLLATE "C"`
-		], { mode: READONLY });
+		], { mode: READONLY }));
 		return {
 			lastWatermark,
 			backfillRequests: parse(result, backfillRequestsSchema),
@@ -157,12 +188,52 @@ var Storer = class {
 		};
 	}
 	async getMinWatermarkForCatchup() {
-		const [{ minWatermark }] = await this.#db`
-      SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}`;
+		const [{ minWatermark }] = await this.#withTimeout("get-min-watermark", this.#db`
+      SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}`);
 		return minWatermark;
 	}
+	/**
+	* Returns the retained Postgres catchup bounds.
+	* `minWatermark` is the oldest transaction, or `null` for a new replica.
+	* `lastWatermark` is the durable head.
+	*/
+	async getCatchupBounds() {
+		const [bounds] = await this.#withTimeout("get-catchup-bounds", this.#db`
+      SELECT
+        (SELECT min(watermark) FROM ${this.#cdc("changeLog")}) as "minWatermark",
+        (SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}) as "lastWatermark"`);
+		return bounds;
+	}
+	/**
+	* Lists at most `limit` committed transaction watermarks in
+	* `(afterWatermark, throughWatermark]`, in ascending order.
+	*/
+	async listCommitWatermarks(afterWatermark, throughWatermark, limit) {
+		return (await this.#db`
+      SELECT watermark FROM ${this.#cdc("changeLog")}
+       WHERE precommit IS NOT NULL
+         AND watermark > ${afterWatermark}
+         AND watermark <= ${throughWatermark}
+       ORDER BY watermark
+       LIMIT ${limit}`).map(({ watermark }) => watermark);
+	}
+	/**
+	* Reads `(afterWatermark, throughWatermark]` in stream order and bounded batches.
+	* This query matches the subscriber catchup query.
+	*
+	* Both queries fail when an escaped NUL reaches `change->'tag'`.
+	* Postgres cannot convert that value to text.
+	*/
+	readCatchupRange(afterWatermark, throughWatermark, batchRows = 2e3) {
+		assert(Number.isSafeInteger(batchRows) && batchRows > 0, "Postgres change log batch size must be a positive safe integer");
+		return this.#db`
+      SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc("changeLog")}
+       WHERE watermark > ${afterWatermark}
+         AND watermark <= ${throughWatermark}
+       ORDER BY watermark, pos`.cursor(batchRows);
+	}
 	purgeRecordsBefore(watermark) {
-		return runTx(this.#db, async (sql) => {
+		return runTx(this.#purger, async (sql) => {
 			await sql`
           SELECT watermark FROM ${this.#cdc("changeLog")}
             ORDER BY watermark, pos LIMIT 1
@@ -214,30 +285,22 @@ var Storer = class {
 		}]);
 	}
 	#readyForMore = null;
-	#backpressureTimeout;
 	readyForMore() {
 		if (!this.#running) return;
 		if (this.#readyForMore === null && this.#approximateQueuedBytes > this.#backPressureThresholdBytes) {
 			this.#lc.warn?.(`applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\n\nTo inspect changeLog backlog in your change DB:\n  SELECT\n    (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\n    change->>'tag' AS operation,\n    COUNT(*) AS count\n  FROM "<app_id>/cdc"."changeLog"\n  GROUP BY 1, 2\n  ORDER BY 3 DESC\n  LIMIT 20;`);
 			this.#readyForMore = resolver();
-			this.#resetBackpressureTimeout(true);
 		}
 		return this.#readyForMore?.promise;
 	}
-	#resetBackpressureTimeout(reschedule) {
-		clearTimeout(this.#backpressureTimeout);
-		this.#backpressureTimeout = reschedule ? setTimeout(() => {
-			this.#readyForMore?.reject(new AbortError(`No statement processed within the last ${this.#statementTimeoutMs}ms. Considering the change-log to be wedged.`));
-			this.#readyForMore = null;
-		}, this.#statementTimeoutMs) : void 0;
-	}
 	#maybeReleaseBackPressure() {
-		if (this.#readyForMore !== null) if (this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
-			this.#lc.info?.(`releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`);
-			this.#readyForMore.resolve();
-			this.#readyForMore = null;
-			this.#resetBackpressureTimeout(false);
-		} else this.#resetBackpressureTimeout(true);
+		if (this.#readyForMore !== null) {
+			if (this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
+				this.#lc.info?.(`releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`);
+				this.#readyForMore.resolve();
+				this.#readyForMore = null;
+			}
+		}
 	}
 	/**
 	* Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.
@@ -276,6 +339,7 @@ var Storer = class {
 		this.#running = true;
 		this.#stopped = stopped;
 		this.#lc.info?.("starting storer");
+		this.#progressMonitor.start();
 		let err;
 		try {
 			await this.#processQueue();
@@ -287,7 +351,6 @@ var Storer = class {
 				this.#readyForMore.resolve();
 				this.#readyForMore = null;
 			}
-			this.#resetBackpressureTimeout(false);
 			this.#cancelQueueEntries(this.#queue.drain().filter((entry) => entry !== void 0), err);
 			this.#running = false;
 			signalStopped();
@@ -313,9 +376,20 @@ var Storer = class {
 	async #processQueue() {
 		let tx = null;
 		let msg;
+		let lastTaskDone;
+		const nextMessage = async () => {
+			lastTaskDone?.();
+			lastTaskDone = void 0;
+			const entry = await this.#queue.dequeue();
+			if (entry !== "stop") lastTaskDone = this.#progressMonitor.trackTask({
+				task: "queue-entry",
+				type: entry[0]
+			});
+			return entry;
+		};
 		const catchupQueue = [];
 		try {
-			while ((msg = await this.#queue.dequeue()) !== "stop") {
+			while ((msg = await nextMessage()) !== "stop") {
 				const [msgType] = msg;
 				switch (msgType) {
 					case "ready": {
@@ -345,10 +419,7 @@ var Storer = class {
 					const { promise, resolve, reject } = resolver();
 					promise.catch(() => {});
 					tx = {
-						pool: new TransactionPool(this.#lc.withContext("watermark", watermark), {
-							mode: READ_COMMITTED,
-							statementResponseTimeout: this.#statementTimeoutMs
-						}),
+						pool: new TransactionPool(this.#lc.withContext("watermark", watermark), { mode: READ_COMMITTED }),
 						preCommitWatermark: watermark,
 						pos: 0,
 						startingReplicationState: promise,
@@ -356,7 +427,7 @@ var Storer = class {
 						batch: [],
 						lastFlush: void 0
 					};
-					tx.pool.run(this.#db);
+					tx.pool.run(this.#inserter);
 					tx.pool.process((tx) => {
 						tx`
           SELECT "owner" FROM ${this.#cdc("replicationState")} FOR UPDATE`.then(([result]) => resolve(result), reject);
@@ -415,22 +486,46 @@ var Storer = class {
 		}
 	}
 	async #startCatchup(subs) {
-		if (subs.length === 0) return;
+		const numCatchups = subs.length;
+		if (numCatchups === 0) return;
 		const lc = this.#lc.withContext("pool", "catchup");
-		const reader = new TransactionPool(lc, { mode: READONLY });
-		reader.run(this.#db);
+		const { init, cleanup, snapshotID } = sharedSnapshot();
+		const reader = new TransactionPool(lc, {
+			mode: READONLY,
+			init,
+			cleanup,
+			initialWorkers: subs.length
+		});
+		const catchupConns = this.#makeConnectionPool("subscriber-catchup", numCatchups);
+		reader.run(catchupConns);
 		let lastWatermark;
+		const catchupSnapshotted = this.#progressMonitor.trackTask({
+			task: "capture-catchup-snapshot",
+			subscribers: subs.map(({ subscriber: s }) => s.id)
+		});
 		try {
 			[{lastWatermark}] = await reader.processReadTask((sql) => sql`
         SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}
       `);
+			lc.info?.(`snapshotted db at ${lastWatermark} (${await snapshotID}) to catchup ${numCatchups} subscriber(s)`);
 		} catch (e) {
 			subs.map(({ subscriber }) => subscriber.fail(e));
 			throw e;
+		} finally {
+			catchupSnapshotted();
 		}
-		Promise.all(subs.map((sub) => this.#catchup(lc.withContext("subscriber", sub.subscriber.id), sub, lastWatermark, reader))).finally(() => reader.setDone());
+		Promise.allSettled(subs.map((sub) => this.#catchup(lc.withContext("subscriber", sub.subscriber.id), sub, lastWatermark, reader))).finally(() => {
+			reader.setDone();
+			catchupConns.end().catch(() => {});
+		});
 	}
 	async #catchup(lc, { subscriber: sub, mode }, lastWatermark, reader) {
+		let entriesReceived = 0;
+		let catchupProgressed = this.#progressMonitor.trackTask({
+			task: "catchup",
+			subscriber: sub.id,
+			entriesReceived
+		});
 		try {
 			lc.info?.(`starting catchup`);
 			await reader.processReadTask(async (tx) => {
@@ -444,21 +539,32 @@ var Storer = class {
            WHERE watermark >= ${sub.watermark}
              AND watermark <= ${lastWatermark}
            ORDER BY watermark, pos`.cursor(2e3)) {
-					const start = performance.now();
-					await lastBatchConsumed;
-					const elapsed = performance.now() - start;
-					if (lastBatchConsumed) lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`);
-					for (const entry of entries) if (entry.watermark === sub.watermark) watermarkFound = true;
-					else if (watermarkFound) {
-						lastBatchConsumed = sub.catchup(reconstructWatermarkedChange(entry));
-						count++;
-					} else if (mode === "backup") throw new AutoResetSignal(`backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`);
-					else {
-						lc.warn?.(`rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`);
-						sub.close(2, `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`);
-						return;
+					catchupProgressed();
+					entriesReceived += entries.length;
+					try {
+						const start = performance.now();
+						await lastBatchConsumed;
+						const elapsed = performance.now() - start;
+						if (lastBatchConsumed) lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`);
+						for (const entry of entries) if (entry.watermark === sub.watermark) watermarkFound = true;
+						else if (watermarkFound) {
+							lastBatchConsumed = sub.catchup(reconstructWatermarkedChange(entry));
+							count++;
+						} else if (mode === "backup") throw new AutoResetSignal(`backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`);
+						else {
+							lc.warn?.(`rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`);
+							sub.close(2, `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`);
+							return;
+						}
+					} finally {
+						catchupProgressed = this.#progressMonitor.trackTask({
+							task: "catchup",
+							subscriber: sub.id,
+							entriesReceived
+						});
 					}
 				}
+				catchupProgressed();
 				if (watermarkFound) {
 					await lastBatchConsumed;
 					lc.info?.(`caught up ${sub.id} with ${count} changes (${Date.now() - start} ms)`);
@@ -472,6 +578,8 @@ var Storer = class {
 				this.#onFatal(err);
 			}
 			sub.fail(err);
+		} finally {
+			catchupProgressed();
 		}
 	}
 	/**
@@ -569,6 +677,7 @@ var Storer = class {
 	*/
 	async stop() {
 		if (this.#running) {
+			this.#progressMonitor.stop();
 			this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);
 			this.abort();
 			this.#queue.enqueue("stop");
@@ -629,6 +738,65 @@ var PurgeLocker = class {
 		return new PurgeLock(this.#lc, tx, replicaVersion, watermark);
 	}
 };
+/**
+* Periodic monitor for detecting the pathological condition in which a db
+* query hangs forever because of a connection problem, such as a half-closed
+* socket.
+*
+* Use this for db calls that recur or accumulate progress within a single
+* logical operation on the main storer loop (e.g. the queue-processing loop,
+* or a multi-batch catchup read), where "no progress for N ms" is the
+* meaningful signal. For a one-off db call -- including ones made before
+* {@link Storer.run} starts this monitor's polling -- use a plain timeout
+* (see `Storer.#withTimeout`) instead.
+*
+* The failure handler should stop the server. The ProgressMonitor will
+* assume this and stop itself after calling it.
+*
+* Internal to the Storer, but exported for testing.
+*/
+var ProgressMonitor = class {
+	#lc;
+	#failureThresholdMs;
+	#onFailure;
+	#taskUpdates = /* @__PURE__ */ new Map();
+	#timer;
+	constructor(lc, failureThresholdMs, onFailure) {
+		this.#lc = lc.withContext("component", "storer-progress-monitor");
+		this.#failureThresholdMs = failureThresholdMs;
+		this.#onFailure = onFailure;
+	}
+	start() {
+		clearInterval(this.#timer);
+		this.#timer = setInterval(() => this.checkTaskProgress(/* @__PURE__ */ new Date()), this.#failureThresholdMs).unref();
+	}
+	checkTaskProgress(now) {
+		for (const [task, lastUpdate] of this.#taskUpdates.entries()) if (now.getTime() - lastUpdate.getTime() >= this.#failureThresholdMs) {
+			this.#lc.error?.(`Last task update was more than ${this.#failureThresholdMs}ms ago`, {
+				task,
+				lastUpdate
+			});
+			this.#onFailure(/* @__PURE__ */ new Error(`Task failed to progress for over ${this.#failureThresholdMs}ms: ${BigIntJSON.stringify(task)}`));
+			this.stop();
+			return;
+		}
+	}
+	/**
+	* Starts a task to be tracked.
+	*
+	* @return a callback to invoke when the task is done.
+	*/
+	trackTask(task, now = /* @__PURE__ */ new Date()) {
+		this.#taskUpdates.set(task, now);
+		return () => {
+			this.#taskUpdates.delete(task);
+		};
+	}
+	stop() {
+		clearInterval(this.#timer);
+		this.#taskUpdates.clear();
+	}
+};
 //#endregion
 export { PurgeLocker, Storer };