@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.10

package/out/zero-cache/src/services/change-streamer/storer.js

@@ -1,4 +1,4 @@
-import { assert } from "../../../../shared/src/asserts.js";
+import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
 import { AbortError } from "../../../../shared/src/abort-error.js";
 import { parse, valita_exports } from "../../../../shared/src/valita.js";
@@ -8,12 +8,15 @@ import { isDataChange, isSchemaChange } from "../change-source/protocol/current/
 import "../change-source/protocol/current/downstream.js";
 import { backfillRequestSchema } from "../change-source/protocol/current/upstream.js";
 import "../change-source/protocol/current.js";
+import { cookieOps } from "../replicator/change-log-cookies.js";
 import "../../types/pg.js";
 import { READONLY, READ_COMMITTED } from "../../db/mode-enum.js";
 import { runTx } from "../../db/run-transaction.js";
-import { AutoResetSignal, markResetRequired } from "./schema/tables.js";
 import { Queue } from "../../../../shared/src/queue.js";
+import { AutoResetSignal, markResetRequired } from "./schema/tables.js";
 import { TransactionPool } from "../../db/transaction-pool.js";
+import { orTimeout } from "../../types/timeout.js";
+import { extractChangeSubstring, reconstructWatermarkedChange, serializeChangeStreamData } from "./change-log-codec.js";
 import { resolver } from "@rocicorp/resolver";
 import "postgres";
 import { getHeapStatistics } from "node:v8";
@@ -58,15 +61,17 @@ var Storer = class {
 	#discoveryProtocol;
 	#db;
 	#replicaVersion;
-	#onConsumed;
+	#onCommitted;
 	#onFatal;
 	#queue = new Queue();
 	#backPressureThresholdBytes;
 	#statementTimeoutMs;
 	#changeLogBatchSize;
+	#drainTimeoutMs;
+	#progressMonitor;
 	#approximateQueuedBytes = 0;
 	#running = false;
-	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onConsumed, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize }) {
+	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onCommitted, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs = 3e4 }) {
 		this.#lc = lc.withContext("component", "change-log");
 		this.#shard = shard;
 		this.#taskID = taskID;
@@ -74,10 +79,12 @@ var Storer = class {
 		this.#discoveryProtocol = discoveryProtocol;
 		this.#db = db;
 		this.#replicaVersion = replicaVersion;
-		this.#onConsumed = onConsumed;
+		this.#onCommitted = onCommitted;
 		this.#onFatal = onFatal;
 		this.#statementTimeoutMs = statementTimeoutMs;
 		this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);
+		this.#drainTimeoutMs = drainTimeoutMs;
+		this.#progressMonitor = new ProgressMonitor(lc, statementTimeoutMs, onFatal);
 		const heapStats = getHeapStatistics();
 		this.#backPressureThresholdBytes = (heapStats.heap_size_limit - heapStats.used_heap_size) * backPressureLimitHeapProportion;
 		this.#lc.info?.(`Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of --max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) to absorb upstream spikes`, { heapStats });
@@ -85,6 +92,28 @@ var Storer = class {
 	#cdc(table) {
 		return this.#db(`${cdcSchema(this.#shard)}.${table}`);
 	}
+	/**
+	* Bounds a one-off db call (i.e. not part of the main storer loop or the
+	* background catchup read, which are tracked by the ProgressMonitor
+	* instead) with a plain timeout. This covers calls like
+	* {@link assumeOwnership} and {@link getStartStreamInitializationParameters}
+	* that are made by the caller *before* {@link run()} -- and thus before the
+	* ProgressMonitor's polling starts -- as well as ones made well after,
+	* where a continuously-polling watchdog would be overkill for a single
+	* bounded db round trip.
+	*
+	* Does not cancel `promise` on timeout (there is no cancellation mechanism
+	* for a postgres.js query); it just stops waiting for it, and swallows a
+	* later rejection so it doesn't surface as an unhandled rejection.
+	*/
+	async #withTimeout(name, promise) {
+		const result = await orTimeout(promise, this.#statementTimeoutMs);
+		if (result === "timed-out") {
+			promise.catch(() => {});
+			throw new AbortError(`${name} did not complete within ${this.#statementTimeoutMs}ms`);
+		}
+		return result;
+	}
 	async assumeOwnership(purgeLock) {
 		const db = this.#db;
 		const owner = this.#taskID;
@@ -93,40 +122,110 @@ var Storer = class {
 		const addressWithProtocol = ownerProtocol === "ws" ? ownerAddress : `${ownerProtocol}://${ownerAddress}`;
 		this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);
 		const start = performance.now();
-		await db`UPDATE ${this.#cdc("replicationState")} SET ${db({
+		await this.#withTimeout("assume-ownership", db`UPDATE ${this.#cdc("replicationState")} SET ${db({
 			owner,
 			ownerAddress: addressWithProtocol
-		})}`;
+		})}`);
 		const elapsed = (performance.now() - start).toFixed(2);
 		this.#lc.info?.(`assumed ownership at ${addressWithProtocol} (${elapsed} ms)`);
 		if (purgeLock) purgeLock.release();
 	}
+	/**
+	* The whole of what a stream connection starts from: where to resume, what to
+	* ask the change source to backfill, and the raw cookie set those requests
+	* were derived from.
+	*
+	* All four statements run in one REPEATABLE READ transaction, so they see one
+	* snapshot. That is invariant 15 — the resume watermark and the cookies must
+	* come from the same store, read at the same position — and it is why the
+	* cookies are read here rather than by a second call: a cookie set paired
+	* with a watermark it was not folded to loses every backfill that completed
+	* in the interval, and those rows are not replayable from the slot.
+	*
+	* `cookies` is not `backfillRequests` in another shape. The request list is
+	* driven off `backfilling`, so it drops the metadata of every table with no
+	* in-flight backfill — correct for starting a stream, lossy as the cookie
+	* snapshot that the SQLite change log continues folding onto.
+	*/
 	async getStartStreamInitializationParameters() {
-		const [[{ lastWatermark }], result] = await runTx(this.#db, (sql) => [sql`
-        SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}`, sql`
-        SELECT 
+		const [[{ lastWatermark }], result, tableMetadata, backfilling] = await this.#withTimeout("get-stream-params", runTx(this.#db, (sql) => [
+			sql`
+        SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}`,
+			sql`
+        SELECT
             json_build_object(
               'schema', b."schema",
               'name', b."table",
               'metadata', t."metadata"
             ) as "table",
-            json_object_agg(b."column", b."backfill") 
+            json_object_agg(b."column", b."backfill")
               as "columns"
           FROM ${this.#cdc("backfilling")} as b
           LEFT JOIN ${this.#cdc("tableMetadata")} as t
           ON (b."schema" = t."schema" AND b."table" = t."table")
           GROUP BY b."schema", b."table", t."metadata"
-        `], { mode: READONLY });
+        `,
+			sql`
+        SELECT "schema", "table", "metadata" FROM ${this.#cdc("tableMetadata")}
+          ORDER BY "schema" COLLATE "C", "table" COLLATE "C"`,
+			sql`
+        SELECT "schema", "table", "column", "backfill" FROM ${this.#cdc("backfilling")}
+          ORDER BY "schema" COLLATE "C", "table" COLLATE "C", "column" COLLATE "C"`
+		], { mode: READONLY }));
 		return {
 			lastWatermark,
-			backfillRequests: parse(result, backfillRequestsSchema)
+			backfillRequests: parse(result, backfillRequestsSchema),
+			cookies: {
+				tableMetadata: [...tableMetadata],
+				backfilling: [...backfilling]
+			}
 		};
 	}
 	async getMinWatermarkForCatchup() {
-		const [{ minWatermark }] = await this.#db`
-      SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}`;
+		const [{ minWatermark }] = await this.#withTimeout("get-min-watermark", this.#db`
+      SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}`);
 		return minWatermark;
 	}
+	/**
+	* Returns the retained Postgres catchup bounds.
+	* `minWatermark` is the oldest transaction, or `null` for a new replica.
+	* `lastWatermark` is the durable head.
+	*/
+	async getCatchupBounds() {
+		const [bounds] = await this.#withTimeout("get-catchup-bounds", this.#db`
+      SELECT
+        (SELECT min(watermark) FROM ${this.#cdc("changeLog")}) as "minWatermark",
+        (SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}) as "lastWatermark"`);
+		return bounds;
+	}
+	/**
+	* Lists at most `limit` committed transaction watermarks in
+	* `(afterWatermark, throughWatermark]`, in ascending order.
+	*/
+	async listCommitWatermarks(afterWatermark, throughWatermark, limit) {
+		return (await this.#db`
+      SELECT watermark FROM ${this.#cdc("changeLog")}
+       WHERE precommit IS NOT NULL
+         AND watermark > ${afterWatermark}
+         AND watermark <= ${throughWatermark}
+       ORDER BY watermark
+       LIMIT ${limit}`).map(({ watermark }) => watermark);
+	}
+	/**
+	* Reads `(afterWatermark, throughWatermark]` in stream order and bounded batches.
+	* This query matches the subscriber catchup query.
+	*
+	* Both queries fail when an escaped NUL reaches `change->'tag'`.
+	* Postgres cannot convert that value to text.
+	*/
+	readCatchupRange(afterWatermark, throughWatermark, batchRows = 2e3) {
+		assert(Number.isSafeInteger(batchRows) && batchRows > 0, "Postgres change log batch size must be a positive safe integer");
+		return this.#db`
+      SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc("changeLog")}
+       WHERE watermark > ${afterWatermark}
+         AND watermark <= ${throughWatermark}
+       ORDER BY watermark, pos`.cursor(batchRows);
+	}
 	purgeRecordsBefore(watermark) {
 		return runTx(this.#db, async (sql) => {
 			await sql`
@@ -159,7 +258,7 @@ var Storer = class {
 	* @returns The JSON stringified stream message to be sent downstream.
 	*/
 	store(watermark, data) {
-		const json = BigIntJSON.stringify(data);
+		const json = serializeChangeStreamData(data);
 		this.#approximateQueuedBytes += json.length;
 		const change = data[1];
 		this.#queue.enqueue([
@@ -173,9 +272,6 @@ var Storer = class {
 	abort() {
 		this.#queue.enqueue(["abort"]);
 	}
-	status(s) {
-		this.#queue.enqueue(s);
-	}
 	catchup(subscriber, mode) {
 		this.#queue.enqueue(["subscriber", {
 			subscriber,
@@ -192,10 +288,12 @@ var Storer = class {
 		return this.#readyForMore?.promise;
 	}
 	#maybeReleaseBackPressure() {
-		if (this.#readyForMore !== null && this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
-			this.#lc.info?.(`releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`);
-			this.#readyForMore.resolve();
-			this.#readyForMore = null;
+		if (this.#readyForMore !== null) {
+			if (this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
+				this.#lc.info?.(`releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`);
+				this.#readyForMore.resolve();
+				this.#readyForMore = null;
+			}
 		}
 	}
 	/**
@@ -235,6 +333,7 @@ var Storer = class {
 		this.#running = true;
 		this.#stopped = stopped;
 		this.#lc.info?.("starting storer");
+		this.#progressMonitor.start();
 		let err;
 		try {
 			await this.#processQueue();
@@ -271,9 +370,20 @@ var Storer = class {
 	async #processQueue() {
 		let tx = null;
 		let msg;
+		let lastTaskDone;
+		const nextMessage = async () => {
+			lastTaskDone?.();
+			lastTaskDone = void 0;
+			const entry = await this.#queue.dequeue();
+			if (entry !== "stop") lastTaskDone = this.#progressMonitor.trackTask({
+				task: "queue-entry",
+				type: entry[0]
+			});
+			return entry;
+		};
 		const catchupQueue = [];
 		try {
-			while ((msg = await this.#queue.dequeue()) !== "stop") {
+			while ((msg = await nextMessage()) !== "stop") {
 				const [msgType] = msg;
 				switch (msgType) {
 					case "ready": {
@@ -287,9 +397,6 @@ var Storer = class {
 						else await this.#startCatchup([subscriber]);
 						continue;
 					}
-					case "status":
-						this.#onConsumed(msg);
-						continue;
 					case "abort":
 						if (tx) {
 							tx.pool.abort();
@@ -306,10 +413,7 @@ var Storer = class {
 					const { promise, resolve, reject } = resolver();
 					promise.catch(() => {});
 					tx = {
-						pool: new TransactionPool(this.#lc.withContext("watermark", watermark), {
-							mode: READ_COMMITTED,
-							statementResponseTimeout: this.#statementTimeoutMs
-						}),
+						pool: new TransactionPool(this.#lc.withContext("watermark", watermark), { mode: READ_COMMITTED }),
 						preCommitWatermark: watermark,
 						pos: 0,
 						startingReplicationState: promise,
@@ -356,7 +460,7 @@ var Storer = class {
 						tx.pool.setDone();
 					}
 					await tx.pool.done();
-					if (tx.ack) this.#onConsumed([
+					if (tx.ack) this.#onCommitted([
 						"commit",
 						change,
 						{ watermark }
@@ -377,9 +481,14 @@ var Storer = class {
 	}
 	async #startCatchup(subs) {
 		if (subs.length === 0) return;
-		const reader = new TransactionPool(this.#lc.withContext("pool", "catchup"), { mode: READONLY });
+		const lc = this.#lc.withContext("pool", "catchup");
+		const reader = new TransactionPool(lc, { mode: READONLY });
 		reader.run(this.#db);
 		let lastWatermark;
+		const catchupSnapshotted = this.#progressMonitor.trackTask({
+			task: "capture-catchup-snapshot",
+			subscribers: subs.map(({ subscriber: s }) => s.id)
+		});
 		try {
 			[{lastWatermark}] = await reader.processReadTask((sql) => sql`
         SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}
@@ -387,12 +496,22 @@ var Storer = class {
 		} catch (e) {
 			subs.map(({ subscriber }) => subscriber.fail(e));
 			throw e;
+		} finally {
+			catchupSnapshotted();
 		}
-		Promise.all(subs.map((sub) => this.#catchup(sub, lastWatermark, reader))).finally(() => reader.setDone());
+		Promise.all(subs.map((sub) => this.#catchup(lc.withContext("subscriber", sub.subscriber.id), sub, lastWatermark, reader))).finally(() => reader.setDone());
 	}
-	async #catchup({ subscriber: sub, mode }, lastWatermark, reader) {
+	async #catchup(lc, { subscriber: sub, mode }, lastWatermark, reader) {
+		let entriesReceived = 0;
+		let catchupProgressed = this.#progressMonitor.trackTask({
+			task: "catchup",
+			subscriber: sub.id,
+			entriesReceived
+		});
 		try {
+			lc.info?.(`starting catchup`);
 			await reader.processReadTask(async (tx) => {
+				lc.info?.(`catching up`);
 				const start = Date.now();
 				let watermarkFound = sub.watermark === this.#replicaVersion;
 				let count = 0;
@@ -402,100 +521,100 @@ var Storer = class {
            WHERE watermark >= ${sub.watermark}
              AND watermark <= ${lastWatermark}
            ORDER BY watermark, pos`.cursor(2e3)) {
-					const start = performance.now();
-					await lastBatchConsumed;
-					const elapsed = performance.now() - start;
-					if (lastBatchConsumed) this.#lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`);
-					for (const entry of entries) if (entry.watermark === sub.watermark) watermarkFound = true;
-					else if (watermarkFound) {
-						lastBatchConsumed = sub.catchup(toDownstream(entry));
-						count++;
-					} else if (mode === "backup") throw new AutoResetSignal(`backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`);
-					else {
-						this.#lc.warn?.(`rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`);
-						sub.close(2, `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`);
-						return;
+					catchupProgressed();
+					entriesReceived += entries.length;
+					try {
+						const start = performance.now();
+						await lastBatchConsumed;
+						const elapsed = performance.now() - start;
+						if (lastBatchConsumed) lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`);
+						for (const entry of entries) if (entry.watermark === sub.watermark) watermarkFound = true;
+						else if (watermarkFound) {
+							lastBatchConsumed = sub.catchup(reconstructWatermarkedChange(entry));
+							count++;
+						} else if (mode === "backup") throw new AutoResetSignal(`backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`);
+						else {
+							lc.warn?.(`rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`);
+							sub.close(2, `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`);
+							return;
+						}
+					} finally {
+						catchupProgressed = this.#progressMonitor.trackTask({
+							task: "catchup",
+							subscriber: sub.id,
+							entriesReceived
+						});
 					}
 				}
+				catchupProgressed();
 				if (watermarkFound) {
 					await lastBatchConsumed;
-					this.#lc.info?.(`caught up ${sub.id} with ${count} changes (${Date.now() - start} ms)`);
-				} else this.#lc.warn?.(`subscriber ${sub.id} at watermark ${sub.watermark} is ahead of the latest durable watermark ${lastWatermark}; waiting for the change DB to catch up`);
+					lc.info?.(`caught up ${sub.id} with ${count} changes (${Date.now() - start} ms)`);
+				} else lc.warn?.(`subscriber ${sub.id} at watermark ${sub.watermark} is ahead of the latest durable watermark ${lastWatermark}; waiting for the change DB to catch up`);
 				sub.setCaughtUp();
 			});
 		} catch (err) {
-			this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);
+			lc.error?.(`error while catching up subscriber ${sub.id}`, err);
 			if (err instanceof AutoResetSignal) {
 				await markResetRequired(this.#db, this.#shard);
 				this.#onFatal(err);
 			}
 			sub.fail(err);
+		} finally {
+			catchupProgressed();
 		}
 	}
 	/**
 	* Returns the db statements necessary to track backfill and table metadata
 	* presented in the `change`, if any.
+	*
+	* This is the Postgres interpreter of {@link cookieOps}, which is also
+	* interpreted against the SQLite change log by `ChangeLogCookieWriter`. The
+	* two stores have to agree on every transition forever, and the failure mode
+	* if they drift is a backfill that is silently never re-requested — so the
+	* decision of what a schema change *means* lives in one place and only its
+	* transport lives here.
 	*/
 	#trackBackfillMetadata(sql, change) {
-		const stmts = [];
-		switch (change.tag) {
-			case "update-table-metadata": {
-				const { table, new: metadata } = change;
-				stmts.push(this.#upsertTableMetadataStmt(sql, table, metadata));
-				break;
-			}
-			case "create-table": {
-				const { spec, metadata, backfill } = change;
-				if (metadata) stmts.push(this.#upsertTableMetadataStmt(sql, spec, metadata));
-				if (backfill) Object.entries(backfill).forEach(([col, backfill]) => {
-					stmts.push(this.#upsertColumnBackfillStmt(sql, spec, col, backfill));
-				});
-				break;
-			}
+		return cookieOps(change).flatMap((op) => this.#cookieStmts(sql, op));
+	}
+	#cookieStmts(sql, op) {
+		switch (op.op) {
+			case "upsert-metadata": return [this.#upsertTableMetadataStmt(sql, op.table, op.metadata)];
+			case "upsert-backfill": return [this.#upsertColumnBackfillStmt(sql, op.table, op.column, op.backfill)];
 			case "rename-table": {
-				const { old } = change;
+				const { old } = op;
 				const row = {
-					schema: change.new.schema,
-					table: change.new.name
+					schema: op.new.schema,
+					table: op.new.name
 				};
-				stmts.push(sql`UPDATE ${this.#cdc("tableMetadata")} SET ${sql(row)}
+				return [sql`UPDATE ${this.#cdc("tableMetadata")} SET ${sql(row)}
                 WHERE "schema" = ${old.schema} AND "table" = ${old.name}`, sql`UPDATE ${this.#cdc("backfilling")} SET ${sql(row)}
-                WHERE "schema" = ${old.schema} AND "table" = ${old.name}`);
-				break;
+                WHERE "schema" = ${old.schema} AND "table" = ${old.name}`];
 			}
 			case "drop-table": {
-				const { id: { schema, name } } = change;
-				stmts.push(sql`DELETE FROM ${this.#cdc("tableMetadata")}
+				const { schema, name } = op.table;
+				return [sql`DELETE FROM ${this.#cdc("tableMetadata")}
                 WHERE "schema" = ${schema} AND "table" = ${name}`, sql`DELETE FROM ${this.#cdc("backfilling")}
-                WHERE "schema" = ${schema} AND "table" = ${name}`);
-				break;
+                WHERE "schema" = ${schema} AND "table" = ${name}`];
 			}
-			case "add-column": {
-				const { table, tableMetadata, column, backfill } = change;
-				if (tableMetadata) stmts.push(this.#upsertTableMetadataStmt(sql, table, tableMetadata));
-				if (backfill) stmts.push(this.#upsertColumnBackfillStmt(sql, table, column.name, backfill));
-				break;
-			}
-			case "update-column": {
-				const { table: { schema, name: table }, old: { name: oldName }, new: { name: newName } } = change;
-				if (oldName !== newName) stmts.push(sql`UPDATE ${this.#cdc("backfilling")} SET "column" = ${newName}
-                WHERE "schema" = ${schema} AND "table" = ${table} AND "column" = ${oldName}`);
-				break;
+			case "rename-column": {
+				const { schema, name: table } = op.table;
+				return [sql`UPDATE ${this.#cdc("backfilling")} SET "column" = ${op.new}
+                WHERE "schema" = ${schema} AND "table" = ${table} AND "column" = ${op.old}`];
 			}
 			case "drop-column": {
-				const { table: { schema, name }, column } = change;
-				stmts.push(sql`DELETE FROM ${this.#cdc("backfilling")}
-                WHERE "schema" = ${schema} AND "table" = ${name} AND "column" = ${column}`);
-				break;
+				const { schema, name } = op.table;
+				return [sql`DELETE FROM ${this.#cdc("backfilling")}
+                WHERE "schema" = ${schema} AND "table" = ${name} AND "column" = ${op.column}`];
 			}
-			case "backfill-completed": {
-				const { relation: { schema, name: table, rowKey }, columns } = change;
-				const cols = [...rowKey.columns, ...columns];
-				stmts.push(sql`DELETE FROM ${this.#cdc("backfilling")}
-                WHERE "schema" = ${schema} AND "table" = ${table} AND "column" IN ${sql(cols)}`);
+			case "complete-backfill": {
+				const { schema, name: table } = op.table;
+				return [sql`DELETE FROM ${this.#cdc("backfilling")}
+                WHERE "schema" = ${schema} AND "table" = ${table} AND "column" IN ${sql([...op.columns])}`];
 			}
+			default: unreachable(op);
 		}
-		return stmts;
 	}
 	#upsertTableMetadataStmt(sql, { schema, name: table }, metadata) {
 		const row = {
@@ -505,7 +624,7 @@ var Storer = class {
 		};
 		return sql`
         INSERT INTO ${this.#cdc("tableMetadata")} ${sql(row)}
-          ON CONFLICT ("schema", "table") 
+          ON CONFLICT ("schema", "table")
           DO UPDATE SET ${sql(row)};
     `;
 	}
@@ -518,7 +637,7 @@ var Storer = class {
 		};
 		return sql`
         INSERT INTO ${this.#cdc("backfilling")} ${sql(row)}
-          ON CONFLICT ("schema", "table", "column") 
+          ON CONFLICT ("schema", "table", "column")
           DO UPDATE SET ${sql(row)};
     `;
 	}
@@ -533,54 +652,21 @@ var Storer = class {
 			await promise;
 		}
 	}
-	stop() {
+	/**
+	* Stops the storer and waits up to a drain timeout for entries to drain,
+	* throwing an exception if it doesn't drain in time, in order to abort
+	* the server when PG (or the connection to it) appears to be wedged.
+	*/
+	async stop() {
 		if (this.#running) {
+			this.#progressMonitor.stop();
 			this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);
+			this.abort();
 			this.#queue.enqueue("stop");
 		}
-		return this.#stopped;
+		if (await orTimeout(this.#stopped, this.#drainTimeoutMs) === "timed-out") throw new AbortError(`changeLog did not drain within ${this.#drainTimeoutMs}ms`);
 	}
 };
-/**
-* Extracts the stringified change message from the stringified
-* stream message (e.g. the second tuple element). This optimization
-* facilitates stringifying (and sharing the result of) the stream
-* message exactly once, but storing only the change message substring
-* in the changeLog for backwards compatibility.
-*/
-function extractChangeSubstring(streamMessageJSON, tag) {
-	switch (tag) {
-		case "begin":
-		case "commit": return streamMessageJSON.substring(streamMessageJSON.indexOf(",") + 1, streamMessageJSON.lastIndexOf(","));
-		default: return streamMessageJSON.substring(streamMessageJSON.indexOf(",") + 1, streamMessageJSON.lastIndexOf("]"));
-	}
-}
-function toDownstream(entry) {
-	const { watermark, change } = entry;
-	const tag = entry.tag;
-	switch (tag) {
-		case "begin": return [
-			watermark,
-			tag,
-			`["begin",${change},{"commitWatermark":"${watermark}"}]`
-		];
-		case "commit": return [
-			watermark,
-			tag,
-			`["commit",${change},{"watermark":"${watermark}"}]`
-		];
-		case "rollback": return [
-			watermark,
-			tag,
-			`["rollback",${change}]`
-		];
-		default: return [
-			watermark,
-			tag,
-			`["data",${change}]`
-		];
-	}
-}
 var PurgeLock = class {
 	#lc;
 	#tx;
@@ -634,6 +720,65 @@ var PurgeLocker = class {
 		return new PurgeLock(this.#lc, tx, replicaVersion, watermark);
 	}
 };
+/**
+* Periodic monitor for detecting the pathological condition in which a db
+* query hangs forever because of a connection problem, such as a half-closed
+* socket.
+*
+* Use this for db calls that recur or accumulate progress within a single
+* logical operation on the main storer loop (e.g. the queue-processing loop,
+* or a multi-batch catchup read), where "no progress for N ms" is the
+* meaningful signal. For a one-off db call -- including ones made before
+* {@link Storer.run} starts this monitor's polling -- use a plain timeout
+* (see `Storer.#withTimeout`) instead.
+*
+* The failure handler should stop the server. The ProgressMonitor will
+* assume this and stop itself after calling it.
+*
+* Internal to the Storer, but exported for testing.
+*/
+var ProgressMonitor = class {
+	#lc;
+	#failureThresholdMs;
+	#onFailure;
+	#taskUpdates = /* @__PURE__ */ new Map();
+	#timer;
+	constructor(lc, failureThresholdMs, onFailure) {
+		this.#lc = lc.withContext("component", "storer-progress-monitor");
+		this.#failureThresholdMs = failureThresholdMs;
+		this.#onFailure = onFailure;
+	}
+	start() {
+		clearInterval(this.#timer);
+		this.#timer = setInterval(() => this.checkTaskProgress(/* @__PURE__ */ new Date()), this.#failureThresholdMs).unref();
+	}
+	checkTaskProgress(now) {
+		for (const [task, lastUpdate] of this.#taskUpdates.entries()) if (now.getTime() - lastUpdate.getTime() >= this.#failureThresholdMs) {
+			this.#lc.error?.(`Last task update was more than ${this.#failureThresholdMs}ms ago`, {
+				task,
+				lastUpdate
+			});
+			this.#onFailure(/* @__PURE__ */ new Error(`Task failed to progress for over ${this.#failureThresholdMs}ms: ${BigIntJSON.stringify(task)}`));
+			this.stop();
+			return;
+		}
+	}
+	/**
+	* Starts a task to be tracked.
+	*
+	* @return a callback to invoke when the task is done.
+	*/
+	trackTask(task, now = /* @__PURE__ */ new Date()) {
+		this.#taskUpdates.set(task, now);
+		return () => {
+			this.#taskUpdates.delete(task);
+		};
+	}
+	stop() {
+		clearInterval(this.#timer);
+		this.#taskUpdates.clear();
+	}
+};
 //#endregion
 export { PurgeLocker, Storer };