@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.2

package/out/zero-cache/src/services/change-streamer/storer.d.ts

@@ -2,9 +2,10 @@ import type { LogContext } from '@rocicorp/logger';
 import { TransactionPool } from '../../db/transaction-pool.ts';
 import { type PostgresDB } from '../../types/pg.ts';
 import { type ShardID } from '../../types/shards.ts';
-import { type BackfillRequest, type Change } from '../change-source/protocol/current.ts';
+import { type BackfillRequest } from '../change-source/protocol/current.ts';
 import { type ChangeStreamData, type Commit } from '../change-source/protocol/current/downstream.ts';
-import type { DownstreamStatusMessage, UpstreamStatusMessage } from '../change-source/protocol/current/status.ts';
+import type { UpstreamStatusMessage } from '../change-source/protocol/current/status.ts';
+import { type CookieSet } from '../replicator/change-log-cookies.ts';
 import type { ReplicatorMode } from '../replicator/replicator.ts';
 import type { Service } from '../service.ts';
 import type { Subscriber } from './subscriber.ts';
@@ -12,6 +13,7 @@ export type TuningOptions = {
     backPressureLimitHeapProportion: number;
     statementTimeoutMs: number;
     changeLogBatchSize: number;
+    drainTimeoutMs?: number | undefined;
 };
 /**
  * Handles the storage of changes and the catchup of subscribers
@@ -46,11 +48,29 @@ export type TuningOptions = {
 export declare class Storer implements Service {
     #private;
     readonly id = "storer";
-    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, db: PostgresDB, replicaVersion: string, onConsumed: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, }: TuningOptions);
+    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, db: PostgresDB, replicaVersion: string, onCommitted: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs, }: TuningOptions);
     assumeOwnership(purgeLock?: PurgeLock | null): Promise<void>;
+    /**
+     * The whole of what a stream connection starts from: where to resume, what to
+     * ask the change source to backfill, and the raw cookie set those requests
+     * were derived from.
+     *
+     * All four statements run in one REPEATABLE READ transaction, so they see one
+     * snapshot. That is invariant 15 — the resume watermark and the cookies must
+     * come from the same store, read at the same position — and it is why the
+     * cookies are read here rather than by a second call: a cookie set paired
+     * with a watermark it was not folded to loses every backfill that completed
+     * in the interval, and those rows are not replayable from the slot.
+     *
+     * `cookies` is not `backfillRequests` in another shape. The request list is
+     * driven off `backfilling`, so it drops the metadata of every table with no
+     * in-flight backfill — correct for starting a stream, lossy as the cookie
+     * snapshot that the SQLite change log continues folding onto.
+     */
     getStartStreamInitializationParameters(): Promise<{
         lastWatermark: string;
         backfillRequests: BackfillRequest[];
+        cookies: CookieSet;
     }>;
     getMinWatermarkForCatchup(): Promise<string | null>;
     purgeRecordsBefore(watermark: string): Promise<number>;
@@ -59,7 +79,6 @@ export declare class Storer implements Service {
      */
     store(watermark: string, data: ChangeStreamData): string;
     abort(): void;
-    status(s: DownstreamStatusMessage): void;
     catchup(subscriber: Subscriber, mode: ReplicatorMode): void;
     readyForMore(): Promise<void> | undefined;
     /**
@@ -72,16 +91,13 @@ export declare class Storer implements Service {
      * This is only used in tests.
      */
     allProcessed(): Promise<void>;
+    /**
+     * Stops the storer and waits up to a drain timeout for entries to drain,
+     * throwing an exception if it doesn't drain in time, in order to abort
+     * the server when PG (or the connection to it) appears to be wedged.
+     */
     stop(): Promise<void>;
 }
-/**
- * Extracts the stringified change message from the stringified
- * stream message (e.g. the second tuple element). This optimization
- * facilitates stringifying (and sharing the result of) the stream
- * message exactly once, but storing only the change message substring
- * in the changeLog for backwards compatibility.
- */
-export declare function extractChangeSubstring(streamMessageJSON: string, tag: Change['tag'] | undefined): string;
 export declare class PurgeLock {
     #private;
     readonly replicaVersion: string;

package/out/zero-cache/src/services/change-streamer/storer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,EAAC,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAKL,KAAK,eAAe,EACpB,KAAK,MAAM,EAKZ,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EACV,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAS3C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAuChD,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAmBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACvD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,GACnB,EAAE,aAAa;IAgCZ,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI;IA2B5C,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;KACrC,CAAC;IAkCI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAOzD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyCtD;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB;IAkB/C,KAAK;IAIL,MAAM,CAAC,CAAC,EAAE,uBAAuB;IAIjC,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAMpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IAyEzC;;;OAGG;IACG,GAAG;IA0eT;;;OAGG;IACG,YAAY;IAQlB,IAAI;CAOL;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,UAmB/B;AA+BD,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAG5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM;IAUb,OAAO;CAWd;AAED,qBAAa,WAAW;;gBAKV,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU;IAWpD,OAAO;CA4Bd"}
+{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,EAAC,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,6CAA6C,CAAC;AACvF,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAc3C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAsChD,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAoBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACxD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa;IAiCZ,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI;IA2BlD;;;;;;;;;;;;;;;;OAgBG;IACG,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;QACpC,OAAO,EAAE,SAAS,CAAC;KACpB,CAAC;IAoDI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAOzD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyCtD;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB;IAkB/C,KAAK;IAIL,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAOpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IA6FzC;;;OAGG;IACG,GAAG;IAmdT;;;OAGG;IACG,YAAY;IAQlB;;;;OAIG;IACG,IAAI;CAcX;AAED,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAG5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM;IAUb,OAAO;CAWd;AAED,qBAAa,WAAW;;gBAKV,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU;IAWpD,OAAO;CA4Bd"}

package/out/zero-cache/src/services/change-streamer/storer.js

@@ -1,19 +1,21 @@
-import { assert } from "../../../../shared/src/asserts.js";
+import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
 import { AbortError } from "../../../../shared/src/abort-error.js";
 import { parse, valita_exports } from "../../../../shared/src/valita.js";
-import { BigIntJSON } from "../../../../shared/src/bigint-json.js";
 import { cdcSchema } from "../../types/shards.js";
 import { isDataChange, isSchemaChange } from "../change-source/protocol/current/data.js";
 import "../change-source/protocol/current/downstream.js";
 import { backfillRequestSchema } from "../change-source/protocol/current/upstream.js";
 import "../change-source/protocol/current.js";
+import { cookieOps } from "../replicator/change-log-cookies.js";
 import "../../types/pg.js";
 import { READONLY, READ_COMMITTED } from "../../db/mode-enum.js";
 import { runTx } from "../../db/run-transaction.js";
-import { AutoResetSignal, markResetRequired } from "./schema/tables.js";
 import { Queue } from "../../../../shared/src/queue.js";
+import { AutoResetSignal, markResetRequired } from "./schema/tables.js";
 import { TransactionPool } from "../../db/transaction-pool.js";
+import { orTimeout } from "../../types/timeout.js";
+import { extractChangeSubstring, reconstructWatermarkedChange, serializeChangeStreamData } from "./change-log-codec.js";
 import { resolver } from "@rocicorp/resolver";
 import "postgres";
 import { getHeapStatistics } from "node:v8";
@@ -58,15 +60,16 @@ var Storer = class {
 	#discoveryProtocol;
 	#db;
 	#replicaVersion;
-	#onConsumed;
+	#onCommitted;
 	#onFatal;
 	#queue = new Queue();
 	#backPressureThresholdBytes;
 	#statementTimeoutMs;
 	#changeLogBatchSize;
+	#drainTimeoutMs;
 	#approximateQueuedBytes = 0;
 	#running = false;
-	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onConsumed, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize }) {
+	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onCommitted, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs = 3e4 }) {
 		this.#lc = lc.withContext("component", "change-log");
 		this.#shard = shard;
 		this.#taskID = taskID;
@@ -74,10 +77,11 @@ var Storer = class {
 		this.#discoveryProtocol = discoveryProtocol;
 		this.#db = db;
 		this.#replicaVersion = replicaVersion;
-		this.#onConsumed = onConsumed;
+		this.#onCommitted = onCommitted;
 		this.#onFatal = onFatal;
 		this.#statementTimeoutMs = statementTimeoutMs;
 		this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);
+		this.#drainTimeoutMs = drainTimeoutMs;
 		const heapStats = getHeapStatistics();
 		this.#backPressureThresholdBytes = (heapStats.heap_size_limit - heapStats.used_heap_size) * backPressureLimitHeapProportion;
 		this.#lc.info?.(`Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of --max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) to absorb upstream spikes`, { heapStats });
@@ -101,25 +105,55 @@ var Storer = class {
 		this.#lc.info?.(`assumed ownership at ${addressWithProtocol} (${elapsed} ms)`);
 		if (purgeLock) purgeLock.release();
 	}
+	/**
+	* The whole of what a stream connection starts from: where to resume, what to
+	* ask the change source to backfill, and the raw cookie set those requests
+	* were derived from.
+	*
+	* All four statements run in one REPEATABLE READ transaction, so they see one
+	* snapshot. That is invariant 15 — the resume watermark and the cookies must
+	* come from the same store, read at the same position — and it is why the
+	* cookies are read here rather than by a second call: a cookie set paired
+	* with a watermark it was not folded to loses every backfill that completed
+	* in the interval, and those rows are not replayable from the slot.
+	*
+	* `cookies` is not `backfillRequests` in another shape. The request list is
+	* driven off `backfilling`, so it drops the metadata of every table with no
+	* in-flight backfill — correct for starting a stream, lossy as the cookie
+	* snapshot that the SQLite change log continues folding onto.
+	*/
 	async getStartStreamInitializationParameters() {
-		const [[{ lastWatermark }], result] = await runTx(this.#db, (sql) => [sql`
-        SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}`, sql`
-        SELECT 
+		const [[{ lastWatermark }], result, tableMetadata, backfilling] = await runTx(this.#db, (sql) => [
+			sql`
+        SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}`,
+			sql`
+        SELECT
             json_build_object(
               'schema', b."schema",
               'name', b."table",
               'metadata', t."metadata"
             ) as "table",
-            json_object_agg(b."column", b."backfill") 
+            json_object_agg(b."column", b."backfill")
               as "columns"
           FROM ${this.#cdc("backfilling")} as b
           LEFT JOIN ${this.#cdc("tableMetadata")} as t
           ON (b."schema" = t."schema" AND b."table" = t."table")
           GROUP BY b."schema", b."table", t."metadata"
-        `], { mode: READONLY });
+        `,
+			sql`
+        SELECT "schema", "table", "metadata" FROM ${this.#cdc("tableMetadata")}
+          ORDER BY "schema" COLLATE "C", "table" COLLATE "C"`,
+			sql`
+        SELECT "schema", "table", "column", "backfill" FROM ${this.#cdc("backfilling")}
+          ORDER BY "schema" COLLATE "C", "table" COLLATE "C", "column" COLLATE "C"`
+		], { mode: READONLY });
 		return {
 			lastWatermark,
-			backfillRequests: parse(result, backfillRequestsSchema)
+			backfillRequests: parse(result, backfillRequestsSchema),
+			cookies: {
+				tableMetadata: [...tableMetadata],
+				backfilling: [...backfilling]
+			}
 		};
 	}
 	async getMinWatermarkForCatchup() {
@@ -159,7 +193,7 @@ var Storer = class {
 	* @returns The JSON stringified stream message to be sent downstream.
 	*/
 	store(watermark, data) {
-		const json = BigIntJSON.stringify(data);
+		const json = serializeChangeStreamData(data);
 		this.#approximateQueuedBytes += json.length;
 		const change = data[1];
 		this.#queue.enqueue([
@@ -173,9 +207,6 @@ var Storer = class {
 	abort() {
 		this.#queue.enqueue(["abort"]);
 	}
-	status(s) {
-		this.#queue.enqueue(s);
-	}
 	catchup(subscriber, mode) {
 		this.#queue.enqueue(["subscriber", {
 			subscriber,
@@ -183,20 +214,30 @@ var Storer = class {
 		}]);
 	}
 	#readyForMore = null;
+	#backpressureTimeout;
 	readyForMore() {
 		if (!this.#running) return;
 		if (this.#readyForMore === null && this.#approximateQueuedBytes > this.#backPressureThresholdBytes) {
 			this.#lc.warn?.(`applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\n\nTo inspect changeLog backlog in your change DB:\n  SELECT\n    (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\n    change->>'tag' AS operation,\n    COUNT(*) AS count\n  FROM "<app_id>/cdc"."changeLog"\n  GROUP BY 1, 2\n  ORDER BY 3 DESC\n  LIMIT 20;`);
 			this.#readyForMore = resolver();
+			this.#resetBackpressureTimeout(true);
 		}
 		return this.#readyForMore?.promise;
 	}
+	#resetBackpressureTimeout(reschedule) {
+		clearTimeout(this.#backpressureTimeout);
+		this.#backpressureTimeout = reschedule ? setTimeout(() => {
+			this.#readyForMore?.reject(new AbortError(`No statement processed within the last ${this.#statementTimeoutMs}ms. Considering the change-log to be wedged.`));
+			this.#readyForMore = null;
+		}, this.#statementTimeoutMs) : void 0;
+	}
 	#maybeReleaseBackPressure() {
-		if (this.#readyForMore !== null && this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
+		if (this.#readyForMore !== null) if (this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
 			this.#lc.info?.(`releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`);
 			this.#readyForMore.resolve();
 			this.#readyForMore = null;
-		}
+			this.#resetBackpressureTimeout(false);
+		} else this.#resetBackpressureTimeout(true);
 	}
 	/**
 	* Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.
@@ -246,6 +287,7 @@ var Storer = class {
 				this.#readyForMore.resolve();
 				this.#readyForMore = null;
 			}
+			this.#resetBackpressureTimeout(false);
 			this.#cancelQueueEntries(this.#queue.drain().filter((entry) => entry !== void 0), err);
 			this.#running = false;
 			signalStopped();
@@ -287,9 +329,6 @@ var Storer = class {
 						else await this.#startCatchup([subscriber]);
 						continue;
 					}
-					case "status":
-						this.#onConsumed(msg);
-						continue;
 					case "abort":
 						if (tx) {
 							tx.pool.abort();
@@ -356,7 +395,7 @@ var Storer = class {
 						tx.pool.setDone();
 					}
 					await tx.pool.done();
-					if (tx.ack) this.#onConsumed([
+					if (tx.ack) this.#onCommitted([
 						"commit",
 						change,
 						{ watermark }
@@ -377,7 +416,8 @@ var Storer = class {
 	}
 	async #startCatchup(subs) {
 		if (subs.length === 0) return;
-		const reader = new TransactionPool(this.#lc.withContext("pool", "catchup"), { mode: READONLY });
+		const lc = this.#lc.withContext("pool", "catchup");
+		const reader = new TransactionPool(lc, { mode: READONLY });
 		reader.run(this.#db);
 		let lastWatermark;
 		try {
@@ -388,11 +428,13 @@ var Storer = class {
 			subs.map(({ subscriber }) => subscriber.fail(e));
 			throw e;
 		}
-		Promise.all(subs.map((sub) => this.#catchup(sub, lastWatermark, reader))).finally(() => reader.setDone());
+		Promise.all(subs.map((sub) => this.#catchup(lc.withContext("subscriber", sub.subscriber.id), sub, lastWatermark, reader))).finally(() => reader.setDone());
 	}
-	async #catchup({ subscriber: sub, mode }, lastWatermark, reader) {
+	async #catchup(lc, { subscriber: sub, mode }, lastWatermark, reader) {
 		try {
+			lc.info?.(`starting catchup`);
 			await reader.processReadTask(async (tx) => {
+				lc.info?.(`catching up`);
 				const start = Date.now();
 				let watermarkFound = sub.watermark === this.#replicaVersion;
 				let count = 0;
@@ -405,26 +447,26 @@ var Storer = class {
 					const start = performance.now();
 					await lastBatchConsumed;
 					const elapsed = performance.now() - start;
-					if (lastBatchConsumed) this.#lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`);
+					if (lastBatchConsumed) lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`);
 					for (const entry of entries) if (entry.watermark === sub.watermark) watermarkFound = true;
 					else if (watermarkFound) {
-						lastBatchConsumed = sub.catchup(toDownstream(entry));
+						lastBatchConsumed = sub.catchup(reconstructWatermarkedChange(entry));
 						count++;
 					} else if (mode === "backup") throw new AutoResetSignal(`backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`);
 					else {
-						this.#lc.warn?.(`rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`);
+						lc.warn?.(`rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`);
 						sub.close(2, `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`);
 						return;
 					}
 				}
 				if (watermarkFound) {
 					await lastBatchConsumed;
-					this.#lc.info?.(`caught up ${sub.id} with ${count} changes (${Date.now() - start} ms)`);
-				} else this.#lc.warn?.(`subscriber ${sub.id} at watermark ${sub.watermark} is ahead of the latest durable watermark ${lastWatermark}; waiting for the change DB to catch up`);
+					lc.info?.(`caught up ${sub.id} with ${count} changes (${Date.now() - start} ms)`);
+				} else lc.warn?.(`subscriber ${sub.id} at watermark ${sub.watermark} is ahead of the latest durable watermark ${lastWatermark}; waiting for the change DB to catch up`);
 				sub.setCaughtUp();
 			});
 		} catch (err) {
-			this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);
+			lc.error?.(`error while catching up subscriber ${sub.id}`, err);
 			if (err instanceof AutoResetSignal) {
 				await markResetRequired(this.#db, this.#shard);
 				this.#onFatal(err);
@@ -435,67 +477,54 @@ var Storer = class {
 	/**
 	* Returns the db statements necessary to track backfill and table metadata
 	* presented in the `change`, if any.
+	*
+	* This is the Postgres interpreter of {@link cookieOps}, which is also
+	* interpreted against the SQLite change log by `ChangeLogCookieWriter`. The
+	* two stores have to agree on every transition forever, and the failure mode
+	* if they drift is a backfill that is silently never re-requested — so the
+	* decision of what a schema change *means* lives in one place and only its
+	* transport lives here.
 	*/
 	#trackBackfillMetadata(sql, change) {
-		const stmts = [];
-		switch (change.tag) {
-			case "update-table-metadata": {
-				const { table, new: metadata } = change;
-				stmts.push(this.#upsertTableMetadataStmt(sql, table, metadata));
-				break;
-			}
-			case "create-table": {
-				const { spec, metadata, backfill } = change;
-				if (metadata) stmts.push(this.#upsertTableMetadataStmt(sql, spec, metadata));
-				if (backfill) Object.entries(backfill).forEach(([col, backfill]) => {
-					stmts.push(this.#upsertColumnBackfillStmt(sql, spec, col, backfill));
-				});
-				break;
-			}
+		return cookieOps(change).flatMap((op) => this.#cookieStmts(sql, op));
+	}
+	#cookieStmts(sql, op) {
+		switch (op.op) {
+			case "upsert-metadata": return [this.#upsertTableMetadataStmt(sql, op.table, op.metadata)];
+			case "upsert-backfill": return [this.#upsertColumnBackfillStmt(sql, op.table, op.column, op.backfill)];
 			case "rename-table": {
-				const { old } = change;
+				const { old } = op;
 				const row = {
-					schema: change.new.schema,
-					table: change.new.name
+					schema: op.new.schema,
+					table: op.new.name
 				};
-				stmts.push(sql`UPDATE ${this.#cdc("tableMetadata")} SET ${sql(row)}
+				return [sql`UPDATE ${this.#cdc("tableMetadata")} SET ${sql(row)}
                 WHERE "schema" = ${old.schema} AND "table" = ${old.name}`, sql`UPDATE ${this.#cdc("backfilling")} SET ${sql(row)}
-                WHERE "schema" = ${old.schema} AND "table" = ${old.name}`);
-				break;
+                WHERE "schema" = ${old.schema} AND "table" = ${old.name}`];
 			}
 			case "drop-table": {
-				const { id: { schema, name } } = change;
-				stmts.push(sql`DELETE FROM ${this.#cdc("tableMetadata")}
+				const { schema, name } = op.table;
+				return [sql`DELETE FROM ${this.#cdc("tableMetadata")}
                 WHERE "schema" = ${schema} AND "table" = ${name}`, sql`DELETE FROM ${this.#cdc("backfilling")}
-                WHERE "schema" = ${schema} AND "table" = ${name}`);
-				break;
+                WHERE "schema" = ${schema} AND "table" = ${name}`];
 			}
-			case "add-column": {
-				const { table, tableMetadata, column, backfill } = change;
-				if (tableMetadata) stmts.push(this.#upsertTableMetadataStmt(sql, table, tableMetadata));
-				if (backfill) stmts.push(this.#upsertColumnBackfillStmt(sql, table, column.name, backfill));
-				break;
-			}
-			case "update-column": {
-				const { table: { schema, name: table }, old: { name: oldName }, new: { name: newName } } = change;
-				if (oldName !== newName) stmts.push(sql`UPDATE ${this.#cdc("backfilling")} SET "column" = ${newName}
-                WHERE "schema" = ${schema} AND "table" = ${table} AND "column" = ${oldName}`);
-				break;
+			case "rename-column": {
+				const { schema, name: table } = op.table;
+				return [sql`UPDATE ${this.#cdc("backfilling")} SET "column" = ${op.new}
+                WHERE "schema" = ${schema} AND "table" = ${table} AND "column" = ${op.old}`];
 			}
 			case "drop-column": {
-				const { table: { schema, name }, column } = change;
-				stmts.push(sql`DELETE FROM ${this.#cdc("backfilling")}
-                WHERE "schema" = ${schema} AND "table" = ${name} AND "column" = ${column}`);
-				break;
+				const { schema, name } = op.table;
+				return [sql`DELETE FROM ${this.#cdc("backfilling")}
+                WHERE "schema" = ${schema} AND "table" = ${name} AND "column" = ${op.column}`];
 			}
-			case "backfill-completed": {
-				const { relation: { schema, name: table, rowKey }, columns } = change;
-				const cols = [...rowKey.columns, ...columns];
-				stmts.push(sql`DELETE FROM ${this.#cdc("backfilling")}
-                WHERE "schema" = ${schema} AND "table" = ${table} AND "column" IN ${sql(cols)}`);
+			case "complete-backfill": {
+				const { schema, name: table } = op.table;
+				return [sql`DELETE FROM ${this.#cdc("backfilling")}
+                WHERE "schema" = ${schema} AND "table" = ${table} AND "column" IN ${sql([...op.columns])}`];
 			}
+			default: unreachable(op);
 		}
-		return stmts;
 	}
 	#upsertTableMetadataStmt(sql, { schema, name: table }, metadata) {
 		const row = {
@@ -505,7 +534,7 @@ var Storer = class {
 		};
 		return sql`
         INSERT INTO ${this.#cdc("tableMetadata")} ${sql(row)}
-          ON CONFLICT ("schema", "table") 
+          ON CONFLICT ("schema", "table")
           DO UPDATE SET ${sql(row)};
     `;
 	}
@@ -518,7 +547,7 @@ var Storer = class {
 		};
 		return sql`
         INSERT INTO ${this.#cdc("backfilling")} ${sql(row)}
-          ON CONFLICT ("schema", "table", "column") 
+          ON CONFLICT ("schema", "table", "column")
           DO UPDATE SET ${sql(row)};
     `;
 	}
@@ -533,54 +562,20 @@ var Storer = class {
 			await promise;
 		}
 	}
-	stop() {
+	/**
+	* Stops the storer and waits up to a drain timeout for entries to drain,
+	* throwing an exception if it doesn't drain in time, in order to abort
+	* the server when PG (or the connection to it) appears to be wedged.
+	*/
+	async stop() {
 		if (this.#running) {
 			this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);
+			this.abort();
 			this.#queue.enqueue("stop");
 		}
-		return this.#stopped;
+		if (await orTimeout(this.#stopped, this.#drainTimeoutMs) === "timed-out") throw new AbortError(`changeLog did not drain within ${this.#drainTimeoutMs}ms`);
 	}
 };
-/**
-* Extracts the stringified change message from the stringified
-* stream message (e.g. the second tuple element). This optimization
-* facilitates stringifying (and sharing the result of) the stream
-* message exactly once, but storing only the change message substring
-* in the changeLog for backwards compatibility.
-*/
-function extractChangeSubstring(streamMessageJSON, tag) {
-	switch (tag) {
-		case "begin":
-		case "commit": return streamMessageJSON.substring(streamMessageJSON.indexOf(",") + 1, streamMessageJSON.lastIndexOf(","));
-		default: return streamMessageJSON.substring(streamMessageJSON.indexOf(",") + 1, streamMessageJSON.lastIndexOf("]"));
-	}
-}
-function toDownstream(entry) {
-	const { watermark, change } = entry;
-	const tag = entry.tag;
-	switch (tag) {
-		case "begin": return [
-			watermark,
-			tag,
-			`["begin",${change},{"commitWatermark":"${watermark}"}]`
-		];
-		case "commit": return [
-			watermark,
-			tag,
-			`["commit",${change},{"watermark":"${watermark}"}]`
-		];
-		case "rollback": return [
-			watermark,
-			tag,
-			`["rollback",${change}]`
-		];
-		default: return [
-			watermark,
-			tag,
-			`["data",${change}]`
-		];
-	}
-}
 var PurgeLock = class {
 	#lc;
 	#tx;