@rocicorp/zero 1.11.0-canary.2 → 1.11.0-canary.21

package/out/zero-cache/src/services/change-source/common/backfill-manager.js

@@ -1,9 +1,12 @@
 import { assert } from "../../../../../shared/src/asserts.js";
+import { AbortError } from "../../../../../shared/src/abort-error.js";
 import { must } from "../../../../../shared/src/must.js";
+import { promiseOrAbort } from "../../../../../shared/src/promise-race.js";
 import { stringify } from "../../../../../shared/src/bigint-json.js";
 import "../../../types/lite.js";
 import { randInt } from "../../../../../shared/src/rand.js";
 import { stateVersionFromString, stateVersionToString } from "../../../types/state-version.js";
+import { Queue } from "../../../../../shared/src/queue.js";
 import { CustomKeyMap } from "../../../../../shared/src/custom-key-map.js";
 import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-source/common/backfill-manager.ts
@@ -16,6 +19,34 @@ function tableKey({ schema, name }) {
 * COMMIT/checkpoint so it can't monopolize the replica.
 */
 var COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;
+/**
+* How many backfill messages the prefetch producer may read ahead of the
+* reservation-holding consumer loop. This bounds the in-memory buffer (~this
+* many {@link BackfillMessage}s, each up to the stream's flush threshold, e.g.
+* 64 KiB) while giving the consumer a supply of ready messages so it never
+* has to wait on the upstream COPY while holding the change-stream reservation.
+*/
+var BACKFILL_PREFETCH_CREDITS = 16;
+/**
+* While the backfill holds an open transaction but its prefetch queue is empty
+* (i.e. the upstream is momentarily behind), the consumer polls for the next
+* message at this interval so that a replication transaction that begins
+* waiting mid-fetch can preempt the backfill within this bound, rather than
+* being blocked for the entire (possibly slow) upstream read.
+*
+* If stats indicate that these stalls are highly correlative with replication
+* lag (indicating that replication changes are waiting on these polls), we
+* can address this by adding an API to the change stream multiplexer that
+* would allow backfill to relinquish an in-progress commit on demand (i.e.
+* only if replication asks for a reservation).
+*/
+var BACKFILL_WAITER_POLL_MS = 10;
+/** Interval at which in-progress backfill timing stats are logged. */
+var BACKFILL_STATS_LOG_INTERVAL_MS = 1e4;
+/** Sentinel enqueued by the prefetch producer when the backfill stream ends. */
+var DONE = Symbol("backfill-done");
+/** Sentinel returned by a polled dequeue that timed out (see {@link BACKFILL_WAITER_POLL_MS}). */
+var POLL = Symbol("backfill-poll");
 var MIN_BACKOFF_INTERVAL_MS = 2e3;
 var MAX_BACKOFF_INTERVAL_MS = 6e4;
 /**
@@ -57,6 +88,16 @@ var BackfillManager = class {
 	/** The watermark of the current transaction in the change stream. */
 	#currentTxWatermark = null;
 	#commitThresholdBytes;
+	/**
+	* Aborted when the change stream is canceled. No further backfills are run.
+	*
+	* TODO: Thread this into the backfillStream so that upstream I/O can be
+	* aborted.
+	*/
+	#controller = new AbortController();
+	get #canceled() {
+		return this.#controller.signal.aborted;
+	}
 	constructor(lc, changeStreamer, backfillStreamer, jsonFormat = "s", minBackoffMs = MIN_BACKOFF_INTERVAL_MS, maxBackoffMs = MAX_BACKOFF_INTERVAL_MS, commitThresholdBytes = COMMIT_THRESHOLD_BYTES) {
 		this.#lc = lc.withContext("component", "backfill-manager");
 		this.#changeStreamer = changeStreamer;
@@ -102,12 +143,13 @@ var BackfillManager = class {
 	#retryDelayMs;
 	#backfillRetryTimer;
 	#checkAndStartBackfill() {
-		if (!this.#backfillRetryTimer && !this.#runningBackfill && this.#requiredBackfills.size) {
+		if (!this.#canceled && !this.#backfillRetryTimer && !this.#runningBackfill && this.#requiredBackfills.size) {
 			const candidates = [...this.#requiredBackfills.values()];
 			const request = candidates[randInt(0, candidates.length - 1)];
 			const state = {
 				request,
-				minWatermark: ""
+				minWatermark: "",
+				stopped: new AbortController()
 			};
 			const lc = this.#lc.withContext("table", request.table.name);
 			this.#runningBackfill = state;
@@ -121,8 +163,11 @@ var BackfillManager = class {
 		}
 	}
 	#retryBackfillWithBackoff(e) {
-		const log = this.#retryDelayMs === this.#maxBackoffMs ? "error" : "warn";
-		this.#lc[log]?.(`Error running backfill. Retrying in ${this.#retryDelayMs} ms`, e);
+		if (this.#canceled) {
+			this.#lc.debug?.(`not retrying backfill: change stream canceled`, e);
+			return;
+		}
+		this.#lc.warn?.(`Error running backfill. Retrying in ${this.#retryDelayMs} ms`, e);
 		this.#backfillRetryTimer = setTimeout(() => {
 			this.#backfillRetryTimer = void 0;
 			this.#checkAndStartBackfill();
@@ -133,14 +178,27 @@ var BackfillManager = class {
 		const changeStream = this.#changeStreamer;
 		let backfillTx = null;
 		let uncommittedBytes = 0;
+		const stats = {
+			start: performance.now(),
+			lastLogged: performance.now(),
+			messages: 0,
+			bytes: 0,
+			commits: 0,
+			stalls: 0,
+			produceMs: 0,
+			pushMs: 0,
+			reserveMs: 0,
+			stallMs: 0,
+			waitMs: 0
+		};
 		/**
 		* @returns the new tx watermark, or null if backfill was cancelled
 		*/
 		const beginTxFor = async (msg) => {
 			assert(backfillTx === null, "Expected no active backfill transaction");
 			const lastWatermark = await changeStream.reserve("backfill");
-			if (state.canceledReason || msg.tag === "backfill" && msg.watermark < state.minWatermark) {
-				if (state.canceledReason === void 0) {
+			if (state.stopped.signal.aborted || msg.tag === "backfill" && msg.watermark < state.minWatermark) {
+				if (!state.stopped.signal.aborted) {
 					assert(msg.tag === "backfill", "Expected backfill message tag");
 					this.#stopRunningBackfill(`row key change at ${state.minWatermark} postdates backfill watermark at ${msg.watermark}`, state);
 				}
@@ -152,7 +210,7 @@ var BackfillManager = class {
 				major,
 				minor: BigInt(minor) + 1n
 			});
-			if (msg.tag === "backfill-completed" && tx < msg.watermark) tx = msg.watermark;
+			if (tx < msg.watermark) tx = msg.watermark;
 			changeStream.push([
 				"begin",
 				{
@@ -172,25 +230,138 @@ var BackfillManager = class {
 					{ watermark: backfillTx }
 				]);
 				changeStream.release(backfillTx);
+				stats.commits++;
 			}
 			backfillTx = null;
 			uncommittedBytes = 0;
 		};
-		for await (const { message: msg, byteSize } of this.#backfillStreamer(state.request)) {
-			const mustWaitBeforeFlush = msg.tag === "backfill-completed" && (this.#changeStreamReached(lc, msg.watermark) || backfillTx !== null && backfillTx < msg.watermark);
-			if (backfillTx && (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush || uncommittedBytes >= this.#commitThresholdBytes)) commitTx();
-			mustWaitBeforeFlush && await mustWaitBeforeFlush;
-			if (msg.tag === "backfill" && msg.rowValues.length > 0 && msg.relation.rowKey.columns.length === 0) throw new MissingRowKeyError(state.request);
-			if ((backfillTx ??= await beginTxFor(msg)) === null) {
-				lc.info?.(`backfill stream canceled: ${state.canceledReason}`, state.request);
-				this.#checkAndStartBackfill();
-				return;
+		const ready = new Queue();
+		const credits = new Queue();
+		for (let i = 0; i < BACKFILL_PREFETCH_CREDITS; i++) credits.enqueue(1);
+		let stopProducer = false;
+		const onCancel = () => ready.enqueueRejection(new AbortError("backfill canceled"));
+		if (!this.#canceled) this.#controller.signal.addEventListener("abort", onCancel);
+		const producer = (async () => {
+			try {
+				let t0 = performance.now();
+				for await (const value of this.#backfillStreamer(state.request)) {
+					stats.produceMs += performance.now() - t0;
+					await credits.dequeue();
+					if (stopProducer || this.#canceled) {
+						ready.enqueueRejection(new AbortError("backfill canceled"));
+						return;
+					}
+					ready.enqueue(value);
+					t0 = performance.now();
+				}
+				ready.enqueue(DONE);
+			} catch (e) {
+				ready.enqueueRejection(e);
+			}
+		})();
+		producer.catch(() => {});
+		const nextItem = async () => {
+			if (backfillTx === null || ready.size() > 0) {
+				const n = ready.dequeue();
+				return n instanceof Promise ? await n : n;
+			}
+			for (;;) {
+				if (backfillTx !== null && changeStream.waiterDelay() > 0) commitTx();
+				const n = backfillTx !== null ? ready.dequeue(POLL, BACKFILL_WAITER_POLL_MS) : ready.dequeue();
+				const t0 = performance.now();
+				const v = n instanceof Promise ? await n : n;
+				if (v === POLL) {
+					stats.stallMs += performance.now() - t0;
+					continue;
+				}
+				if (n instanceof Promise) {
+					stats.stallMs += performance.now() - t0;
+					stats.stalls++;
+				}
+				return v;
+			}
+		};
+		try {
+			for (;;) {
+				const item = await nextItem();
+				if (item === DONE) break;
+				credits.enqueue(1);
+				if (this.#canceled) {
+					lc.info?.(`backfill stream canceled: change stream canceled`);
+					return;
+				}
+				const { message: msg, byteSize } = item;
+				const waitForChangeStream = this.#changeStreamReached(lc, msg.watermark);
+				if (backfillTx && (changeStream.waiterDelay() > 0 || waitForChangeStream || uncommittedBytes >= this.#commitThresholdBytes)) commitTx();
+				if (waitForChangeStream) {
+					const t0 = performance.now();
+					await promiseOrAbort(waitForChangeStream, state.stopped.signal);
+					const elapsed = performance.now() - t0;
+					stats.waitMs += elapsed;
+					lc.info?.(`change stream reached ${msg.watermark} after waiting ${Math.round(elapsed)} ms`);
+				}
+				if (msg.tag === "backfill" && msg.rowValues.length > 0 && msg.relation.rowKey.columns.length === 0) throw new MissingRowKeyError(state.request);
+				if (backfillTx === null) {
+					const t0 = performance.now();
+					backfillTx = await beginTxFor(msg);
+					stats.reserveMs += performance.now() - t0;
+					if (backfillTx === null) {
+						lc.info?.(`backfill stream canceled: ${state.stopped.signal.reason}`, state.request);
+						this.#checkAndStartBackfill();
+						return;
+					}
+				}
+				const t0 = performance.now();
+				await changeStream.push(["data", msg]);
+				stats.pushMs += performance.now() - t0;
+				uncommittedBytes += byteSize;
+				stats.messages++;
+				stats.bytes += byteSize;
+				this.#maybeLogBackfillStats(lc, stats);
 			}
-			await changeStream.push(["data", msg]);
-			uncommittedBytes += byteSize;
+			if (backfillTx) commitTx();
+			lc.debug?.(`backfill stream exited`);
+		} finally {
+			stopProducer = true;
+			credits.enqueue(1);
+			if (!this.#canceled) await producer;
+			this.#controller.signal.removeEventListener("abort", onCancel);
+			this.#logBackfillStats(lc, stats);
+		}
+	}
+	#maybeLogBackfillStats(lc, stats) {
+		const now = performance.now();
+		if (now - stats.lastLogged >= BACKFILL_STATS_LOG_INTERVAL_MS) {
+			this.#logBackfillStats(lc, stats);
+			stats.lastLogged = now;
 		}
-		backfillTx && commitTx();
-		lc.debug?.(`backfill stream exited`, state.canceledReason ?? "");
+	}
+	/**
+	* Logs where the backfill spent its time. `produceMs` (upstream read +
+	* parse/decode) accrues on the prefetch producer concurrently with the
+	* consumer's `pushMs` (downstream backpressure), so the two can overlap and
+	* their sum can exceed `elapsedMs`. `reserveMs` is time spent (re)acquiring
+	* the change-stream reservation, and `stallMs` is time the consumer waited on
+	* an empty prefetch queue (i.e. blocked on the upstream) — a large `stallMs`
+	* indicates an upstream-bound backfill. `waitMs` is time spent waiting
+	* (without the reservation) for the change stream to reach the backfill
+	* watermark, during which the upstream snapshot is held open.
+	*/
+	#logBackfillStats(lc, stats) {
+		const elapsed = performance.now() - stats.start;
+		lc.info?.(`backfill timing`, {
+			messages: stats.messages,
+			bytes: stats.bytes,
+			commits: stats.commits,
+			stalls: stats.stalls,
+			elapsedMs: Math.round(elapsed),
+			produceMs: Math.round(stats.produceMs),
+			pushMs: Math.round(stats.pushMs),
+			reserveMs: Math.round(stats.reserveMs),
+			stallMs: Math.round(stats.stallMs),
+			waitMs: Math.round(stats.waitMs),
+			throughputKBps: elapsed > 0 ? Math.round(stats.bytes / 1024 / (elapsed / 1e3)) : 0
+		});
 	}
 	#backfillRunningFor(table) {
 		const state = this.#runningBackfill;
@@ -205,7 +376,7 @@ var BackfillManager = class {
 	#stopRunningBackfill(reason, instance) {
 		const backfill = this.#runningBackfill;
 		if (backfill && backfill === (instance ?? backfill)) {
-			backfill.canceledReason = reason;
+			backfill.stopped.abort(reason);
 			this.#runningBackfill = null;
 			reason && this.#lc.info?.(`canceling backfill:`, reason);
 		}
@@ -269,7 +440,7 @@ var BackfillManager = class {
 						name,
 						metadata
 					},
-					columns: backfill
+					columns: Object.fromEntries(Object.entries(backfill).map(([col, id]) => [col, { id }]))
 				});
 				break;
 			}
@@ -308,7 +479,7 @@ var BackfillManager = class {
 							...table,
 							metadata
 						},
-						columns: { [column.name]: backfill }
+						columns: { [column.name]: { id: backfill } }
 					});
 					else this.#setRequiredBackfill(tag, {
 						...backfillRequest,
@@ -318,7 +489,7 @@ var BackfillManager = class {
 						},
 						columns: {
 							...backfillRequest.columns,
-							[column.name]: backfill
+							[column.name]: { id: backfill }
 						}
 					});
 				}
@@ -387,8 +558,11 @@ var BackfillManager = class {
 		}
 	}
 	cancel() {
+		this.#controller.abort("canceled");
 		this.#stopRunningBackfill(`change stream canceled`);
 		clearTimeout(this.#backfillRetryTimer);
+		this.#backfillRetryTimer = void 0;
+		for (const { reached } of this.#awaitingStatusWatermarks.splice(0)) reached();
 	}
 };
 var BackfillStreamError = class extends Error {

package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-manager.js","names":["#lc","#requiredBackfills","#changeStreamer","#backfillStreamer","#jsonFormat","#awaitingStatusWatermarks","#commitThresholdBytes","#minBackoffMs","#maxBackoffMs","#retryDelayMs","#lastStatusWatermark","#setRequiredBackfill","#checkAndStartBackfill","#backfillRetryTimer","#runningBackfill","#runBackfill","#stopRunningBackfill","#retryBackfillWithBackoff","#changeStreamReached","#currentTxWatermark","#setLastStatusWatermark","#backfillRunningFor","#deleteRequiredBackfill"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n  stateVersionFromString,\n  stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  ChangeStreamMessage,\n  Identifier,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n  Cancelable,\n  ChangeStreamMultiplexer,\n  Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return `${schema}.${name}`;\n}\n\n/**\n * A {@link MessageBackfill} or {@link BackfillCompleted} paired with the\n * approximate number of upstream bytes it represents. The BackfillManager uses\n * `byteSize` to bound the size of each replica transaction, committing and\n * reopening once {@link COMMIT_THRESHOLD_BYTES} is reached.\n */\nexport type BackfillMessage = {\n  message: MessageBackfill | BackfillCompleted;\n  byteSize: number;\n};\n\ntype BackfillStreamer = (\n  req: BackfillRequest,\n) => AsyncGenerator<BackfillMessage>;\n\n/**\n * Commit (and reopen) the current backfill transaction once this many bytes of\n * backfill data have accumulated in it. Bounds the size of a single replica\n * COMMIT/checkpoint so it can't monopolize the replica.\n */\nconst COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;\n\ntype RunningBackfillState = {\n  request: BackfillRequest;\n  canceledReason?: string | undefined;\n  minWatermark: string;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n  watermark: string;\n  reached: () => void;\n};\n\n/**\n * The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n *\n * The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n *\n * The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n */\nexport class BackfillManager implements Cancelable, Listener {\n  readonly #lc: LogContext;\n\n  /**\n   * Tracks the metadata of required backfills based on schema changes\n   * and initial backfill requests.\n   */\n  readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n    tableKey,\n  );\n  readonly #changeStreamer: ChangeStreamMultiplexer;\n  readonly #backfillStreamer: BackfillStreamer;\n  readonly #jsonFormat: JSONFormat;\n\n  /**\n   * The current running backfill. The backfill request is always also in\n   * `#requiredBackfills` (technically, it can be a subset of what's in\n   * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n   * upon completion.\n   */\n  #runningBackfill: RunningBackfillState | null = null;\n\n  /** The last seen watermark in the change stream. */\n  #lastStatusWatermark: string | null = null;\n\n  readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n  /** The watermark of the current transaction in the change stream. */\n  #currentTxWatermark: string | null = null;\n\n  readonly #commitThresholdBytes: number;\n\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamMultiplexer,\n    backfillStreamer: BackfillStreamer,\n    jsonFormat: JSONFormat = JSON_STRINGIFIED,\n    minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n    maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n    commitThresholdBytes = COMMIT_THRESHOLD_BYTES,\n  ) {\n    this.#lc = lc.withContext('component', 'backfill-manager');\n    this.#changeStreamer = changeStreamer;\n    this.#backfillStreamer = backfillStreamer;\n    this.#jsonFormat = jsonFormat;\n    this.#minBackoffMs = minBackoffMs;\n    this.#maxBackoffMs = maxBackoffMs;\n    this.#retryDelayMs = minBackoffMs;\n    this.#commitThresholdBytes = commitThresholdBytes;\n  }\n\n  run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n    this.#lc.info?.(\n      `starting backfill manager with ${initialRequests.length} initial requests`,\n      {requests: initialRequests},\n    );\n    this.#lastStatusWatermark = lastWatermark;\n    initialRequests.forEach(req =>\n      this.#setRequiredBackfill('initial-request', req),\n    );\n    this.#checkAndStartBackfill();\n  }\n\n  #setLastStatusWatermark({watermark}: {watermark: string}) {\n    // Only allow the watermark to move forward. This prevents a backfill\n    // transaction (whose watermark is unrelated to change-stream state)\n    // from moving the watermark backwards.\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      this.#lastStatusWatermark = watermark;\n      for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n        const awaiting = this.#awaitingStatusWatermarks[i];\n        if (watermark >= awaiting.watermark) {\n          awaiting.reached();\n          this.#awaitingStatusWatermarks.splice(i, 1);\n        }\n      }\n    }\n  }\n\n  #changeStreamReached(\n    lc: LogContext,\n    watermark: string,\n  ): Promise<void> | null {\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      const {promise, resolve: reached} = resolver();\n      this.#awaitingStatusWatermarks.push({watermark, reached});\n      lc.info?.(\n        `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n      );\n      return promise;\n    }\n    return null;\n  }\n\n  readonly #minBackoffMs: number;\n  readonly #maxBackoffMs: number;\n  #retryDelayMs: number;\n  #backfillRetryTimer: NodeJS.Timeout | undefined;\n\n  #checkAndStartBackfill() {\n    if (\n      !this.#backfillRetryTimer &&\n      !this.#runningBackfill &&\n      this.#requiredBackfills.size\n    ) {\n      // Pick a random backfill to avoid head-of-line blocking by a\n      // problematic backfill (e.g. awaiting a primary key). This is\n      // simpler that adding logic to classify (and declassify)\n      // problematic backfills.\n      const candidates = [...this.#requiredBackfills.values()];\n      const request = candidates[randInt(0, candidates.length - 1)];\n      const state = {request, minWatermark: ''};\n      const lc = this.#lc.withContext('table', request.table.name);\n\n      this.#runningBackfill = state;\n      void this.#runBackfill(lc, state)\n        .then(() => {\n          this.#stopRunningBackfill('backfill exited', state);\n          this.#retryDelayMs = this.#minBackoffMs; // reset on success\n        })\n        // For unexpected errors (e.g. upstream replication slot\n        // unavailability), retry with exponential backoff.\n        .catch(e => {\n          this.#stopRunningBackfill(String(e), state);\n          this.#retryBackfillWithBackoff(e);\n        });\n    }\n  }\n\n  #retryBackfillWithBackoff(e: unknown) {\n    const log = this.#retryDelayMs === this.#maxBackoffMs ? 'error' : 'warn';\n    this.#lc[log]?.(\n      `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n      e,\n    );\n    this.#backfillRetryTimer = setTimeout(() => {\n      this.#backfillRetryTimer = undefined;\n      this.#checkAndStartBackfill();\n    }, this.#retryDelayMs);\n\n    this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);\n  }\n\n  async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n    const changeStream = this.#changeStreamer; // Purely for readability\n\n    // backfillTx is set if and only if a changeStreamer reservation has been\n    // acquired and the backfill stream is inside a transaction.\n    let backfillTx: string | null = null;\n    let uncommittedBytes = 0;\n\n    /**\n     * @returns the new tx watermark, or null if backfill was cancelled\n     */\n    const beginTxFor = async (\n      msg: MessageBackfill | BackfillCompleted,\n    ): Promise<string | null> => {\n      assert(backfillTx === null, 'Expected no active backfill transaction');\n      const lastWatermark = await changeStream.reserve('backfill');\n\n      // After obtaining the changeStream reservation, check if the stream\n      // had changes that resulted in invalidating / canceling this backfill.\n      if (\n        state.canceledReason ||\n        (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n      ) {\n        if (state.canceledReason === undefined) {\n          assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n          this.#stopRunningBackfill(\n            `row key change at ${state.minWatermark} ` +\n              `postdates backfill watermark at ${msg.watermark}`,\n            state,\n          );\n        }\n        changeStream.release(lastWatermark);\n        return null;\n      }\n\n      const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n      let tx = stateVersionToString({\n        major,\n        minor: BigInt(minor) + 1n,\n      });\n\n      if (msg.tag === 'backfill-completed' && tx < msg.watermark) {\n        // At this point it must be the case that the #changeStreamReached() the\n        // backfill watermark. Given that guarantee, ensure that the version of the\n        // transaction containing the backfill-completed message is at least up\n        // to the backfill watermark, so that the final database state version is\n        // never earlier than the version of any backfilled rows.\n        tx = msg.watermark;\n      }\n\n      void changeStream.push([\n        'begin',\n        {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n        {commitWatermark: tx},\n      ]);\n      return (backfillTx = tx);\n    };\n\n    const commitTx = () => {\n      if (backfillTx) {\n        void changeStream.push([\n          'commit',\n          {tag: 'commit'},\n          {watermark: backfillTx},\n        ]);\n        changeStream.release(backfillTx);\n      }\n      backfillTx = null;\n      uncommittedBytes = 0;\n    };\n\n    for await (const {message: msg, byteSize} of this.#backfillStreamer(\n      state.request,\n    )) {\n      // Before sending `backfill-completed`, the main replication stream\n      // may need to catch up, and/or the current transaction may need to be\n      // committed to open a new transaction that's up to backfill watermark.\n      const mustWaitBeforeFlush =\n        msg.tag === 'backfill-completed' &&\n        (this.#changeStreamReached(lc, msg.watermark) ||\n          (backfillTx !== null && backfillTx < msg.watermark));\n\n      // Commit (and later reopen) the transaction if the main stream is\n      // waiting on the reservation, if we must catch up before completing, or\n      // if the size of current transaction has reached the commit threshold.\n      if (\n        backfillTx &&\n        (changeStream.waiterDelay() > 0 ||\n          mustWaitBeforeFlush ||\n          uncommittedBytes >= this.#commitThresholdBytes)\n      ) {\n        commitTx();\n      }\n\n      mustWaitBeforeFlush && (await mustWaitBeforeFlush);\n\n      if (\n        msg.tag === 'backfill' &&\n        msg.rowValues.length > 0 &&\n        msg.relation.rowKey.columns.length === 0\n      ) {\n        throw new MissingRowKeyError(state.request);\n      }\n\n      // Reserve the changeStreamer if not in a transaction.\n      if ((backfillTx ??= await beginTxFor(msg)) === null) {\n        lc.info?.(\n          `backfill stream canceled: ${state.canceledReason}`,\n          state.request,\n        );\n        this.#checkAndStartBackfill(); // start the next backfill if present\n        return; // this backfill is canceled\n      }\n\n      // `await` to allow the change streamer to exert back pressure\n      // on backfills.\n      await changeStream.push(['data', msg]);\n      uncommittedBytes += byteSize;\n    }\n\n    // Flush any final tx and release the stream.\n    backfillTx && commitTx();\n    lc.debug?.(`backfill stream exited`, state.canceledReason ?? '');\n  }\n\n  #backfillRunningFor(table: Identifier): RunningBackfillState | null {\n    const state = this.#runningBackfill;\n    return state?.request.table.schema === table.schema &&\n      state.request.table.name === table.name\n      ? state\n      : null;\n  }\n\n  /**\n   * Stops the running backfill for the specified `reason`. If `instance` is\n   * specified, the running backfill is stopped only if it is that instance.\n   * This allows the running backfill itself to clear backfill state without\n   * accidentally stopping a different (e.g. subsequent) backfill.\n   */\n  #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n    const backfill = this.#runningBackfill;\n    if (backfill && backfill === (instance ?? backfill)) {\n      backfill.canceledReason = reason;\n      this.#runningBackfill = null;\n      reason && this.#lc.info?.(`canceling backfill:`, reason);\n    }\n  }\n\n  #setRequiredBackfill(source: string, req: BackfillRequest) {\n    const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n    this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n    this.#requiredBackfills.set(req.table, req);\n  }\n\n  #deleteRequiredBackfill(source: string, id: Identifier) {\n    const req = this.#requiredBackfills.get(id);\n    if (req) {\n      const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n      this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n      this.#requiredBackfills.delete(id);\n    }\n  }\n\n  /**\n   * Implements {@link Listener.onChange()}, invoked by the\n   * {@link ChangeStreamMultiplexer}.\n   */\n  onChange(message: ChangeStreamMessage): void {\n    if (message[0] === 'begin') {\n      this.#currentTxWatermark = message[2].commitWatermark;\n      return;\n    }\n    if (message[0] === 'commit') {\n      this.#currentTxWatermark = null;\n      this.#setLastStatusWatermark(message[2]);\n      // Every commit is a candidate for starting the next backfill\n      // (if one is not currently running).\n      this.#checkAndStartBackfill();\n      return;\n    }\n    if (message[0] === 'status') {\n      this.#setLastStatusWatermark(message[2]);\n      return;\n    }\n    if (message[0] !== 'data') {\n      return;\n    }\n    const change = message[1];\n    const {tag} = change;\n    switch (tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest) {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, metadata},\n          });\n          if (this.#backfillRunningFor(table)) {\n            this.#stopRunningBackfill(`TableMetadata updated`);\n          }\n        }\n        break;\n      }\n      case 'create-table': {\n        const {\n          spec: {schema, name},\n          metadata = null,\n          backfill,\n        } = change;\n\n        if (backfill) {\n          this.#setRequiredBackfill(tag, {\n            table: {schema, name, metadata},\n            columns: backfill,\n          });\n        }\n        break;\n      }\n      case 'rename-table': {\n        const {old, new: newTable} = change;\n        const backfillRequest = this.#requiredBackfills.get(old);\n        if (backfillRequest) {\n          const {schema, name} = newTable;\n          this.#deleteRequiredBackfill(tag, old);\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, schema, name},\n          });\n          if (this.#backfillRunningFor(old)) {\n            this.#stopRunningBackfill(`table renamed`);\n          }\n        }\n        break;\n      }\n      case 'drop-table': {\n        const {id} = change;\n        const backfillRequest = this.#requiredBackfills.get(id);\n        if (backfillRequest) {\n          this.#deleteRequiredBackfill(tag, id);\n          if (this.#backfillRunningFor(id)) {\n            this.#stopRunningBackfill(`table dropped`);\n          }\n        }\n        break;\n      }\n      case 'add-column': {\n        const {\n          table,\n          tableMetadata: metadata = null,\n          column,\n          backfill,\n        } = change;\n        if (backfill) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (!backfillRequest) {\n            this.#setRequiredBackfill(tag, {\n              table: {...table, metadata},\n              columns: {[column.name]: backfill},\n            });\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              table: {...backfillRequest.table, metadata},\n              columns: {\n                ...backfillRequest.columns,\n                [column.name]: backfill,\n              },\n            });\n            // Note: The running backfill need not be canceled if a\n            //   new column is added. The new column will be backfilled\n            //   by its own stream after the current backfill completes.\n          }\n        }\n        break;\n      }\n      case 'update-column': {\n        const {\n          table,\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (backfillRequest && oldName in backfillRequest.columns) {\n            const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: {...otherCols, [newName]: colSpec},\n            });\n            const backfill = this.#backfillRunningFor(table);\n            if (backfill && oldName in backfill.request.columns) {\n              this.#stopRunningBackfill(`column renamed`);\n            }\n          }\n        }\n        break;\n      }\n      case 'drop-column': {\n        const {table, column} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest && column in backfillRequest.columns) {\n          const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n          if (Object.keys(remaining).length === 0) {\n            this.#deleteRequiredBackfill(tag, table);\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: remaining,\n            });\n          }\n          const backfill = this.#backfillRunningFor(table);\n          if (backfill && column in backfill.request.columns) {\n            this.#stopRunningBackfill(`column dropped`);\n          }\n        }\n        break;\n      }\n      case 'update': {\n        const {relation, key, new: row} = change;\n        const backfill = this.#backfillRunningFor(relation);\n        const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n        if (backfill?.request.table.metadata && key !== null) {\n          // A corner case that backfill is unable to correctly handle is\n          // when a row's key changes; this is decomposed into a delete\n          // of the old key and a set of the new key in the replica change\n          // log, at which point the backfill algorithm assumes that the\n          // (old) row is deleted but does not know to backfill the new row.\n          // In these corner cases, the current backfill is canceled and\n          // retried if its version precedes this update.\n          for (const col of Object.keys(\n            backfill.request.table.metadata.rowKey,\n          )) {\n            if (key[col] !== row[col]) {\n              backfill.minWatermark = txWatermark;\n              this.#lc.info?.(\n                `key for row as changed (col: ${col}). ` +\n                  `backfill data must not predate ${backfill.minWatermark}`,\n              );\n              break;\n            }\n          }\n        }\n        break;\n      }\n      case 'backfill-completed': {\n        const {relation, columns} = change;\n        const backfillRequest = this.#requiredBackfills.get(relation);\n        assert(\n          backfillRequest,\n          () => `No BackfillRequest completed backfill ${stringify(change)}`,\n        );\n        const remaining = Object.entries(backfillRequest.columns).filter(\n          ([col]) =>\n            !(columns.includes(col) || relation.rowKey.columns.includes(col)),\n        );\n        if (remaining.length === 0) {\n          this.#deleteRequiredBackfill(tag, relation);\n        } else {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            columns: Object.fromEntries(remaining),\n          });\n        }\n        // Technically the backfill is already stopping, but this method\n        // cleans up the state that tracks it.\n        this.#stopRunningBackfill();\n        break;\n      }\n    }\n  }\n\n  cancel(): void {\n    this.#stopRunningBackfill(`change stream canceled`);\n    clearTimeout(this.#backfillRetryTimer);\n  }\n}\n\nabstract class BackfillStreamError extends Error {\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(\n      `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n        `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n      {cause},\n    );\n  }\n}\n\n/**\n * Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n *\n * In terms of backfill, however, non-empty tables without a row key **cannot**\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n *\n * The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n */\nclass MissingRowKeyError extends BackfillStreamError {\n  readonly name = 'MissingRowKeyError';\n\n  constructor(bf: BackfillRequest, cause?: unknown) {\n    super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n  }\n}\n\n/**\n * Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n  readonly name = 'SchemaIncompatibilityError';\n\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(bf, msg, cause);\n  }\n}\n"],"mappings":";;;;;;;;;AAyBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,GAAG,OAAO,GAAG;AACtB;;;;;;AAsBA,IAAM,yBAAyB,IAAI,OAAO;AAQ1C,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,QACF;CACA;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,CAAC;;CAGjE,sBAAqC;CAErC;CAEA,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf,uBAAuB,wBACvB;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKE,kBAAkB;EACvB,KAAKC,oBAAoB;EACzB,KAAKC,cAAc;EACnB,KAAKG,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKH,wBAAwB;CAC/B;CAEA,IAAI,eAAuB,iBAAoC;EAC7D,KAAKN,IAAI,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,gBAAe,CAC5B;EACA,KAAKU,uBAAuB;EAC5B,gBAAgB,SAAQ,QACtB,KAAKC,qBAAqB,mBAAmB,GAAG,CAClD;EACA,KAAKC,uBAAuB;CAC9B;CAEA,wBAAwB,EAAC,aAAiC;EAIxD,KAAK,KAAKF,wBAAwB,MAAM,WAAW;GACjD,KAAKA,uBAAuB;GAC5B,KAAK,IAAI,IAAI,KAAKL,0BAA0B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,KAAKA,0BAA0B;IAChD,IAAI,aAAa,SAAS,WAAW;KACnC,SAAS,QAAQ;KACjB,KAAKA,0BAA0B,OAAO,GAAG,CAAC;IAC5C;GACF;EACF;CACF;CAEA,qBACE,IACA,WACsB;EACtB,KAAK,KAAKK,wBAAwB,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,SAAS;GAC7C,KAAKL,0BAA0B,KAAK;IAAC;IAAW;GAAO,CAAC;GACxD,GAAG,OACD,iCAAiC,KAAKK,qBAAqB,aAAa,WAC1E;GACA,OAAO;EACT;EACA,OAAO;CACT;CAEA;CACA;CACA;CACA;CAEA,yBAAyB;EACvB,IACE,CAAC,KAAKG,uBACN,CAAC,KAAKC,oBACN,KAAKb,mBAAmB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,KAAKA,mBAAmB,OAAO,CAAC;GACvD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,CAAC;GAC3D,MAAM,QAAQ;IAAC;IAAS,cAAc;GAAE;GACxC,MAAM,KAAK,KAAKD,IAAI,YAAY,SAAS,QAAQ,MAAM,IAAI;GAE3D,KAAKc,mBAAmB;GACxB,KAAUC,aAAa,IAAI,KAAK,EAC7B,WAAW;IACV,KAAKC,qBAAqB,mBAAmB,KAAK;IAClD,KAAKP,gBAAgB,KAAKF;GAC5B,CAAC,EAGA,OAAM,MAAK;IACV,KAAKS,qBAAqB,OAAO,CAAC,GAAG,KAAK;IAC1C,KAAKC,0BAA0B,CAAC;GAClC,CAAC;EACL;CACF;CAEA,0BAA0B,GAAY;EACpC,MAAM,MAAM,KAAKR,kBAAkB,KAAKD,gBAAgB,UAAU;EAClE,KAAKR,IAAI,OACP,uCAAuC,KAAKS,cAAc,MAC1D,CACF;EACA,KAAKI,sBAAsB,iBAAiB;GAC1C,KAAKA,sBAAsB,KAAA;GAC3B,KAAKD,uBAAuB;EAC9B,GAAG,KAAKH,aAAa;EAErB,KAAKA,gBAAgB,KAAK,IAAI,KAAKA,gBAAgB,GAAG,KAAKD,aAAa;CAC1E;CAEA,MAAMO,aAAa,IAAgB,OAA6B;EAC9D,MAAM,eAAe,KAAKb;EAI1B,IAAI,aAA4B;EAChC,IAAI,mBAAmB;;;;EAKvB,MAAM,aAAa,OACjB,QAC2B;GAC3B,OAAO,eAAe,MAAM,yCAAyC;GACrE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,UAAU;GAI3D,IACE,MAAM,kBACL,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;IACA,IAAI,MAAM,mBAAmB,KAAA,GAAW;KACtC,OAAO,IAAI,QAAQ,YAAY,+BAA+B;KAC9D,KAAKc,qBACH,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,KACF;IACF;IACA,aAAa,QAAQ,aAAa;IAClC,OAAO;GACT;GAEA,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,aAAa;GAChE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,KAAK,IAAI;GACzB,CAAC;GAED,IAAI,IAAI,QAAQ,wBAAwB,KAAK,IAAI,WAM/C,KAAK,IAAI;GAGX,aAAkB,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,KAAKZ;KAAa,SAAS;IAAI;IACpD,EAAC,iBAAiB,GAAE;GACtB,CAAC;GACD,OAAQ,aAAa;EACvB;EAEA,MAAM,iBAAiB;GACrB,IAAI,YAAY;IACd,aAAkB,KAAK;KACrB;KACA,EAAC,KAAK,SAAQ;KACd,EAAC,WAAW,WAAU;IACxB,CAAC;IACD,aAAa,QAAQ,UAAU;GACjC;GACA,aAAa;GACb,mBAAmB;EACrB;EAEA,WAAW,MAAM,EAAC,SAAS,KAAK,cAAa,KAAKD,kBAChD,MAAM,OACR,GAAG;GAID,MAAM,sBACJ,IAAI,QAAQ,yBACX,KAAKe,qBAAqB,IAAI,IAAI,SAAS,KACzC,eAAe,QAAQ,aAAa,IAAI;GAK7C,IACE,eACC,aAAa,YAAY,IAAI,KAC5B,uBACA,oBAAoB,KAAKZ,wBAE3B,SAAS;GAGX,uBAAwB,MAAM;GAE9B,IACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,GAEvC,MAAM,IAAI,mBAAmB,MAAM,OAAO;GAI5C,KAAK,eAAe,MAAM,WAAW,GAAG,OAAO,MAAM;IACnD,GAAG,OACD,6BAA6B,MAAM,kBACnC,MAAM,OACR;IACA,KAAKM,uBAAuB;IAC5B;GACF;GAIA,MAAM,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC;GACrC,oBAAoB;EACtB;EAGA,cAAc,SAAS;EACvB,GAAG,QAAQ,0BAA0B,MAAM,kBAAkB,EAAE;CACjE;CAEA,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,KAAKE;EACnB,OAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;CACN;;;;;;;CAQA,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,KAAKA;EACtB,IAAI,YAAY,cAAc,YAAY,WAAW;GACnD,SAAS,iBAAiB;GAC1B,KAAKA,mBAAmB;GACxB,UAAU,KAAKd,IAAI,OAAO,uBAAuB,MAAM;EACzD;CACF;CAEA,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,KAAKC,mBAAmB,IAAI,IAAI,KAAK,IAAI,YAAY;EACpE,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;EAChE,KAAKC,mBAAmB,IAAI,IAAI,OAAO,GAAG;CAC5C;CAEA,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,KAAKA,mBAAmB,IAAI,EAAE;EAC1C,IAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;GAC/D,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;GAChE,KAAKC,mBAAmB,OAAO,EAAE;EACnC;CACF;;;;;CAMA,SAAS,SAAoC;EAC3C,IAAI,QAAQ,OAAO,SAAS;GAC1B,KAAKkB,sBAAsB,QAAQ,GAAG;GACtC;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKA,sBAAsB;GAC3B,KAAKC,wBAAwB,QAAQ,EAAE;GAGvC,KAAKR,uBAAuB;GAC5B;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKQ,wBAAwB,QAAQ,EAAE;GACvC;EACF;EACA,IAAI,QAAQ,OAAO,QACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;EACd,QAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,KAAKnB,mBAAmB,IAAI,KAAK;IACzD,IAAI,iBAAiB;KACnB,KAAKU,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;KAC5C,CAAC;KACD,IAAI,KAAKU,oBAAoB,KAAK,GAChC,KAAKL,qBAAqB,uBAAuB;IAErD;IACA;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;IAEJ,IAAI,UACF,KAAKL,qBAAqB,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;KAAQ;KAC9B,SAAS;IACX,CAAC;IAEH;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,KAAKV,mBAAmB,IAAI,GAAG;IACvD,IAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;KACvB,KAAKqB,wBAAwB,KAAK,GAAG;KACrC,KAAKX,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;MAAI;KAChD,CAAC;KACD,IAAI,KAAKU,oBAAoB,GAAG,GAC9B,KAAKL,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;IAEb,IADwB,KAAKf,mBAAmB,IAAI,EAChD,GAAiB;KACnB,KAAKqB,wBAAwB,KAAK,EAAE;KACpC,IAAI,KAAKD,oBAAoB,EAAE,GAC7B,KAAKL,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;IACJ,IAAI,UAAU;KACZ,MAAM,kBAAkB,KAAKf,mBAAmB,IAAI,KAAK;KACzD,IAAI,CAAC,iBACH,KAAKU,qBAAqB,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;MAAQ;MAC1B,SAAS,GAAE,OAAO,OAAO,SAAQ;KACnC,CAAC;UAED,KAAKA,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;MAC1C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO;MACjB;KACF,CAAC;IAKL;IACA;GACF;GACA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,KAAKV,mBAAmB,IAAI,KAAK;KACzD,IAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;MAC3D,KAAKU,qBAAqB,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;OAAO;MAC5C,CAAC;MACD,MAAM,WAAW,KAAKU,oBAAoB,KAAK;MAC/C,IAAI,YAAY,WAAW,SAAS,QAAQ,SAC1C,KAAKL,qBAAqB,gBAAgB;KAE9C;IACF;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,KAAKf,mBAAmB,IAAI,KAAK;IACzD,IAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;KAC5D,IAAI,OAAO,KAAK,SAAS,EAAE,WAAW,GACpC,KAAKqB,wBAAwB,KAAK,KAAK;UAEvC,KAAKX,qBAAqB,KAAK;MAC7B,GAAG;MACH,SAAS;KACX,CAAC;KAEH,MAAM,WAAW,KAAKU,oBAAoB,KAAK;KAC/C,IAAI,YAAY,UAAU,SAAS,QAAQ,SACzC,KAAKL,qBAAqB,gBAAgB;IAE9C;IACA;GACF;GACA,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,KAAKK,oBAAoB,QAAQ;IAClD,MAAM,cAAc,KAAK,KAAKF,qBAAqB,aAAa;IAChE,IAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ;UAQzC,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,MAClC,GACE,IAAI,IAAI,SAAS,IAAI,MAAM;MACzB,SAAS,eAAe;MACxB,KAAKnB,IAAI,OACP,gCAAgC,IAAI,oCACA,SAAS,cAC/C;MACA;KACF;;IAGJ;GACF;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,KAAKC,mBAAmB,IAAI,QAAQ;IAC5D,OACE,uBACM,yCAAyC,UAAU,MAAM,GACjE;IACA,MAAM,YAAY,OAAO,QAAQ,gBAAgB,OAAO,EAAE,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,GAAG,KAAK,SAAS,OAAO,QAAQ,SAAS,GAAG,EACnE;IACA,IAAI,UAAU,WAAW,GACvB,KAAKqB,wBAAwB,KAAK,QAAQ;SAE1C,KAAKX,qBAAqB,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,SAAS;IACvC,CAAC;IAIH,KAAKK,qBAAqB;IAC1B;GACF;EACF;CACF;CAEA,SAAe;EACb,KAAKA,qBAAqB,wBAAwB;EAClD,aAAa,KAAKH,mBAAmB;CACvC;AACF;AAEA,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,OAC7C,EAAC,MAAK,CACR;CACF;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;EAChD,MAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,KAAK;CAChE;AACF;;;;;;;AAQA,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MAAM,IAAI,KAAK,KAAK;CACtB;AACF"}
+{"version":3,"file":"backfill-manager.js","names":["#lc","#requiredBackfills","#changeStreamer","#backfillStreamer","#jsonFormat","#awaitingStatusWatermarks","#commitThresholdBytes","#controller","#canceled","#minBackoffMs","#maxBackoffMs","#retryDelayMs","#lastStatusWatermark","#setRequiredBackfill","#checkAndStartBackfill","#backfillRetryTimer","#runningBackfill","#runBackfill","#stopRunningBackfill","#retryBackfillWithBackoff","#changeStreamReached","#maybeLogBackfillStats","#logBackfillStats","#currentTxWatermark","#setLastStatusWatermark","#backfillRunningFor","#deleteRequiredBackfill"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseOrAbort} from '../../../../../shared/src/promise-race.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n  stateVersionFromString,\n  stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  ChangeStreamMessage,\n  Identifier,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n  Cancelable,\n  ChangeStreamMultiplexer,\n  Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return `${schema}.${name}`;\n}\n\n/**\n * A {@link MessageBackfill} or {@link BackfillCompleted} paired with the\n * approximate number of upstream bytes it represents. The BackfillManager uses\n * `byteSize` to bound the size of each replica transaction, committing and\n * reopening once {@link COMMIT_THRESHOLD_BYTES} is reached.\n */\nexport type BackfillMessage = {\n  message: MessageBackfill | BackfillCompleted;\n  byteSize: number;\n};\n\ntype BackfillStreamer = (\n  req: BackfillRequest,\n) => AsyncGenerator<BackfillMessage>;\n\n/**\n * Commit (and reopen) the current backfill transaction once this many bytes of\n * backfill data have accumulated in it. Bounds the size of a single replica\n * COMMIT/checkpoint so it can't monopolize the replica.\n */\nconst COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;\n\n/**\n * How many backfill messages the prefetch producer may read ahead of the\n * reservation-holding consumer loop. This bounds the in-memory buffer (~this\n * many {@link BackfillMessage}s, each up to the stream's flush threshold, e.g.\n * 64 KiB) while giving the consumer a supply of ready messages so it never\n * has to wait on the upstream COPY while holding the change-stream reservation.\n */\nconst BACKFILL_PREFETCH_CREDITS = 16;\n\n/**\n * While the backfill holds an open transaction but its prefetch queue is empty\n * (i.e. the upstream is momentarily behind), the consumer polls for the next\n * message at this interval so that a replication transaction that begins\n * waiting mid-fetch can preempt the backfill within this bound, rather than\n * being blocked for the entire (possibly slow) upstream read.\n *\n * If stats indicate that these stalls are highly correlative with replication\n * lag (indicating that replication changes are waiting on these polls), we\n * can address this by adding an API to the change stream multiplexer that\n * would allow backfill to relinquish an in-progress commit on demand (i.e.\n * only if replication asks for a reservation).\n */\nconst BACKFILL_WAITER_POLL_MS = 10;\n\n/** Interval at which in-progress backfill timing stats are logged. */\nconst BACKFILL_STATS_LOG_INTERVAL_MS = 10_000;\n\n/** Sentinel enqueued by the prefetch producer when the backfill stream ends. */\nconst DONE = Symbol('backfill-done');\n/** Sentinel returned by a polled dequeue that timed out (see {@link BACKFILL_WAITER_POLL_MS}). */\nconst POLL = Symbol('backfill-poll');\n\ntype Pull = BackfillMessage | typeof DONE | typeof POLL;\n\n/**\n * Accumulated timing attribution for a single backfill run, logged\n * periodically to reveal where the backfill spends its time (and, in\n * particular, where the change-stream reservation is held). `produceMs`\n * accrues on the prefetch producer task and thus overlaps with the consumer's\n * `pushMs`; the others accrue on the reservation-holding consumer loop.\n */\ntype BackfillTimingStats = {\n  start: number;\n  lastLogged: number;\n  messages: number;\n  bytes: number;\n  commits: number;\n  stalls: number;\n  produceMs: number;\n  pushMs: number;\n  reserveMs: number;\n  stallMs: number;\n  waitMs: number;\n};\n\ntype RunningBackfillState = {\n  request: BackfillRequest;\n  minWatermark: string;\n  /**\n   * Aborted (with the reason, if any) when this backfill is stopped,\n   * e.g. canceled.\n   */\n  stopped: AbortController;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n  watermark: string;\n  reached: () => void;\n};\n\n/**\n * The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n *\n * The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n *\n * The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n */\nexport class BackfillManager implements Cancelable, Listener {\n  readonly #lc: LogContext;\n\n  /**\n   * Tracks the metadata of required backfills based on schema changes\n   * and initial backfill requests.\n   */\n  readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n    tableKey,\n  );\n  readonly #changeStreamer: ChangeStreamMultiplexer;\n  readonly #backfillStreamer: BackfillStreamer;\n  readonly #jsonFormat: JSONFormat;\n\n  /**\n   * The current running backfill. The backfill request is always also in\n   * `#requiredBackfills` (technically, it can be a subset of what's in\n   * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n   * upon completion.\n   */\n  #runningBackfill: RunningBackfillState | null = null;\n\n  /** The last seen watermark in the change stream. */\n  #lastStatusWatermark: string | null = null;\n\n  readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n  /** The watermark of the current transaction in the change stream. */\n  #currentTxWatermark: string | null = null;\n\n  readonly #commitThresholdBytes: number;\n\n  /**\n   * Aborted when the change stream is canceled. No further backfills are run.\n   *\n   * TODO: Thread this into the backfillStream so that upstream I/O can be\n   * aborted.\n   */\n  readonly #controller = new AbortController();\n\n  get #canceled() {\n    return this.#controller.signal.aborted;\n  }\n\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamMultiplexer,\n    backfillStreamer: BackfillStreamer,\n    jsonFormat: JSONFormat = JSON_STRINGIFIED,\n    minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n    maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n    commitThresholdBytes = COMMIT_THRESHOLD_BYTES,\n  ) {\n    this.#lc = lc.withContext('component', 'backfill-manager');\n    this.#changeStreamer = changeStreamer;\n    this.#backfillStreamer = backfillStreamer;\n    this.#jsonFormat = jsonFormat;\n    this.#minBackoffMs = minBackoffMs;\n    this.#maxBackoffMs = maxBackoffMs;\n    this.#retryDelayMs = minBackoffMs;\n    this.#commitThresholdBytes = commitThresholdBytes;\n  }\n\n  run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n    this.#lc.info?.(\n      `starting backfill manager with ${initialRequests.length} initial requests`,\n      {requests: initialRequests},\n    );\n    this.#lastStatusWatermark = lastWatermark;\n    initialRequests.forEach(req =>\n      this.#setRequiredBackfill('initial-request', req),\n    );\n    this.#checkAndStartBackfill();\n  }\n\n  #setLastStatusWatermark({watermark}: {watermark: string}) {\n    // Only allow the watermark to move forward. This prevents a backfill\n    // transaction (whose watermark is unrelated to change-stream state)\n    // from moving the watermark backwards.\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      this.#lastStatusWatermark = watermark;\n      for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n        const awaiting = this.#awaitingStatusWatermarks[i];\n        if (watermark >= awaiting.watermark) {\n          awaiting.reached();\n          this.#awaitingStatusWatermarks.splice(i, 1);\n        }\n      }\n    }\n  }\n\n  #changeStreamReached(\n    lc: LogContext,\n    watermark: string,\n  ): Promise<void> | null {\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      const {promise, resolve: reached} = resolver();\n      this.#awaitingStatusWatermarks.push({watermark, reached});\n      lc.info?.(\n        `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n      );\n      return promise;\n    }\n    return null;\n  }\n\n  readonly #minBackoffMs: number;\n  readonly #maxBackoffMs: number;\n  #retryDelayMs: number;\n  #backfillRetryTimer: NodeJS.Timeout | undefined;\n\n  #checkAndStartBackfill() {\n    if (\n      !this.#canceled &&\n      !this.#backfillRetryTimer &&\n      !this.#runningBackfill &&\n      this.#requiredBackfills.size\n    ) {\n      // Pick a random backfill to avoid head-of-line blocking by a\n      // problematic backfill (e.g. awaiting a primary key). This is\n      // simpler that adding logic to classify (and declassify)\n      // problematic backfills.\n      const candidates = [...this.#requiredBackfills.values()];\n      const request = candidates[randInt(0, candidates.length - 1)];\n      const state = {\n        request,\n        minWatermark: '',\n        stopped: new AbortController(),\n      };\n      const lc = this.#lc.withContext('table', request.table.name);\n\n      this.#runningBackfill = state;\n      void this.#runBackfill(lc, state)\n        .then(() => {\n          this.#stopRunningBackfill('backfill exited', state);\n          this.#retryDelayMs = this.#minBackoffMs; // reset on success\n        })\n        // For unexpected errors (e.g. upstream replication slot\n        // unavailability), retry with exponential backoff.\n        .catch(e => {\n          this.#stopRunningBackfill(String(e), state);\n          this.#retryBackfillWithBackoff(e);\n        });\n    }\n  }\n\n  #retryBackfillWithBackoff(e: unknown) {\n    if (this.#canceled) {\n      this.#lc.debug?.(`not retrying backfill: change stream canceled`, e);\n      return;\n    }\n    this.#lc.warn?.(\n      `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n      e,\n    );\n    this.#backfillRetryTimer = setTimeout(() => {\n      this.#backfillRetryTimer = undefined;\n      this.#checkAndStartBackfill();\n    }, this.#retryDelayMs);\n\n    this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);\n  }\n\n  async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n    const changeStream = this.#changeStreamer; // Purely for readability\n\n    // backfillTx is set if and only if a changeStreamer reservation has been\n    // acquired and the backfill stream is inside a transaction.\n    let backfillTx: string | null = null;\n    let uncommittedBytes = 0;\n\n    const stats: BackfillTimingStats = {\n      start: performance.now(),\n      lastLogged: performance.now(),\n      messages: 0,\n      bytes: 0,\n      commits: 0,\n      stalls: 0,\n      produceMs: 0,\n      pushMs: 0,\n      reserveMs: 0,\n      stallMs: 0,\n      waitMs: 0,\n    };\n\n    /**\n     * @returns the new tx watermark, or null if backfill was cancelled\n     */\n    const beginTxFor = async (\n      msg: MessageBackfill | BackfillCompleted,\n    ): Promise<string | null> => {\n      assert(backfillTx === null, 'Expected no active backfill transaction');\n      const lastWatermark = await changeStream.reserve('backfill');\n\n      // After obtaining the changeStream reservation, check if the stream\n      // had changes that resulted in invalidating / canceling this backfill.\n      if (\n        state.stopped.signal.aborted ||\n        (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n      ) {\n        if (!state.stopped.signal.aborted) {\n          assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n          this.#stopRunningBackfill(\n            `row key change at ${state.minWatermark} ` +\n              `postdates backfill watermark at ${msg.watermark}`,\n            state,\n          );\n        }\n        changeStream.release(lastWatermark);\n        return null;\n      }\n\n      const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n      let tx = stateVersionToString({\n        major,\n        minor: BigInt(minor) + 1n,\n      });\n\n      if (tx < msg.watermark) {\n        // At this point it must be the case that the #changeStreamReached()\n        // the backfill watermark (possibly via a status message, which does\n        // not advance the reservation watermark). Given that guarantee,\n        // ensure that the version of the backfill transaction is at least up\n        // to the backfill watermark, so that the database state version is\n        // never earlier than the version of any backfilled rows.\n        tx = msg.watermark;\n      }\n\n      void changeStream.push([\n        'begin',\n        {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n        {commitWatermark: tx},\n      ]);\n      return (backfillTx = tx);\n    };\n\n    const commitTx = () => {\n      if (backfillTx) {\n        void changeStream.push([\n          'commit',\n          {tag: 'commit'},\n          {watermark: backfillTx},\n        ]);\n        changeStream.release(backfillTx);\n        stats.commits++;\n      }\n      backfillTx = null;\n      uncommittedBytes = 0;\n    };\n\n    // === Prefetch producer ===\n    //\n    // The backfill stream (an upstream Postgres COPY) is drained on a separate\n    // task into a bounded queue, so that the reservation-holding consumer loop\n    // below never blocks on the upstream read while it holds the change-stream\n    // reservation. `credits` bounds how far the producer runs ahead; the\n    // consumer returns a credit for every message it takes.\n    const ready = new Queue<Pull>();\n    const credits = new Queue<1>();\n    for (let i = 0; i < BACKFILL_PREFETCH_CREDITS; i++) {\n      credits.enqueue(1);\n    }\n    let stopProducer = false;\n\n    // Wakes a consumer blocked awaiting the next prefetched message so a\n    // canceled backfill unwinds promptly.\n    const onCancel = () =>\n      ready.enqueueRejection(new AbortError('backfill canceled'));\n    if (!this.#canceled) {\n      this.#controller.signal.addEventListener('abort', onCancel);\n    }\n\n    const producer = (async () => {\n      try {\n        let t0 = performance.now();\n        for await (const value of this.#backfillStreamer(state.request)) {\n          stats.produceMs += performance.now() - t0;\n          await credits.dequeue(); // wait for buffer space (flow control)\n          if (stopProducer || this.#canceled) {\n            ready.enqueueRejection(new AbortError('backfill canceled'));\n            return;\n          }\n          ready.enqueue(value);\n          t0 = performance.now();\n        }\n        ready.enqueue(DONE);\n      } catch (e) {\n        ready.enqueueRejection(e);\n      }\n    })();\n    // The producer routes errors through `ready` and never rejects; guard\n    // anyway so that leaving it un-awaited on the cancellation path (below)\n    // cannot surface as an unhandled rejection.\n    void producer.catch(() => {});\n\n    // === Consumer ===\n\n    // Fetches the next item from the prefetch queue. If a transaction is open\n    // but the queue is empty (the producer/upstream is momentarily behind),\n    // polls so that a replication transaction that begins waiting during the\n    // upstream read can preempt the backfill: rather than hold the reservation\n    // for the whole read, commit and release, letting replication proceed.\n    const nextItem = async (): Promise<BackfillMessage | typeof DONE> => {\n      // Fast path: a message is already buffered, or there's no reservation to\n      // protect. Take it without altering transaction boundaries. `POLL` is\n      // never enqueued (only used as a dequeue timeout value), so the result is\n      // always a message or DONE.\n      if (backfillTx === null || ready.size() > 0) {\n        const n = ready.dequeue();\n        const v = n instanceof Promise ? await n : n;\n        return v as BackfillMessage | typeof DONE;\n      }\n      // Slow path: holding a reservation with an empty queue.\n      for (;;) {\n        if (backfillTx !== null && changeStream.waiterDelay() > 0) {\n          commitTx(); // yield the stream to the waiting replication txn\n        }\n        const n =\n          backfillTx !== null\n            ? ready.dequeue(POLL, BACKFILL_WAITER_POLL_MS)\n            : ready.dequeue(); // released: just wait for the next message\n        const t0 = performance.now();\n        const v = n instanceof Promise ? await n : n;\n        if (v === POLL) {\n          stats.stallMs += performance.now() - t0; // timed out; re-check waiter\n          continue;\n        }\n        if (n instanceof Promise) {\n          stats.stallMs += performance.now() - t0;\n          stats.stalls++;\n        }\n        return v;\n      }\n    };\n\n    try {\n      for (;;) {\n        const item = await nextItem();\n        if (item === DONE) {\n          break;\n        }\n        // Allow the producer to fetch the next message concurrently with the\n        // processing (and downstream push) of this one.\n        credits.enqueue(1);\n\n        if (this.#canceled) {\n          // Returning finalizes the backfill stream (and the upstream resources\n          // it holds) via the producer's `finally`. The reservation, if held,\n          // does not need to be released since the change stream is gone.\n          lc.info?.(`backfill stream canceled: change stream canceled`);\n          return;\n        }\n        const {message: msg, byteSize} = item;\n\n        // Before sending any backfill message, the main replication stream\n        // must reach the backfill (snapshot) watermark. Otherwise, snapshot\n        // rows can constitute \"future\" state (e.g. phantom rows at keys that\n        // the stream has yet to move or insert) that the subsequent\n        // replication changes conflict with (e.g. on unique indexes).\n        const waitForChangeStream = this.#changeStreamReached(\n          lc,\n          msg.watermark,\n        );\n\n        // Commit (and later reopen) the transaction if the main stream is\n        // waiting on the reservation, if we must wait for the main stream, or\n        // if the size of current transaction has reached the commit threshold.\n        if (\n          backfillTx &&\n          (changeStream.waiterDelay() > 0 ||\n            waitForChangeStream ||\n            uncommittedBytes >= this.#commitThresholdBytes)\n        ) {\n          commitTx();\n        }\n\n        if (waitForChangeStream) {\n          const t0 = performance.now();\n          // Exit promptly (via the error path) if the backfill is canceled\n          // while waiting, releasing its upstream resources.\n          await promiseOrAbort(waitForChangeStream, state.stopped.signal);\n          const elapsed = performance.now() - t0;\n          stats.waitMs += elapsed;\n          lc.info?.(\n            `change stream reached ${msg.watermark} ` +\n              `after waiting ${Math.round(elapsed)} ms`,\n          );\n        }\n\n        if (\n          msg.tag === 'backfill' &&\n          msg.rowValues.length > 0 &&\n          msg.relation.rowKey.columns.length === 0\n        ) {\n          throw new MissingRowKeyError(state.request);\n        }\n\n        // Reserve the changeStreamer if not in a transaction.\n        if (backfillTx === null) {\n          const t0 = performance.now();\n          backfillTx = await beginTxFor(msg);\n          stats.reserveMs += performance.now() - t0;\n          if (backfillTx === null) {\n            lc.info?.(\n              `backfill stream canceled: ${state.stopped.signal.reason}`,\n              state.request,\n            );\n            this.#checkAndStartBackfill(); // start the next backfill if present\n            return; // this backfill is canceled\n          }\n        }\n\n        // `await` to allow the change streamer to exert back pressure\n        // on backfills.\n        const t0 = performance.now();\n        await changeStream.push(['data', msg]);\n        stats.pushMs += performance.now() - t0;\n        uncommittedBytes += byteSize;\n        stats.messages++;\n        stats.bytes += byteSize;\n\n        this.#maybeLogBackfillStats(lc, stats);\n      }\n\n      // Flush any final tx and release the stream.\n      if (backfillTx) {\n        commitTx();\n      }\n      lc.debug?.(`backfill stream exited`);\n    } finally {\n      stopProducer = true;\n      credits.enqueue(1);\n      // The producer can potentially be hung on upstream I/O.\n      // TODO: Thread AbortSignal logic into the backfillStreamer.\n      if (!this.#canceled) {\n        await producer;\n      }\n      this.#controller.signal.removeEventListener('abort', onCancel);\n      this.#logBackfillStats(lc, stats);\n    }\n  }\n\n  #maybeLogBackfillStats(lc: LogContext, stats: BackfillTimingStats) {\n    const now = performance.now();\n    if (now - stats.lastLogged >= BACKFILL_STATS_LOG_INTERVAL_MS) {\n      this.#logBackfillStats(lc, stats);\n      stats.lastLogged = now;\n    }\n  }\n\n  /**\n   * Logs where the backfill spent its time. `produceMs` (upstream read +\n   * parse/decode) accrues on the prefetch producer concurrently with the\n   * consumer's `pushMs` (downstream backpressure), so the two can overlap and\n   * their sum can exceed `elapsedMs`. `reserveMs` is time spent (re)acquiring\n   * the change-stream reservation, and `stallMs` is time the consumer waited on\n   * an empty prefetch queue (i.e. blocked on the upstream) — a large `stallMs`\n   * indicates an upstream-bound backfill. `waitMs` is time spent waiting\n   * (without the reservation) for the change stream to reach the backfill\n   * watermark, during which the upstream snapshot is held open.\n   */\n  #logBackfillStats(lc: LogContext, stats: BackfillTimingStats) {\n    const elapsed = performance.now() - stats.start;\n    lc.info?.(`backfill timing`, {\n      messages: stats.messages,\n      bytes: stats.bytes,\n      commits: stats.commits,\n      stalls: stats.stalls,\n      elapsedMs: Math.round(elapsed),\n      produceMs: Math.round(stats.produceMs),\n      pushMs: Math.round(stats.pushMs),\n      reserveMs: Math.round(stats.reserveMs),\n      stallMs: Math.round(stats.stallMs),\n      waitMs: Math.round(stats.waitMs),\n      throughputKBps:\n        elapsed > 0 ? Math.round(stats.bytes / 1024 / (elapsed / 1000)) : 0,\n    });\n  }\n\n  #backfillRunningFor(table: Identifier): RunningBackfillState | null {\n    const state = this.#runningBackfill;\n    return state?.request.table.schema === table.schema &&\n      state.request.table.name === table.name\n      ? state\n      : null;\n  }\n\n  /**\n   * Stops the running backfill for the specified `reason`. If `instance` is\n   * specified, the running backfill is stopped only if it is that instance.\n   * This allows the running backfill itself to clear backfill state without\n   * accidentally stopping a different (e.g. subsequent) backfill.\n   */\n  #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n    const backfill = this.#runningBackfill;\n    if (backfill && backfill === (instance ?? backfill)) {\n      backfill.stopped.abort(reason);\n      this.#runningBackfill = null;\n      reason && this.#lc.info?.(`canceling backfill:`, reason);\n    }\n  }\n\n  #setRequiredBackfill(source: string, req: BackfillRequest) {\n    const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n    this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n    this.#requiredBackfills.set(req.table, req);\n  }\n\n  #deleteRequiredBackfill(source: string, id: Identifier) {\n    const req = this.#requiredBackfills.get(id);\n    if (req) {\n      const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n      this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n      this.#requiredBackfills.delete(id);\n    }\n  }\n\n  /**\n   * Implements {@link Listener.onChange()}, invoked by the\n   * {@link ChangeStreamMultiplexer}.\n   */\n  onChange(message: ChangeStreamMessage): void {\n    if (message[0] === 'begin') {\n      this.#currentTxWatermark = message[2].commitWatermark;\n      return;\n    }\n    if (message[0] === 'commit') {\n      this.#currentTxWatermark = null;\n      this.#setLastStatusWatermark(message[2]);\n      // Every commit is a candidate for starting the next backfill\n      // (if one is not currently running).\n      this.#checkAndStartBackfill();\n      return;\n    }\n    if (message[0] === 'status') {\n      this.#setLastStatusWatermark(message[2]);\n      return;\n    }\n    if (message[0] !== 'data') {\n      return;\n    }\n    const change = message[1];\n    const {tag} = change;\n    switch (tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest) {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, metadata},\n          });\n          if (this.#backfillRunningFor(table)) {\n            this.#stopRunningBackfill(`TableMetadata updated`);\n          }\n        }\n        break;\n      }\n      case 'create-table': {\n        const {\n          spec: {schema, name},\n          metadata = null,\n          backfill,\n        } = change;\n\n        if (backfill) {\n          this.#setRequiredBackfill(tag, {\n            table: {schema, name, metadata},\n            columns: Object.fromEntries(\n              Object.entries(backfill).map(([col, id]) => [col, {id}]),\n            ),\n          });\n        }\n        break;\n      }\n      case 'rename-table': {\n        const {old, new: newTable} = change;\n        const backfillRequest = this.#requiredBackfills.get(old);\n        if (backfillRequest) {\n          const {schema, name} = newTable;\n          this.#deleteRequiredBackfill(tag, old);\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, schema, name},\n          });\n          if (this.#backfillRunningFor(old)) {\n            this.#stopRunningBackfill(`table renamed`);\n          }\n        }\n        break;\n      }\n      case 'drop-table': {\n        const {id} = change;\n        const backfillRequest = this.#requiredBackfills.get(id);\n        if (backfillRequest) {\n          this.#deleteRequiredBackfill(tag, id);\n          if (this.#backfillRunningFor(id)) {\n            this.#stopRunningBackfill(`table dropped`);\n          }\n        }\n        break;\n      }\n      case 'add-column': {\n        const {\n          table,\n          tableMetadata: metadata = null,\n          column,\n          backfill,\n        } = change;\n        if (backfill) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (!backfillRequest) {\n            this.#setRequiredBackfill(tag, {\n              table: {...table, metadata},\n              columns: {[column.name]: {id: backfill}},\n            });\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              table: {...backfillRequest.table, metadata},\n              columns: {\n                ...backfillRequest.columns,\n                [column.name]: {id: backfill},\n              },\n            });\n            // Note: The running backfill need not be canceled if a\n            //   new column is added. The new column will be backfilled\n            //   by its own stream after the current backfill completes.\n          }\n        }\n        break;\n      }\n      case 'update-column': {\n        const {\n          table,\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (backfillRequest && oldName in backfillRequest.columns) {\n            const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: {...otherCols, [newName]: colSpec},\n            });\n            const backfill = this.#backfillRunningFor(table);\n            if (backfill && oldName in backfill.request.columns) {\n              this.#stopRunningBackfill(`column renamed`);\n            }\n          }\n        }\n        break;\n      }\n      case 'drop-column': {\n        const {table, column} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest && column in backfillRequest.columns) {\n          const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n          if (Object.keys(remaining).length === 0) {\n            this.#deleteRequiredBackfill(tag, table);\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: remaining,\n            });\n          }\n          const backfill = this.#backfillRunningFor(table);\n          if (backfill && column in backfill.request.columns) {\n            this.#stopRunningBackfill(`column dropped`);\n          }\n        }\n        break;\n      }\n      case 'update': {\n        const {relation, key, new: row} = change;\n        const backfill = this.#backfillRunningFor(relation);\n        const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n        if (backfill?.request.table.metadata && key !== null) {\n          // A corner case that backfill is unable to correctly handle is\n          // when a row's key changes; this is decomposed into a delete\n          // of the old key and a set of the new key in the replica change\n          // log, at which point the backfill algorithm assumes that the\n          // (old) row is deleted but does not know to backfill the new row.\n          // In these corner cases, the current backfill is canceled and\n          // retried if its version precedes this update.\n          for (const col of Object.keys(\n            backfill.request.table.metadata.rowKey,\n          )) {\n            if (key[col] !== row[col]) {\n              backfill.minWatermark = txWatermark;\n              this.#lc.info?.(\n                `key for row as changed (col: ${col}). ` +\n                  `backfill data must not predate ${backfill.minWatermark}`,\n              );\n              break;\n            }\n          }\n        }\n        break;\n      }\n      case 'backfill-completed': {\n        const {relation, columns} = change;\n        const backfillRequest = this.#requiredBackfills.get(relation);\n        assert(\n          backfillRequest,\n          () => `No BackfillRequest completed backfill ${stringify(change)}`,\n        );\n        const remaining = Object.entries(backfillRequest.columns).filter(\n          ([col]) =>\n            !(columns.includes(col) || relation.rowKey.columns.includes(col)),\n        );\n        if (remaining.length === 0) {\n          this.#deleteRequiredBackfill(tag, relation);\n        } else {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            columns: Object.fromEntries(remaining),\n          });\n        }\n        // Technically the backfill is already stopping, but this method\n        // cleans up the state that tracks it.\n        this.#stopRunningBackfill();\n        break;\n      }\n    }\n  }\n\n  cancel(): void {\n    this.#controller.abort('canceled');\n    this.#stopRunningBackfill(`change stream canceled`);\n    clearTimeout(this.#backfillRetryTimer);\n    this.#backfillRetryTimer = undefined;\n\n    // Wake up a backfill that is waiting for the change stream to reach a\n    // watermark. The change stream never will, and the backfill must be\n    // allowed to unwind (and release its upstream resources) rather than\n    // remain suspended forever.\n    for (const {reached} of this.#awaitingStatusWatermarks.splice(0)) {\n      reached();\n    }\n  }\n}\n\nabstract class BackfillStreamError extends Error {\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(\n      `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n        `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n      {cause},\n    );\n  }\n}\n\n/**\n * Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n *\n * In terms of backfill, however, non-empty tables without a row key **cannot**\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n *\n * The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n */\nclass MissingRowKeyError extends BackfillStreamError {\n  readonly name = 'MissingRowKeyError';\n\n  constructor(bf: BackfillRequest, cause?: unknown) {\n    super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n  }\n}\n\n/**\n * Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n  readonly name = 'SchemaIncompatibilityError';\n\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(bf, msg, cause);\n  }\n}\n"],"mappings":";;;;;;;;;;;;AA4BA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,GAAG,OAAO,GAAG;AACtB;;;;;;AAsBA,IAAM,yBAAyB,IAAI,OAAO;;;;;;;;AAS1C,IAAM,4BAA4B;;;;;;;;;;;;;;AAelC,IAAM,0BAA0B;;AAGhC,IAAM,iCAAiC;;AAGvC,IAAM,OAAO,OAAO,eAAe;;AAEnC,IAAM,OAAO,OAAO,eAAe;AAmCnC,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,QACF;CACA;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,CAAC;;CAGjE,sBAAqC;CAErC;;;;;;;CAQA,cAAuB,IAAI,gBAAgB;CAE3C,IAAIQ,YAAY;EACd,OAAO,KAAKD,YAAY,OAAO;CACjC;CAEA,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf,uBAAuB,wBACvB;EACA,KAAKP,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKE,kBAAkB;EACvB,KAAKC,oBAAoB;EACzB,KAAKC,cAAc;EACnB,KAAKK,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKL,wBAAwB;CAC/B;CAEA,IAAI,eAAuB,iBAAoC;EAC7D,KAAKN,IAAI,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,gBAAe,CAC5B;EACA,KAAKY,uBAAuB;EAC5B,gBAAgB,SAAQ,QACtB,KAAKC,qBAAqB,mBAAmB,GAAG,CAClD;EACA,KAAKC,uBAAuB;CAC9B;CAEA,wBAAwB,EAAC,aAAiC;EAIxD,KAAK,KAAKF,wBAAwB,MAAM,WAAW;GACjD,KAAKA,uBAAuB;GAC5B,KAAK,IAAI,IAAI,KAAKP,0BAA0B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,KAAKA,0BAA0B;IAChD,IAAI,aAAa,SAAS,WAAW;KACnC,SAAS,QAAQ;KACjB,KAAKA,0BAA0B,OAAO,GAAG,CAAC;IAC5C;GACF;EACF;CACF;CAEA,qBACE,IACA,WACsB;EACtB,KAAK,KAAKO,wBAAwB,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,SAAS;GAC7C,KAAKP,0BAA0B,KAAK;IAAC;IAAW;GAAO,CAAC;GACxD,GAAG,OACD,iCAAiC,KAAKO,qBAAqB,aAAa,WAC1E;GACA,OAAO;EACT;EACA,OAAO;CACT;CAEA;CACA;CACA;CACA;CAEA,yBAAyB;EACvB,IACE,CAAC,KAAKJ,aACN,CAAC,KAAKO,uBACN,CAAC,KAAKC,oBACN,KAAKf,mBAAmB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,KAAKA,mBAAmB,OAAO,CAAC;GACvD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,CAAC;GAC3D,MAAM,QAAQ;IACZ;IACA,cAAc;IACd,SAAS,IAAI,gBAAgB;GAC/B;GACA,MAAM,KAAK,KAAKD,IAAI,YAAY,SAAS,QAAQ,MAAM,IAAI;GAE3D,KAAKgB,mBAAmB;GACxB,KAAUC,aAAa,IAAI,KAAK,EAC7B,WAAW;IACV,KAAKC,qBAAqB,mBAAmB,KAAK;IAClD,KAAKP,gBAAgB,KAAKF;GAC5B,CAAC,EAGA,OAAM,MAAK;IACV,KAAKS,qBAAqB,OAAO,CAAC,GAAG,KAAK;IAC1C,KAAKC,0BAA0B,CAAC;GAClC,CAAC;EACL;CACF;CAEA,0BAA0B,GAAY;EACpC,IAAI,KAAKX,WAAW;GAClB,KAAKR,IAAI,QAAQ,iDAAiD,CAAC;GACnE;EACF;EACA,KAAKA,IAAI,OACP,uCAAuC,KAAKW,cAAc,MAC1D,CACF;EACA,KAAKI,sBAAsB,iBAAiB;GAC1C,KAAKA,sBAAsB,KAAA;GAC3B,KAAKD,uBAAuB;EAC9B,GAAG,KAAKH,aAAa;EAErB,KAAKA,gBAAgB,KAAK,IAAI,KAAKA,gBAAgB,GAAG,KAAKD,aAAa;CAC1E;CAEA,MAAMO,aAAa,IAAgB,OAA6B;EAC9D,MAAM,eAAe,KAAKf;EAI1B,IAAI,aAA4B;EAChC,IAAI,mBAAmB;EAEvB,MAAM,QAA6B;GACjC,OAAO,YAAY,IAAI;GACvB,YAAY,YAAY,IAAI;GAC5B,UAAU;GACV,OAAO;GACP,SAAS;GACT,QAAQ;GACR,WAAW;GACX,QAAQ;GACR,WAAW;GACX,SAAS;GACT,QAAQ;EACV;;;;EAKA,MAAM,aAAa,OACjB,QAC2B;GAC3B,OAAO,eAAe,MAAM,yCAAyC;GACrE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,UAAU;GAI3D,IACE,MAAM,QAAQ,OAAO,WACpB,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;IACA,IAAI,CAAC,MAAM,QAAQ,OAAO,SAAS;KACjC,OAAO,IAAI,QAAQ,YAAY,+BAA+B;KAC9D,KAAKgB,qBACH,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,KACF;IACF;IACA,aAAa,QAAQ,aAAa;IAClC,OAAO;GACT;GAEA,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,aAAa;GAChE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,KAAK,IAAI;GACzB,CAAC;GAED,IAAI,KAAK,IAAI,WAOX,KAAK,IAAI;GAGX,aAAkB,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,KAAKd;KAAa,SAAS;IAAI;IACpD,EAAC,iBAAiB,GAAE;GACtB,CAAC;GACD,OAAQ,aAAa;EACvB;EAEA,MAAM,iBAAiB;GACrB,IAAI,YAAY;IACd,aAAkB,KAAK;KACrB;KACA,EAAC,KAAK,SAAQ;KACd,EAAC,WAAW,WAAU;IACxB,CAAC;IACD,aAAa,QAAQ,UAAU;IAC/B,MAAM;GACR;GACA,aAAa;GACb,mBAAmB;EACrB;EASA,MAAM,QAAQ,IAAI,MAAY;EAC9B,MAAM,UAAU,IAAI,MAAS;EAC7B,KAAK,IAAI,IAAI,GAAG,IAAI,2BAA2B,KAC7C,QAAQ,QAAQ,CAAC;EAEnB,IAAI,eAAe;EAInB,MAAM,iBACJ,MAAM,iBAAiB,IAAI,WAAW,mBAAmB,CAAC;EAC5D,IAAI,CAAC,KAAKI,WACR,KAAKD,YAAY,OAAO,iBAAiB,SAAS,QAAQ;EAG5D,MAAM,YAAY,YAAY;GAC5B,IAAI;IACF,IAAI,KAAK,YAAY,IAAI;IACzB,WAAW,MAAM,SAAS,KAAKJ,kBAAkB,MAAM,OAAO,GAAG;KAC/D,MAAM,aAAa,YAAY,IAAI,IAAI;KACvC,MAAM,QAAQ,QAAQ;KACtB,IAAI,gBAAgB,KAAKK,WAAW;MAClC,MAAM,iBAAiB,IAAI,WAAW,mBAAmB,CAAC;MAC1D;KACF;KACA,MAAM,QAAQ,KAAK;KACnB,KAAK,YAAY,IAAI;IACvB;IACA,MAAM,QAAQ,IAAI;GACpB,SAAS,GAAG;IACV,MAAM,iBAAiB,CAAC;GAC1B;EACF,GAAG;EAIH,SAAc,YAAY,CAAC,CAAC;EAS5B,MAAM,WAAW,YAAoD;GAKnE,IAAI,eAAe,QAAQ,MAAM,KAAK,IAAI,GAAG;IAC3C,MAAM,IAAI,MAAM,QAAQ;IAExB,OADU,aAAa,UAAU,MAAM,IAAI;GAE7C;GAEA,SAAS;IACP,IAAI,eAAe,QAAQ,aAAa,YAAY,IAAI,GACtD,SAAS;IAEX,MAAM,IACJ,eAAe,OACX,MAAM,QAAQ,MAAM,uBAAuB,IAC3C,MAAM,QAAQ;IACpB,MAAM,KAAK,YAAY,IAAI;IAC3B,MAAM,IAAI,aAAa,UAAU,MAAM,IAAI;IAC3C,IAAI,MAAM,MAAM;KACd,MAAM,WAAW,YAAY,IAAI,IAAI;KACrC;IACF;IACA,IAAI,aAAa,SAAS;KACxB,MAAM,WAAW,YAAY,IAAI,IAAI;KACrC,MAAM;IACR;IACA,OAAO;GACT;EACF;EAEA,IAAI;GACF,SAAS;IACP,MAAM,OAAO,MAAM,SAAS;IAC5B,IAAI,SAAS,MACX;IAIF,QAAQ,QAAQ,CAAC;IAEjB,IAAI,KAAKA,WAAW;KAIlB,GAAG,OAAO,kDAAkD;KAC5D;IACF;IACA,MAAM,EAAC,SAAS,KAAK,aAAY;IAOjC,MAAM,sBAAsB,KAAKY,qBAC/B,IACA,IAAI,SACN;IAKA,IACE,eACC,aAAa,YAAY,IAAI,KAC5B,uBACA,oBAAoB,KAAKd,wBAE3B,SAAS;IAGX,IAAI,qBAAqB;KACvB,MAAM,KAAK,YAAY,IAAI;KAG3B,MAAM,eAAe,qBAAqB,MAAM,QAAQ,MAAM;KAC9D,MAAM,UAAU,YAAY,IAAI,IAAI;KACpC,MAAM,UAAU;KAChB,GAAG,OACD,yBAAyB,IAAI,UAAU,iBACpB,KAAK,MAAM,OAAO,EAAE,IACzC;IACF;IAEA,IACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,GAEvC,MAAM,IAAI,mBAAmB,MAAM,OAAO;IAI5C,IAAI,eAAe,MAAM;KACvB,MAAM,KAAK,YAAY,IAAI;KAC3B,aAAa,MAAM,WAAW,GAAG;KACjC,MAAM,aAAa,YAAY,IAAI,IAAI;KACvC,IAAI,eAAe,MAAM;MACvB,GAAG,OACD,6BAA6B,MAAM,QAAQ,OAAO,UAClD,MAAM,OACR;MACA,KAAKQ,uBAAuB;MAC5B;KACF;IACF;IAIA,MAAM,KAAK,YAAY,IAAI;IAC3B,MAAM,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC;IACrC,MAAM,UAAU,YAAY,IAAI,IAAI;IACpC,oBAAoB;IACpB,MAAM;IACN,MAAM,SAAS;IAEf,KAAKO,uBAAuB,IAAI,KAAK;GACvC;GAGA,IAAI,YACF,SAAS;GAEX,GAAG,QAAQ,wBAAwB;EACrC,UAAU;GACR,eAAe;GACf,QAAQ,QAAQ,CAAC;GAGjB,IAAI,CAAC,KAAKb,WACR,MAAM;GAER,KAAKD,YAAY,OAAO,oBAAoB,SAAS,QAAQ;GAC7D,KAAKe,kBAAkB,IAAI,KAAK;EAClC;CACF;CAEA,uBAAuB,IAAgB,OAA4B;EACjE,MAAM,MAAM,YAAY,IAAI;EAC5B,IAAI,MAAM,MAAM,cAAc,gCAAgC;GAC5D,KAAKA,kBAAkB,IAAI,KAAK;GAChC,MAAM,aAAa;EACrB;CACF;;;;;;;;;;;;CAaA,kBAAkB,IAAgB,OAA4B;EAC5D,MAAM,UAAU,YAAY,IAAI,IAAI,MAAM;EAC1C,GAAG,OAAO,mBAAmB;GAC3B,UAAU,MAAM;GAChB,OAAO,MAAM;GACb,SAAS,MAAM;GACf,QAAQ,MAAM;GACd,WAAW,KAAK,MAAM,OAAO;GAC7B,WAAW,KAAK,MAAM,MAAM,SAAS;GACrC,QAAQ,KAAK,MAAM,MAAM,MAAM;GAC/B,WAAW,KAAK,MAAM,MAAM,SAAS;GACrC,SAAS,KAAK,MAAM,MAAM,OAAO;GACjC,QAAQ,KAAK,MAAM,MAAM,MAAM;GAC/B,gBACE,UAAU,IAAI,KAAK,MAAM,MAAM,QAAQ,QAAQ,UAAU,IAAK,IAAI;EACtE,CAAC;CACH;CAEA,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,KAAKN;EACnB,OAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;CACN;;;;;;;CAQA,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,KAAKA;EACtB,IAAI,YAAY,cAAc,YAAY,WAAW;GACnD,SAAS,QAAQ,MAAM,MAAM;GAC7B,KAAKA,mBAAmB;GACxB,UAAU,KAAKhB,IAAI,OAAO,uBAAuB,MAAM;EACzD;CACF;CAEA,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,KAAKC,mBAAmB,IAAI,IAAI,KAAK,IAAI,YAAY;EACpE,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;EAChE,KAAKC,mBAAmB,IAAI,IAAI,OAAO,GAAG;CAC5C;CAEA,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,KAAKA,mBAAmB,IAAI,EAAE;EAC1C,IAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;GAC/D,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;GAChE,KAAKC,mBAAmB,OAAO,EAAE;EACnC;CACF;;;;;CAMA,SAAS,SAAoC;EAC3C,IAAI,QAAQ,OAAO,SAAS;GAC1B,KAAKsB,sBAAsB,QAAQ,GAAG;GACtC;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKA,sBAAsB;GAC3B,KAAKC,wBAAwB,QAAQ,EAAE;GAGvC,KAAKV,uBAAuB;GAC5B;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKU,wBAAwB,QAAQ,EAAE;GACvC;EACF;EACA,IAAI,QAAQ,OAAO,QACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;EACd,QAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,KAAKvB,mBAAmB,IAAI,KAAK;IACzD,IAAI,iBAAiB;KACnB,KAAKY,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;KAC5C,CAAC;KACD,IAAI,KAAKY,oBAAoB,KAAK,GAChC,KAAKP,qBAAqB,uBAAuB;IAErD;IACA;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;IAEJ,IAAI,UACF,KAAKL,qBAAqB,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;KAAQ;KAC9B,SAAS,OAAO,YACd,OAAO,QAAQ,QAAQ,EAAE,KAAK,CAAC,KAAK,QAAQ,CAAC,KAAK,EAAC,GAAE,CAAC,CAAC,CACzD;IACF,CAAC;IAEH;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,KAAKZ,mBAAmB,IAAI,GAAG;IACvD,IAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;KACvB,KAAKyB,wBAAwB,KAAK,GAAG;KACrC,KAAKb,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;MAAI;KAChD,CAAC;KACD,IAAI,KAAKY,oBAAoB,GAAG,GAC9B,KAAKP,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;IAEb,IADwB,KAAKjB,mBAAmB,IAAI,EAChD,GAAiB;KACnB,KAAKyB,wBAAwB,KAAK,EAAE;KACpC,IAAI,KAAKD,oBAAoB,EAAE,GAC7B,KAAKP,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;IACJ,IAAI,UAAU;KACZ,MAAM,kBAAkB,KAAKjB,mBAAmB,IAAI,KAAK;KACzD,IAAI,CAAC,iBACH,KAAKY,qBAAqB,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;MAAQ;MAC1B,SAAS,GAAE,OAAO,OAAO,EAAC,IAAI,SAAQ,EAAC;KACzC,CAAC;UAED,KAAKA,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;MAC1C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO,EAAC,IAAI,SAAQ;MAC9B;KACF,CAAC;IAKL;IACA;GACF;GACA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,KAAKZ,mBAAmB,IAAI,KAAK;KACzD,IAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;MAC3D,KAAKY,qBAAqB,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;OAAO;MAC5C,CAAC;MACD,MAAM,WAAW,KAAKY,oBAAoB,KAAK;MAC/C,IAAI,YAAY,WAAW,SAAS,QAAQ,SAC1C,KAAKP,qBAAqB,gBAAgB;KAE9C;IACF;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,KAAKjB,mBAAmB,IAAI,KAAK;IACzD,IAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;KAC5D,IAAI,OAAO,KAAK,SAAS,EAAE,WAAW,GACpC,KAAKyB,wBAAwB,KAAK,KAAK;UAEvC,KAAKb,qBAAqB,KAAK;MAC7B,GAAG;MACH,SAAS;KACX,CAAC;KAEH,MAAM,WAAW,KAAKY,oBAAoB,KAAK;KAC/C,IAAI,YAAY,UAAU,SAAS,QAAQ,SACzC,KAAKP,qBAAqB,gBAAgB;IAE9C;IACA;GACF;GACA,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,KAAKO,oBAAoB,QAAQ;IAClD,MAAM,cAAc,KAAK,KAAKF,qBAAqB,aAAa;IAChE,IAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ;UAQzC,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,MAClC,GACE,IAAI,IAAI,SAAS,IAAI,MAAM;MACzB,SAAS,eAAe;MACxB,KAAKvB,IAAI,OACP,gCAAgC,IAAI,oCACA,SAAS,cAC/C;MACA;KACF;;IAGJ;GACF;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,KAAKC,mBAAmB,IAAI,QAAQ;IAC5D,OACE,uBACM,yCAAyC,UAAU,MAAM,GACjE;IACA,MAAM,YAAY,OAAO,QAAQ,gBAAgB,OAAO,EAAE,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,GAAG,KAAK,SAAS,OAAO,QAAQ,SAAS,GAAG,EACnE;IACA,IAAI,UAAU,WAAW,GACvB,KAAKyB,wBAAwB,KAAK,QAAQ;SAE1C,KAAKb,qBAAqB,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,SAAS;IACvC,CAAC;IAIH,KAAKK,qBAAqB;IAC1B;GACF;EACF;CACF;CAEA,SAAe;EACb,KAAKX,YAAY,MAAM,UAAU;EACjC,KAAKW,qBAAqB,wBAAwB;EAClD,aAAa,KAAKH,mBAAmB;EACrC,KAAKA,sBAAsB,KAAA;EAM3B,KAAK,MAAM,EAAC,aAAY,KAAKV,0BAA0B,OAAO,CAAC,GAC7D,QAAQ;CAEZ;AACF;AAEA,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,OAC7C,EAAC,MAAK,CACR;CACF;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;EAChD,MAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,KAAK;CAChE;AACF;;;;;;;AAQA,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MAAM,IAAI,KAAK,KAAK;CACtB;AACF"}

package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-stream-multiplexer.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,2BAA2B,CAAC;AAEtD,OAAO,KAAK,EACV,mBAAmB,EACnB,uBAAuB,EACxB,MAAM,wBAAwB,CAAC;AAEhC,MAAM,MAAM,UAAU,GAAG;IACvB,MAAM,IAAI,IAAI,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAAC;CAC7C,CAAC;AAQF;;;GAGG;AACH,qBAAa,uBAAuB;;IAmBlC,YAAY,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,EAMhD;IAED,YAAY,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,GAAG,IAAI,CAGrC;IAED,YAAY,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,GAAG,IAAI,CAGnC;IAED;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAelD;IAED;;;;OAIG;IACH,WAAW,IAAI,MAAM,CAKpB;IAED;;;OAGG;IACH,OAAO,CAAC,YAAY,EAAE,MAAM,QAY3B;IAED;;;;OAIG;IACH,UAAU,CAAC,OAAO,EAAE,uBAAuB,QAS1C;IAED;;;OAGG;IACH,IAAI,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,CAOnD;IAED,IAAI,CAAC,GAAG,EAAE,KAAK,QAEd;IAED,QAAQ,IAAI,MAAM,CAAC,mBAAmB,CAAC,CAEtC;CACF"}
+{"version":3,"file":"change-stream-multiplexer.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,2BAA2B,CAAC;AAEtD,OAAO,KAAK,EACV,mBAAmB,EACnB,uBAAuB,EACxB,MAAM,wBAAwB,CAAC;AAEhC,MAAM,MAAM,UAAU,GAAG;IACvB,MAAM,IAAI,IAAI,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAAC;CAC7C,CAAC;AASF;;;GAGG;AACH,qBAAa,uBAAuB;;IAqBlC,YAAY,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,EAMhD;IAmBD,YAAY,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,GAAG,IAAI,CAGrC;IAED,YAAY,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,GAAG,IAAI,CAGnC;IAED;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA2BlD;IAED;;;;OAIG;IACH,WAAW,IAAI,MAAM,CAKpB;IAED;;;OAGG;IACH,OAAO,CAAC,YAAY,EAAE,MAAM,QAY3B;IAED;;;;OAIG;IACH,UAAU,CAAC,OAAO,EAAE,uBAAuB,QAS1C;IAED;;;OAGG;IACH,IAAI,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,CAOnD;IAED,IAAI,CAAC,GAAG,EAAE,KAAK,QAEd;IAED,QAAQ,IAAI,MAAM,CAAC,mBAAmB,CAAC,CAEtC;CACF"}

package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js

@@ -1,4 +1,5 @@
 import { assert } from "../../../../../shared/src/asserts.js";
+import { AbortError } from "../../../../../shared/src/abort-error.js";
 import { Subscription } from "../../../types/subscription.js";
 import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts
@@ -22,11 +23,20 @@ var ChangeStreamMultiplexer = class {
 	* Tracks the queue of producers waiting for a reservation.
 	*/
 	#waiters = [];
+	#canceled = false;
 	constructor(lc, lastWatermark) {
 		this.#lc = lc;
-		this.#sub = Subscription.create({ cleanup: () => this.#producers.forEach((p) => p.cancel()) });
+		this.#sub = Subscription.create({ cleanup: () => this.#cancel() });
 		this.#lastWatermark = lastWatermark;
 	}
+	#cancel() {
+		this.#canceled = true;
+		this.#producers.forEach((p) => p.cancel());
+		for (const { producer, rejectReservation } of this.#waiters.splice(0)) {
+			this.#lc.info?.(`rejecting reservation request from ${producer}: stream canceled`);
+			rejectReservation(new AbortError("change stream canceled"));
+		}
+	}
 	addProducers(...p) {
 		this.#producers.push(...p);
 		return this;
@@ -46,17 +56,19 @@ var ChangeStreamMultiplexer = class {
 	* @param producer The name of the producer, purely for debugging output
 	*/
 	reserve(producer) {
+		if (this.#canceled) return Promise.reject(new AbortError("change stream canceled"));
 		if (this.#lastWatermark !== null) {
 			const lastWatermark = this.#lastWatermark;
 			this.#lastWatermark = null;
 			return lastWatermark;
 		}
 		const startTime = performance.now();
-		const { promise, resolve: grantReservation } = resolver();
+		const { promise, resolve: grantReservation, reject: rejectReservation } = resolver();
 		this.#waiters.push({
 			producer,
 			startTime,
-			grantReservation
+			grantReservation,
+			rejectReservation
 		});
 		return promise;
 	}

package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-stream-multiplexer.js","names":["#lc","#sub","#producers","#listeners","#waiters","#lastWatermark"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {Source} from '../../../types/streams.ts';\nimport {Subscription} from '../../../types/subscription.ts';\nimport type {\n  ChangeStreamMessage,\n  DownstreamStatusMessage,\n} from '../protocol/current.ts';\n\nexport type Cancelable = {\n  cancel(): void;\n};\n\nexport type Listener = {\n  onChange(change: ChangeStreamMessage): void;\n};\n\ntype Waiter = {\n  producer: string;\n  startTime: number;\n  grantReservation: (watermark: string) => void;\n};\n\n/**\n * Facilitates cooperative multiplexing of transactions from\n * multiple producers into a single change stream.\n */\nexport class ChangeStreamMultiplexer {\n  readonly #lc: LogContext;\n  readonly #sub: Subscription<ChangeStreamMessage>;\n  readonly #producers: Cancelable[] = [];\n  readonly #listeners: Listener[] = [];\n\n  /**\n   * The `#lastWatermark` tracks the watermark of the last transaction\n   * committed to stream. It is set to null when a producer has reserved\n   * the stream, and set to the new watermark when the producer releases\n   * the reservation.\n   */\n  #lastWatermark: string | null;\n\n  /**\n   * Tracks the queue of producers waiting for a reservation.\n   */\n  readonly #waiters: Waiter[] = [];\n\n  constructor(lc: LogContext, lastWatermark: string) {\n    this.#lc = lc;\n    this.#sub = Subscription.create<ChangeStreamMessage>({\n      cleanup: () => this.#producers.forEach(p => p.cancel()),\n    });\n    this.#lastWatermark = lastWatermark;\n  }\n\n  addProducers(...p: Cancelable[]): this {\n    this.#producers.push(...p);\n    return this;\n  }\n\n  addListeners(...l: Listener[]): this {\n    this.#listeners.push(...l);\n    return this;\n  }\n\n  /**\n   * Called by a producer to \"reserve\" the exclusive right to push data\n   * changes to the stream. If the stream is currently reserved,\n   * the resulting Promise must be awaited.\n   *\n   * The producer must then {@link release()} the reservation when it sees\n   * fit.\n   *\n   * @param producer The name of the producer, purely for debugging output\n   */\n  reserve(producer: string): string | Promise<string> {\n    if (this.#lastWatermark !== null) {\n      // If the stream is not reserved, reserve it and return the\n      // watermark.\n      const lastWatermark = this.#lastWatermark;\n      this.#lastWatermark = null;\n      return lastWatermark;\n    }\n\n    // Otherwise, wait for the current reservation to be released.\n    const startTime = performance.now();\n    const {promise, resolve: grantReservation} = resolver<string>();\n    this.#waiters.push({producer, startTime, grantReservation});\n\n    return promise;\n  }\n\n  /**\n   * If there are producers currently awaiting a reservation, returns\n   * the duration (in milliseconds) of the oldest reservation request.\n   * Returns a negative number if there are no waiters.\n   */\n  waiterDelay(): number {\n    if (this.#waiters.length === 0) {\n      return -1;\n    }\n    return performance.now() - this.#waiters[0].startTime;\n  }\n\n  /**\n   * Called by a producer to release its reservation after committing its\n   * last transaction.\n   */\n  release(newWatermark: string) {\n    const waiter = this.#waiters.shift();\n    if (!waiter) {\n      this.#lastWatermark = newWatermark;\n    } else {\n      const {producer, startTime, grantReservation} = waiter;\n      grantReservation(newWatermark);\n      const elapsed = performance.now() - startTime;\n      this.#lc.info?.(\n        `${producer} waited ${elapsed.toFixed(3)} ms for stream reservation`,\n      );\n    }\n  }\n\n  /**\n   * `pushStatus()` can be called without a reservation, as it\n   * does not constitute a data change and can appear anywhere\n   * in the stream.\n   */\n  pushStatus(message: DownstreamStatusMessage) {\n    // Let listeners know about all status messages.\n    this.#listeners.forEach(l => l.onChange(message));\n    // The ChangeStreamer only cares about status messages requiring an ack\n    // or containing a lagReport. To reduce churn, avoid sending other status\n    // messages.\n    if (message[1].ack || message[1].lagReport) {\n      this.#sub.push(message);\n    }\n  }\n\n  /**\n   * `push()` must only be called by a producer after it has\n   * {@link reserve}d the stream.\n   */\n  push(message: ChangeStreamMessage): Promise<unknown> {\n    assert(\n      this.#lastWatermark === null,\n      `push() called without reserve()-ing the stream`,\n    );\n    this.#listeners.forEach(l => l.onChange(message));\n    return this.#sub.push(message).result;\n  }\n\n  fail(err: Error) {\n    this.#sub.fail(err);\n  }\n\n  asSource(): Source<ChangeStreamMessage> {\n    return this.#sub;\n  }\n}\n"],"mappings":";;;;;;;;AA4BA,IAAa,0BAAb,MAAqC;CACnC;CACA;CACA,aAAoC,CAAC;CACrC,aAAkC,CAAC;;;;;;;CAQnC;;;;CAKA,WAA8B,CAAC;CAE/B,YAAY,IAAgB,eAAuB;EACjD,KAAKA,MAAM;EACX,KAAKC,OAAO,aAAa,OAA4B,EACnD,eAAe,KAAKC,WAAW,SAAQ,MAAK,EAAE,OAAO,CAAC,EACxD,CAAC;EACD,KAAKG,iBAAiB;CACxB;CAEA,aAAa,GAAG,GAAuB;EACrC,KAAKH,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;CAEA,aAAa,GAAG,GAAqB;EACnC,KAAKC,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;;;;;;;;;;;CAYA,QAAQ,UAA4C;EAClD,IAAI,KAAKE,mBAAmB,MAAM;GAGhC,MAAM,gBAAgB,KAAKA;GAC3B,KAAKA,iBAAiB;GACtB,OAAO;EACT;EAGA,MAAM,YAAY,YAAY,IAAI;EAClC,MAAM,EAAC,SAAS,SAAS,qBAAoB,SAAiB;EAC9D,KAAKD,SAAS,KAAK;GAAC;GAAU;GAAW;EAAgB,CAAC;EAE1D,OAAO;CACT;;;;;;CAOA,cAAsB;EACpB,IAAI,KAAKA,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO,YAAY,IAAI,IAAI,KAAKA,SAAS,GAAG;CAC9C;;;;;CAMA,QAAQ,cAAsB;EAC5B,MAAM,SAAS,KAAKA,SAAS,MAAM;EACnC,IAAI,CAAC,QACH,KAAKC,iBAAiB;OACjB;GACL,MAAM,EAAC,UAAU,WAAW,qBAAoB;GAChD,iBAAiB,YAAY;GAC7B,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,KAAKL,IAAI,OACP,GAAG,SAAS,UAAU,QAAQ,QAAQ,CAAC,EAAE,2BAC3C;EACF;CACF;;;;;;CAOA,WAAW,SAAkC;EAE3C,KAAKG,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAIhD,IAAI,QAAQ,GAAG,OAAO,QAAQ,GAAG,WAC/B,KAAKF,KAAK,KAAK,OAAO;CAE1B;;;;;CAMA,KAAK,SAAgD;EACnD,OACE,KAAKI,mBAAmB,MACxB,gDACF;EACA,KAAKF,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAChD,OAAO,KAAKF,KAAK,KAAK,OAAO,EAAE;CACjC;CAEA,KAAK,KAAY;EACf,KAAKA,KAAK,KAAK,GAAG;CACpB;CAEA,WAAwC;EACtC,OAAO,KAAKA;CACd;AACF"}
+{"version":3,"file":"change-stream-multiplexer.js","names":["#lc","#sub","#producers","#listeners","#waiters","#cancel","#lastWatermark","#canceled"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {Source} from '../../../types/streams.ts';\nimport {Subscription} from '../../../types/subscription.ts';\nimport type {\n  ChangeStreamMessage,\n  DownstreamStatusMessage,\n} from '../protocol/current.ts';\n\nexport type Cancelable = {\n  cancel(): void;\n};\n\nexport type Listener = {\n  onChange(change: ChangeStreamMessage): void;\n};\n\ntype Waiter = {\n  producer: string;\n  startTime: number;\n  grantReservation: (watermark: string) => void;\n  rejectReservation: (err: Error) => void;\n};\n\n/**\n * Facilitates cooperative multiplexing of transactions from\n * multiple producers into a single change stream.\n */\nexport class ChangeStreamMultiplexer {\n  readonly #lc: LogContext;\n  readonly #sub: Subscription<ChangeStreamMessage>;\n  readonly #producers: Cancelable[] = [];\n  readonly #listeners: Listener[] = [];\n\n  /**\n   * The `#lastWatermark` tracks the watermark of the last transaction\n   * committed to stream. It is set to null when a producer has reserved\n   * the stream, and set to the new watermark when the producer releases\n   * the reservation.\n   */\n  #lastWatermark: string | null;\n\n  /**\n   * Tracks the queue of producers waiting for a reservation.\n   */\n  readonly #waiters: Waiter[] = [];\n\n  #canceled = false;\n\n  constructor(lc: LogContext, lastWatermark: string) {\n    this.#lc = lc;\n    this.#sub = Subscription.create<ChangeStreamMessage>({\n      cleanup: () => this.#cancel(),\n    });\n    this.#lastWatermark = lastWatermark;\n  }\n\n  #cancel() {\n    this.#canceled = true;\n    this.#producers.forEach(p => p.cancel());\n\n    // A producer awaiting a reservation would otherwise remain suspended\n    // forever, since the reservation held when the stream was canceled is\n    // never released. Reject the pending reservations so that the producers\n    // unwind and release the resources (e.g. upstream connections and\n    // transactions) that they were holding for the stream.\n    for (const {producer, rejectReservation} of this.#waiters.splice(0)) {\n      this.#lc.info?.(\n        `rejecting reservation request from ${producer}: stream canceled`,\n      );\n      rejectReservation(new AbortError('change stream canceled'));\n    }\n  }\n\n  addProducers(...p: Cancelable[]): this {\n    this.#producers.push(...p);\n    return this;\n  }\n\n  addListeners(...l: Listener[]): this {\n    this.#listeners.push(...l);\n    return this;\n  }\n\n  /**\n   * Called by a producer to \"reserve\" the exclusive right to push data\n   * changes to the stream. If the stream is currently reserved,\n   * the resulting Promise must be awaited.\n   *\n   * The producer must then {@link release()} the reservation when it sees\n   * fit.\n   *\n   * @param producer The name of the producer, purely for debugging output\n   */\n  reserve(producer: string): string | Promise<string> {\n    if (this.#canceled) {\n      return Promise.reject(new AbortError('change stream canceled'));\n    }\n    if (this.#lastWatermark !== null) {\n      // If the stream is not reserved, reserve it and return the\n      // watermark.\n      const lastWatermark = this.#lastWatermark;\n      this.#lastWatermark = null;\n      return lastWatermark;\n    }\n\n    // Otherwise, wait for the current reservation to be released.\n    const startTime = performance.now();\n    const {\n      promise,\n      resolve: grantReservation,\n      reject: rejectReservation,\n    } = resolver<string>();\n    this.#waiters.push({\n      producer,\n      startTime,\n      grantReservation,\n      rejectReservation,\n    });\n\n    return promise;\n  }\n\n  /**\n   * If there are producers currently awaiting a reservation, returns\n   * the duration (in milliseconds) of the oldest reservation request.\n   * Returns a negative number if there are no waiters.\n   */\n  waiterDelay(): number {\n    if (this.#waiters.length === 0) {\n      return -1;\n    }\n    return performance.now() - this.#waiters[0].startTime;\n  }\n\n  /**\n   * Called by a producer to release its reservation after committing its\n   * last transaction.\n   */\n  release(newWatermark: string) {\n    const waiter = this.#waiters.shift();\n    if (!waiter) {\n      this.#lastWatermark = newWatermark;\n    } else {\n      const {producer, startTime, grantReservation} = waiter;\n      grantReservation(newWatermark);\n      const elapsed = performance.now() - startTime;\n      this.#lc.info?.(\n        `${producer} waited ${elapsed.toFixed(3)} ms for stream reservation`,\n      );\n    }\n  }\n\n  /**\n   * `pushStatus()` can be called without a reservation, as it\n   * does not constitute a data change and can appear anywhere\n   * in the stream.\n   */\n  pushStatus(message: DownstreamStatusMessage) {\n    // Let listeners know about all status messages.\n    this.#listeners.forEach(l => l.onChange(message));\n    // The ChangeStreamer only cares about status messages requiring an ack\n    // or containing a lagReport. To reduce churn, avoid sending other status\n    // messages.\n    if (message[1].ack || message[1].lagReport) {\n      this.#sub.push(message);\n    }\n  }\n\n  /**\n   * `push()` must only be called by a producer after it has\n   * {@link reserve}d the stream.\n   */\n  push(message: ChangeStreamMessage): Promise<unknown> {\n    assert(\n      this.#lastWatermark === null,\n      `push() called without reserve()-ing the stream`,\n    );\n    this.#listeners.forEach(l => l.onChange(message));\n    return this.#sub.push(message).result;\n  }\n\n  fail(err: Error) {\n    this.#sub.fail(err);\n  }\n\n  asSource(): Source<ChangeStreamMessage> {\n    return this.#sub;\n  }\n}\n"],"mappings":";;;;;;;;;AA8BA,IAAa,0BAAb,MAAqC;CACnC;CACA;CACA,aAAoC,CAAC;CACrC,aAAkC,CAAC;;;;;;;CAQnC;;;;CAKA,WAA8B,CAAC;CAE/B,YAAY;CAEZ,YAAY,IAAgB,eAAuB;EACjD,KAAKA,MAAM;EACX,KAAKC,OAAO,aAAa,OAA4B,EACnD,eAAe,KAAKI,QAAQ,EAC9B,CAAC;EACD,KAAKC,iBAAiB;CACxB;CAEA,UAAU;EACR,KAAKC,YAAY;EACjB,KAAKL,WAAW,SAAQ,MAAK,EAAE,OAAO,CAAC;EAOvC,KAAK,MAAM,EAAC,UAAU,uBAAsB,KAAKE,SAAS,OAAO,CAAC,GAAG;GACnE,KAAKJ,IAAI,OACP,sCAAsC,SAAS,kBACjD;GACA,kBAAkB,IAAI,WAAW,wBAAwB,CAAC;EAC5D;CACF;CAEA,aAAa,GAAG,GAAuB;EACrC,KAAKE,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;CAEA,aAAa,GAAG,GAAqB;EACnC,KAAKC,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;;;;;;;;;;;CAYA,QAAQ,UAA4C;EAClD,IAAI,KAAKI,WACP,OAAO,QAAQ,OAAO,IAAI,WAAW,wBAAwB,CAAC;EAEhE,IAAI,KAAKD,mBAAmB,MAAM;GAGhC,MAAM,gBAAgB,KAAKA;GAC3B,KAAKA,iBAAiB;GACtB,OAAO;EACT;EAGA,MAAM,YAAY,YAAY,IAAI;EAClC,MAAM,EACJ,SACA,SAAS,kBACT,QAAQ,sBACN,SAAiB;EACrB,KAAKF,SAAS,KAAK;GACjB;GACA;GACA;GACA;EACF,CAAC;EAED,OAAO;CACT;;;;;;CAOA,cAAsB;EACpB,IAAI,KAAKA,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO,YAAY,IAAI,IAAI,KAAKA,SAAS,GAAG;CAC9C;;;;;CAMA,QAAQ,cAAsB;EAC5B,MAAM,SAAS,KAAKA,SAAS,MAAM;EACnC,IAAI,CAAC,QACH,KAAKE,iBAAiB;OACjB;GACL,MAAM,EAAC,UAAU,WAAW,qBAAoB;GAChD,iBAAiB,YAAY;GAC7B,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,KAAKN,IAAI,OACP,GAAG,SAAS,UAAU,QAAQ,QAAQ,CAAC,EAAE,2BAC3C;EACF;CACF;;;;;;CAOA,WAAW,SAAkC;EAE3C,KAAKG,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAIhD,IAAI,QAAQ,GAAG,OAAO,QAAQ,GAAG,WAC/B,KAAKF,KAAK,KAAK,OAAO;CAE1B;;;;;CAMA,KAAK,SAAgD;EACnD,OACE,KAAKK,mBAAmB,MACxB,gDACF;EACA,KAAKH,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAChD,OAAO,KAAKF,KAAK,KAAK,OAAO,EAAE;CACjC;CAEA,KAAK,KAAY;EACf,KAAKA,KAAK,KAAK,GAAG;CACpB;CAEA,WAAwC;EACtC,OAAO,KAAKA;CACd;AACF"}

package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts

@@ -6,6 +6,7 @@ export declare function upgradeReplica(log: LogContext, debugName: string, dbPat
 export declare const CREATE_V6_COLUMN_METADATA_TABLE = "\n  CREATE TABLE \"_zero.column_metadata\" (\n    table_name TEXT NOT NULL,\n    column_name TEXT NOT NULL,\n    upstream_type TEXT NOT NULL,\n    is_not_null INTEGER NOT NULL,\n    is_enum INTEGER NOT NULL,\n    is_array INTEGER NOT NULL,\n    character_max_length INTEGER,\n    PRIMARY KEY (table_name, column_name)\n  );\n";
 export declare const CREATE_V7_CHANGE_LOG = "\n  CREATE TABLE \"_zero.changeLog2\" (\n    \"stateVersion\"              TEXT NOT NULL,\n    \"pos\"                       INT  NOT NULL,\n    \"table\"                     TEXT NOT NULL,\n    \"rowKey\"                    TEXT NOT NULL,\n    \"op\"                        TEXT NOT NULL,\n    PRIMARY KEY(\"stateVersion\", \"pos\"),\n    UNIQUE(\"table\", \"rowKey\")\n  );\n";
 export declare const CREATE_V9_TABLE_METADATA_TABLE = "\n  CREATE TABLE \"_zero.tableMetadata\" (\n    \"schema\"    TEXT NOT NULL,\n    \"table\"     TEXT NOT NULL,\n    \"metadata\"  TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n";
+export declare const CREATE_V17_BACKFILLING_TABLE = "\n  CREATE TABLE \"_zero.backfilling\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n";
 export declare const V14_CHANGE_LOG_STREAM_TABLE = "_zero.changeLogStream";
 export declare const CREATE_V14_CHANGE_LOG_STREAM = "\n  CREATE TABLE \"_zero.changeLogStream\" (\n    \"watermark\"   TEXT NOT NULL,\n    \"pos\"         INTEGER NOT NULL,\n    \"change\"      TEXT NOT NULL,\n    \"precommit\"   TEXT,\n    \"writeTimeMs\" INTEGER,\n    PRIMARY KEY (\"watermark\", \"pos\")\n  );\n\n  CREATE INDEX \"_zero.changeLogStream_writeTimeMs\"\n    ON \"_zero.changeLogStream\" (\"writeTimeMs\", \"watermark\")\n    WHERE \"writeTimeMs\" IS NOT NULL;\n";
 export declare const schemaVersionMigrationMap: IncrementalMigrationMap;