@rocicorp/zero 0.26.1 → 0.26.2-canary.1

package/out/zero-cache/src/services/change-source/common/backfill-manager.js

@@ -1,426 +1,427 @@
-import { resolver } from "@rocicorp/resolver";
 import { assert } from "../../../../../shared/src/asserts.js";
-import { stringify } from "../../../../../shared/src/bigint-json.js";
-import { CustomKeyMap } from "../../../../../shared/src/custom-key-map.js";
 import { must } from "../../../../../shared/src/must.js";
+import { stringify } from "../../../../../shared/src/bigint-json.js";
 import { randInt } from "../../../../../shared/src/rand.js";
-import { JSON_STRINGIFIED } from "../../../types/lite.js";
 import { stateVersionFromString, stateVersionToString } from "../../../types/state-version.js";
+import "../../../types/lite.js";
+import { CustomKeyMap } from "../../../../../shared/src/custom-key-map.js";
+import { resolver } from "@rocicorp/resolver";
+//#region ../zero-cache/src/services/change-source/common/backfill-manager.ts
 function tableKey({ schema, name }) {
-  return `${schema}.${name}`;
-}
-const MIN_BACKOFF_INTERVAL_MS = 2e3;
-const MAX_BACKOFF_INTERVAL_MS = 6e4;
-class BackfillManager {
-  #lc;
-  /**
-   * Tracks the metadata of required backfills based on schema changes
-   * and initial backfill requests.
-   */
-  #requiredBackfills = new CustomKeyMap(
-    tableKey
-  );
-  #changeStreamer;
-  #backfillStreamer;
-  #jsonFormat;
-  /**
-   * The current running backfill. The backfill request is always also in
-   * `#requiredBackfills` (technically, it can be a subset of what's in
-   * `#requiredBackfills`); the request is removed from `#requiredBackfills`
-   * upon completion.
-   */
-  #runningBackfill = null;
-  /** The last seen watermark in the change stream. */
-  #lastStatusWatermark = null;
-  #awaitingStatusWatermarks = [];
-  /** The watermark of the current transaction in the change stream. */
-  #currentTxWatermark = null;
-  constructor(lc, changeStreamer, backfillStreamer, jsonFormat = JSON_STRINGIFIED, minBackoffMs = MIN_BACKOFF_INTERVAL_MS, maxBackoffMs = MAX_BACKOFF_INTERVAL_MS) {
-    this.#lc = lc.withContext("component", "backfill-manager");
-    this.#changeStreamer = changeStreamer;
-    this.#backfillStreamer = backfillStreamer;
-    this.#jsonFormat = jsonFormat;
-    this.#minBackoffMs = minBackoffMs;
-    this.#maxBackoffMs = maxBackoffMs;
-    this.#retryDelayMs = minBackoffMs;
-  }
-  run(lastWatermark, initialRequests) {
-    this.#lc.info?.(
-      `starting backfill manager with ${initialRequests.length} initial requests`,
-      { requests: initialRequests }
-    );
-    this.#lastStatusWatermark = lastWatermark;
-    initialRequests.forEach(
-      (req) => this.#setRequiredBackfill("initial-request", req)
-    );
-    this.#checkAndStartBackfill();
-  }
-  #setLastStatusWatermark({ watermark }) {
-    if ((this.#lastStatusWatermark ?? "") < watermark) {
-      this.#lastStatusWatermark = watermark;
-      for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {
-        const awaiting = this.#awaitingStatusWatermarks[i];
-        if (watermark >= awaiting.watermark) {
-          awaiting.reached();
-          this.#awaitingStatusWatermarks.splice(i, 1);
-        }
-      }
-    }
-  }
-  #changeStreamReached(lc, watermark) {
-    if ((this.#lastStatusWatermark ?? "") < watermark) {
-      const { promise, resolve: reached } = resolver();
-      this.#awaitingStatusWatermarks.push({ watermark, reached });
-      lc.info?.(
-        `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`
-      );
-      return promise;
-    }
-    return null;
-  }
-  #minBackoffMs;
-  #maxBackoffMs;
-  #retryDelayMs;
-  #backfillRetryTimer;
-  #checkAndStartBackfill() {
-    if (!this.#backfillRetryTimer && !this.#runningBackfill && this.#requiredBackfills.size) {
-      const candidates = [...this.#requiredBackfills.values()];
-      const request = candidates[randInt(0, candidates.length - 1)];
-      const state = { request, minWatermark: "" };
-      const lc = this.#lc.withContext("table", request.table.name);
-      this.#runningBackfill = state;
-      void this.#runBackfill(lc, state).then(() => {
-        this.#stopRunningBackfill("backfill exited", state);
-        this.#retryDelayMs = this.#minBackoffMs;
-      }).catch((e) => {
-        this.#stopRunningBackfill(String(e), state);
-        this.#retryBackfillWithBackoff(e);
-      });
-    }
-  }
-  #retryBackfillWithBackoff(e) {
-    const log = this.#retryDelayMs === this.#maxBackoffMs ? "error" : "warn";
-    this.#lc[log]?.(
-      `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,
-      e
-    );
-    this.#backfillRetryTimer = setTimeout(() => {
-      this.#backfillRetryTimer = void 0;
-      this.#checkAndStartBackfill();
-    }, this.#retryDelayMs);
-    this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);
-  }
-  async #runBackfill(lc, state) {
-    const changeStream = this.#changeStreamer;
-    let backfillTx = null;
-    const beginTxFor = async (msg) => {
-      assert(backfillTx === null, "Expected no active backfill transaction");
-      const lastWatermark = await changeStream.reserve("backfill");
-      if (state.canceledReason || msg.tag === "backfill" && msg.watermark < state.minWatermark) {
-        if (state.canceledReason === void 0) {
-          assert(msg.tag === "backfill", "Expected backfill message tag");
-          this.#stopRunningBackfill(
-            `row key change at ${state.minWatermark} postdates backfill watermark at ${msg.watermark}`,
-            state
-          );
-        }
-        changeStream.release(lastWatermark);
-        return null;
-      }
-      const { major, minor = 0n } = stateVersionFromString(lastWatermark);
-      const tx = stateVersionToString({
-        major,
-        minor: BigInt(minor) + 1n
-      });
-      void changeStream.push([
-        "begin",
-        { tag: "begin", json: this.#jsonFormat, skipAck: true },
-        { commitWatermark: tx }
-      ]);
-      return backfillTx = tx;
-    };
-    const commitTx = () => {
-      if (backfillTx) {
-        void changeStream.push([
-          "commit",
-          { tag: "commit" },
-          { watermark: backfillTx }
-        ]);
-        changeStream.release(backfillTx);
-      }
-      backfillTx = null;
-    };
-    for await (const msg of this.#backfillStreamer(state.request)) {
-      const mustWaitBeforeFlush = msg.tag === "backfill-completed" && this.#changeStreamReached(lc, msg.watermark);
-      if (backfillTx && (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush)) {
-        commitTx();
-      }
-      mustWaitBeforeFlush && await mustWaitBeforeFlush;
-      if (msg.tag === "backfill" && msg.rowValues.length > 0 && msg.relation.rowKey.columns.length === 0) {
-        throw new MissingRowKeyError(state.request);
-      }
-      if ((backfillTx ??= await beginTxFor(msg)) === null) {
-        lc.info?.(
-          `backfill stream canceled: ${state.canceledReason}`,
-          state.request
-        );
-        this.#checkAndStartBackfill();
-        return;
-      }
-      await changeStream.push(["data", msg]);
-    }
-    backfillTx && commitTx();
-    lc.debug?.(`backfill stream exited`, state.canceledReason ?? "");
-  }
-  #backfillRunningFor(table) {
-    const state = this.#runningBackfill;
-    return state?.request.table.schema === table.schema && state.request.table.name === table.name ? state : null;
-  }
-  /**
-   * Stops the running backfill for the specified `reason`. If `instance` is
-   * specified, the running backfill is stopped only if it is that instance.
-   * This allows the running backfill itself to clear backfill state without
-   * accidentally stopping a different (e.g. subsequent) backfill.
-   */
-  #stopRunningBackfill(reason, instance) {
-    const backfill = this.#runningBackfill;
-    if (backfill && backfill === (instance ?? backfill)) {
-      backfill.canceledReason = reason;
-      this.#runningBackfill = null;
-      reason && this.#lc.info?.(`canceling backfill:`, reason);
-    }
-  }
-  #setRequiredBackfill(source, req) {
-    const action = this.#requiredBackfills.has(req.table) ? "updated" : "added";
-    this.#lc.info?.(`Backfill ${action}: ${source}`, { backfill: req });
-    this.#requiredBackfills.set(req.table, req);
-  }
-  #deleteRequiredBackfill(source, id) {
-    const req = this.#requiredBackfills.get(id);
-    if (req) {
-      const action = source === "backfill-completed" ? "completed" : "dropped";
-      this.#lc.info?.(`Backfill ${action}: ${source}`, { backfill: req });
-      this.#requiredBackfills.delete(id);
-    }
-  }
-  /**
-   * Implements {@link Listener.onChange()}, invoked by the
-   * {@link ChangeStreamMultiplexer}.
-   */
-  onChange(message) {
-    if (message[0] === "begin") {
-      this.#currentTxWatermark = message[2].commitWatermark;
-      return;
-    }
-    if (message[0] === "commit") {
-      this.#currentTxWatermark = null;
-      this.#setLastStatusWatermark(message[2]);
-      this.#checkAndStartBackfill();
-      return;
-    }
-    if (message[0] === "status") {
-      this.#setLastStatusWatermark(message[2]);
-      return;
-    }
-    if (message[0] !== "data") {
-      return;
-    }
-    const change = message[1];
-    const { tag } = change;
-    switch (tag) {
-      case "update-table-metadata": {
-        const { table, new: metadata } = change;
-        const backfillRequest = this.#requiredBackfills.get(table);
-        if (backfillRequest) {
-          this.#setRequiredBackfill(tag, {
-            ...backfillRequest,
-            table: { ...backfillRequest.table, metadata }
-          });
-          if (this.#backfillRunningFor(table)) {
-            this.#stopRunningBackfill(`TableMetadata updated`);
-          }
-        }
-        break;
-      }
-      case "create-table": {
-        const {
-          spec: { schema, name },
-          metadata = null,
-          backfill
-        } = change;
-        if (backfill) {
-          this.#setRequiredBackfill(tag, {
-            table: { schema, name, metadata },
-            columns: backfill
-          });
-        }
-        break;
-      }
-      case "rename-table": {
-        const { old, new: newTable } = change;
-        const backfillRequest = this.#requiredBackfills.get(old);
-        if (backfillRequest) {
-          const { schema, name } = newTable;
-          this.#deleteRequiredBackfill(tag, old);
-          this.#setRequiredBackfill(tag, {
-            ...backfillRequest,
-            table: { ...backfillRequest.table, schema, name }
-          });
-          if (this.#backfillRunningFor(old)) {
-            this.#stopRunningBackfill(`table renamed`);
-          }
-        }
-        break;
-      }
-      case "drop-table": {
-        const { id } = change;
-        const backfillRequest = this.#requiredBackfills.get(id);
-        if (backfillRequest) {
-          this.#deleteRequiredBackfill(tag, id);
-          if (this.#backfillRunningFor(id)) {
-            this.#stopRunningBackfill(`table dropped`);
-          }
-        }
-        break;
-      }
-      case "add-column": {
-        const {
-          table,
-          tableMetadata: metadata = null,
-          column,
-          backfill
-        } = change;
-        if (backfill) {
-          const backfillRequest = this.#requiredBackfills.get(table);
-          if (!backfillRequest) {
-            this.#setRequiredBackfill(tag, {
-              table: { ...table, metadata },
-              columns: { [column.name]: backfill }
-            });
-          } else {
-            this.#setRequiredBackfill(tag, {
-              ...backfillRequest,
-              table: { ...backfillRequest.table, metadata },
-              columns: {
-                ...backfillRequest.columns,
-                [column.name]: backfill
-              }
-            });
-          }
-        }
-        break;
-      }
-      case "update-column": {
-        const {
-          table,
-          old: { name: oldName },
-          new: { name: newName }
-        } = change;
-        if (oldName !== newName) {
-          const backfillRequest = this.#requiredBackfills.get(table);
-          if (backfillRequest && oldName in backfillRequest.columns) {
-            const { [oldName]: colSpec, ...otherCols } = backfillRequest.columns;
-            this.#setRequiredBackfill(tag, {
-              ...backfillRequest,
-              columns: { ...otherCols, [newName]: colSpec }
-            });
-            const backfill = this.#backfillRunningFor(table);
-            if (backfill && oldName in backfill.request.columns) {
-              this.#stopRunningBackfill(`column renamed`);
-            }
-          }
-        }
-        break;
-      }
-      case "drop-column": {
-        const { table, column } = change;
-        const backfillRequest = this.#requiredBackfills.get(table);
-        if (backfillRequest && column in backfillRequest.columns) {
-          const { [column]: _excluded, ...remaining } = backfillRequest.columns;
-          if (Object.keys(remaining).length === 0) {
-            this.#deleteRequiredBackfill(tag, table);
-          } else {
-            this.#setRequiredBackfill(tag, {
-              ...backfillRequest,
-              columns: remaining
-            });
-          }
-          const backfill = this.#backfillRunningFor(table);
-          if (backfill && column in backfill.request.columns) {
-            this.#stopRunningBackfill(`column dropped`);
-          }
-        }
-        break;
-      }
-      case "update": {
-        const { relation, key, new: row } = change;
-        const backfill = this.#backfillRunningFor(relation);
-        const txWatermark = must(this.#currentTxWatermark, `not in a tx`);
-        if (backfill?.request.table.metadata && key !== null) {
-          for (const col of Object.keys(
-            backfill.request.table.metadata.rowKey
-          )) {
-            if (key[col] !== row[col]) {
-              backfill.minWatermark = txWatermark;
-              this.#lc.info?.(
-                `key for row as changed (col: ${col}). backfill data must not predate ${backfill.minWatermark}`
-              );
-              break;
-            }
-          }
-        }
-        break;
-      }
-      case "backfill-completed": {
-        const { relation, columns } = change;
-        const backfillRequest = this.#requiredBackfills.get(relation);
-        assert(
-          backfillRequest,
-          () => `No BackfillRequest completed backfill ${stringify(change)}`
-        );
-        const remaining = Object.entries(backfillRequest.columns).filter(
-          ([col]) => !(columns.includes(col) || relation.rowKey.columns.includes(col))
-        );
-        if (remaining.length === 0) {
-          this.#deleteRequiredBackfill(tag, relation);
-        } else {
-          this.#setRequiredBackfill(tag, {
-            ...backfillRequest,
-            columns: Object.fromEntries(remaining)
-          });
-        }
-        this.#stopRunningBackfill();
-        break;
-      }
-    }
-  }
-  cancel() {
-    this.#stopRunningBackfill(`change stream canceled`);
-    clearTimeout(this.#backfillRetryTimer);
-  }
+	return `${schema}.${name}`;
 }
-class BackfillStreamError extends Error {
-  constructor(bf, msg, cause) {
-    super(
-      `Cannot backfill ${bf.table.schema}.${bf.table.name}[${Object.keys(bf.columns).join(",")}]: ${msg}`,
-      { cause }
-    );
-  }
-}
-class MissingRowKeyError extends BackfillStreamError {
-  name = "MissingRowKeyError";
-  constructor(bf, cause) {
-    super(bf, `"${bf.table.name}" is missing a PRIMARY KEY`, cause);
-  }
-}
-class SchemaIncompatibilityError extends BackfillStreamError {
-  name = "SchemaIncompatibilityError";
-  constructor(bf, msg, cause) {
-    super(bf, msg, cause);
-  }
-}
-export {
-  BackfillManager,
-  SchemaIncompatibilityError
+var MIN_BACKOFF_INTERVAL_MS = 2e3;
+var MAX_BACKOFF_INTERVAL_MS = 6e4;
+/**
+* The BackfillManager initiates backfills for BackfillRequests from the
+* change-streamer (i.e. unfinished backfills from previous sessions)
+* or for new backfills signaled by `create-table` or `add-column` messages
+* from the change-source.
+*
+* The BackfillManager registers itself as a change stream listener in order
+* to track necessary backfills, and potentially invalidate the in-progress
+* backfill (e.g. due to a schema change) so that it can be retried at a
+* new snapshot.
+*
+* The manager also handles low priority streaming of the backfill messages
+* using the {@link ChangeStreamMultiplexer}, implementing a policy of always
+* releasing its reservation if another producer (i.e. the main change stream)
+* has messages to stream.
+*/
+var BackfillManager = class {
+	#lc;
+	/**
+	* Tracks the metadata of required backfills based on schema changes
+	* and initial backfill requests.
+	*/
+	#requiredBackfills = new CustomKeyMap(tableKey);
+	#changeStreamer;
+	#backfillStreamer;
+	#jsonFormat;
+	/**
+	* The current running backfill. The backfill request is always also in
+	* `#requiredBackfills` (technically, it can be a subset of what's in
+	* `#requiredBackfills`); the request is removed from `#requiredBackfills`
+	* upon completion.
+	*/
+	#runningBackfill = null;
+	/** The last seen watermark in the change stream. */
+	#lastStatusWatermark = null;
+	#awaitingStatusWatermarks = [];
+	/** The watermark of the current transaction in the change stream. */
+	#currentTxWatermark = null;
+	constructor(lc, changeStreamer, backfillStreamer, jsonFormat = "s", minBackoffMs = MIN_BACKOFF_INTERVAL_MS, maxBackoffMs = MAX_BACKOFF_INTERVAL_MS) {
+		this.#lc = lc.withContext("component", "backfill-manager");
+		this.#changeStreamer = changeStreamer;
+		this.#backfillStreamer = backfillStreamer;
+		this.#jsonFormat = jsonFormat;
+		this.#minBackoffMs = minBackoffMs;
+		this.#maxBackoffMs = maxBackoffMs;
+		this.#retryDelayMs = minBackoffMs;
+	}
+	run(lastWatermark, initialRequests) {
+		this.#lc.info?.(`starting backfill manager with ${initialRequests.length} initial requests`, { requests: initialRequests });
+		this.#lastStatusWatermark = lastWatermark;
+		initialRequests.forEach((req) => this.#setRequiredBackfill("initial-request", req));
+		this.#checkAndStartBackfill();
+	}
+	#setLastStatusWatermark({ watermark }) {
+		if ((this.#lastStatusWatermark ?? "") < watermark) {
+			this.#lastStatusWatermark = watermark;
+			for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {
+				const awaiting = this.#awaitingStatusWatermarks[i];
+				if (watermark >= awaiting.watermark) {
+					awaiting.reached();
+					this.#awaitingStatusWatermarks.splice(i, 1);
+				}
+			}
+		}
+	}
+	#changeStreamReached(lc, watermark) {
+		if ((this.#lastStatusWatermark ?? "") < watermark) {
+			const { promise, resolve: reached } = resolver();
+			this.#awaitingStatusWatermarks.push({
+				watermark,
+				reached
+			});
+			lc.info?.(`waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`);
+			return promise;
+		}
+		return null;
+	}
+	#minBackoffMs;
+	#maxBackoffMs;
+	#retryDelayMs;
+	#backfillRetryTimer;
+	#checkAndStartBackfill() {
+		if (!this.#backfillRetryTimer && !this.#runningBackfill && this.#requiredBackfills.size) {
+			const candidates = [...this.#requiredBackfills.values()];
+			const request = candidates[randInt(0, candidates.length - 1)];
+			const state = {
+				request,
+				minWatermark: ""
+			};
+			const lc = this.#lc.withContext("table", request.table.name);
+			this.#runningBackfill = state;
+			this.#runBackfill(lc, state).then(() => {
+				this.#stopRunningBackfill("backfill exited", state);
+				this.#retryDelayMs = this.#minBackoffMs;
+			}).catch((e) => {
+				this.#stopRunningBackfill(String(e), state);
+				this.#retryBackfillWithBackoff(e);
+			});
+		}
+	}
+	#retryBackfillWithBackoff(e) {
+		const log = this.#retryDelayMs === this.#maxBackoffMs ? "error" : "warn";
+		this.#lc[log]?.(`Error running backfill. Retrying in ${this.#retryDelayMs} ms`, e);
+		this.#backfillRetryTimer = setTimeout(() => {
+			this.#backfillRetryTimer = void 0;
+			this.#checkAndStartBackfill();
+		}, this.#retryDelayMs);
+		this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);
+	}
+	async #runBackfill(lc, state) {
+		const changeStream = this.#changeStreamer;
+		let backfillTx = null;
+		/**
+		* @returns the new tx watermark, or null if backfill was cancelled
+		*/
+		const beginTxFor = async (msg) => {
+			assert(backfillTx === null, "Expected no active backfill transaction");
+			const lastWatermark = await changeStream.reserve("backfill");
+			if (state.canceledReason || msg.tag === "backfill" && msg.watermark < state.minWatermark) {
+				if (state.canceledReason === void 0) {
+					assert(msg.tag === "backfill", "Expected backfill message tag");
+					this.#stopRunningBackfill(`row key change at ${state.minWatermark} postdates backfill watermark at ${msg.watermark}`, state);
+				}
+				changeStream.release(lastWatermark);
+				return null;
+			}
+			const { major, minor = 0n } = stateVersionFromString(lastWatermark);
+			const tx = stateVersionToString({
+				major,
+				minor: BigInt(minor) + 1n
+			});
+			changeStream.push([
+				"begin",
+				{
+					tag: "begin",
+					json: this.#jsonFormat,
+					skipAck: true
+				},
+				{ commitWatermark: tx }
+			]);
+			return backfillTx = tx;
+		};
+		const commitTx = () => {
+			if (backfillTx) {
+				changeStream.push([
+					"commit",
+					{ tag: "commit" },
+					{ watermark: backfillTx }
+				]);
+				changeStream.release(backfillTx);
+			}
+			backfillTx = null;
+		};
+		for await (const msg of this.#backfillStreamer(state.request)) {
+			const mustWaitBeforeFlush = msg.tag === "backfill-completed" && this.#changeStreamReached(lc, msg.watermark);
+			if (backfillTx && (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush)) commitTx();
+			mustWaitBeforeFlush && await mustWaitBeforeFlush;
+			if (msg.tag === "backfill" && msg.rowValues.length > 0 && msg.relation.rowKey.columns.length === 0) throw new MissingRowKeyError(state.request);
+			if ((backfillTx ??= await beginTxFor(msg)) === null) {
+				lc.info?.(`backfill stream canceled: ${state.canceledReason}`, state.request);
+				this.#checkAndStartBackfill();
+				return;
+			}
+			await changeStream.push(["data", msg]);
+		}
+		backfillTx && commitTx();
+		lc.debug?.(`backfill stream exited`, state.canceledReason ?? "");
+	}
+	#backfillRunningFor(table) {
+		const state = this.#runningBackfill;
+		return state?.request.table.schema === table.schema && state.request.table.name === table.name ? state : null;
+	}
+	/**
+	* Stops the running backfill for the specified `reason`. If `instance` is
+	* specified, the running backfill is stopped only if it is that instance.
+	* This allows the running backfill itself to clear backfill state without
+	* accidentally stopping a different (e.g. subsequent) backfill.
+	*/
+	#stopRunningBackfill(reason, instance) {
+		const backfill = this.#runningBackfill;
+		if (backfill && backfill === (instance ?? backfill)) {
+			backfill.canceledReason = reason;
+			this.#runningBackfill = null;
+			reason && this.#lc.info?.(`canceling backfill:`, reason);
+		}
+	}
+	#setRequiredBackfill(source, req) {
+		const action = this.#requiredBackfills.has(req.table) ? "updated" : "added";
+		this.#lc.info?.(`Backfill ${action}: ${source}`, { backfill: req });
+		this.#requiredBackfills.set(req.table, req);
+	}
+	#deleteRequiredBackfill(source, id) {
+		const req = this.#requiredBackfills.get(id);
+		if (req) {
+			const action = source === "backfill-completed" ? "completed" : "dropped";
+			this.#lc.info?.(`Backfill ${action}: ${source}`, { backfill: req });
+			this.#requiredBackfills.delete(id);
+		}
+	}
+	/**
+	* Implements {@link Listener.onChange()}, invoked by the
+	* {@link ChangeStreamMultiplexer}.
+	*/
+	onChange(message) {
+		if (message[0] === "begin") {
+			this.#currentTxWatermark = message[2].commitWatermark;
+			return;
+		}
+		if (message[0] === "commit") {
+			this.#currentTxWatermark = null;
+			this.#setLastStatusWatermark(message[2]);
+			this.#checkAndStartBackfill();
+			return;
+		}
+		if (message[0] === "status") {
+			this.#setLastStatusWatermark(message[2]);
+			return;
+		}
+		if (message[0] !== "data") return;
+		const change = message[1];
+		const { tag } = change;
+		switch (tag) {
+			case "update-table-metadata": {
+				const { table, new: metadata } = change;
+				const backfillRequest = this.#requiredBackfills.get(table);
+				if (backfillRequest) {
+					this.#setRequiredBackfill(tag, {
+						...backfillRequest,
+						table: {
+							...backfillRequest.table,
+							metadata
+						}
+					});
+					if (this.#backfillRunningFor(table)) this.#stopRunningBackfill(`TableMetadata updated`);
+				}
+				break;
+			}
+			case "create-table": {
+				const { spec: { schema, name }, metadata = null, backfill } = change;
+				if (backfill) this.#setRequiredBackfill(tag, {
+					table: {
+						schema,
+						name,
+						metadata
+					},
+					columns: backfill
+				});
+				break;
+			}
+			case "rename-table": {
+				const { old, new: newTable } = change;
+				const backfillRequest = this.#requiredBackfills.get(old);
+				if (backfillRequest) {
+					const { schema, name } = newTable;
+					this.#deleteRequiredBackfill(tag, old);
+					this.#setRequiredBackfill(tag, {
+						...backfillRequest,
+						table: {
+							...backfillRequest.table,
+							schema,
+							name
+						}
+					});
+					if (this.#backfillRunningFor(old)) this.#stopRunningBackfill(`table renamed`);
+				}
+				break;
+			}
+			case "drop-table": {
+				const { id } = change;
+				if (this.#requiredBackfills.get(id)) {
+					this.#deleteRequiredBackfill(tag, id);
+					if (this.#backfillRunningFor(id)) this.#stopRunningBackfill(`table dropped`);
+				}
+				break;
+			}
+			case "add-column": {
+				const { table, tableMetadata: metadata = null, column, backfill } = change;
+				if (backfill) {
+					const backfillRequest = this.#requiredBackfills.get(table);
+					if (!backfillRequest) this.#setRequiredBackfill(tag, {
+						table: {
+							...table,
+							metadata
+						},
+						columns: { [column.name]: backfill }
+					});
+					else this.#setRequiredBackfill(tag, {
+						...backfillRequest,
+						table: {
+							...backfillRequest.table,
+							metadata
+						},
+						columns: {
+							...backfillRequest.columns,
+							[column.name]: backfill
+						}
+					});
+				}
+				break;
+			}
+			case "update-column": {
+				const { table, old: { name: oldName }, new: { name: newName } } = change;
+				if (oldName !== newName) {
+					const backfillRequest = this.#requiredBackfills.get(table);
+					if (backfillRequest && oldName in backfillRequest.columns) {
+						const { [oldName]: colSpec, ...otherCols } = backfillRequest.columns;
+						this.#setRequiredBackfill(tag, {
+							...backfillRequest,
+							columns: {
+								...otherCols,
+								[newName]: colSpec
+							}
+						});
+						const backfill = this.#backfillRunningFor(table);
+						if (backfill && oldName in backfill.request.columns) this.#stopRunningBackfill(`column renamed`);
+					}
+				}
+				break;
+			}
+			case "drop-column": {
+				const { table, column } = change;
+				const backfillRequest = this.#requiredBackfills.get(table);
+				if (backfillRequest && column in backfillRequest.columns) {
+					const { [column]: _excluded, ...remaining } = backfillRequest.columns;
+					if (Object.keys(remaining).length === 0) this.#deleteRequiredBackfill(tag, table);
+					else this.#setRequiredBackfill(tag, {
+						...backfillRequest,
+						columns: remaining
+					});
+					const backfill = this.#backfillRunningFor(table);
+					if (backfill && column in backfill.request.columns) this.#stopRunningBackfill(`column dropped`);
+				}
+				break;
+			}
+			case "update": {
+				const { relation, key, new: row } = change;
+				const backfill = this.#backfillRunningFor(relation);
+				const txWatermark = must(this.#currentTxWatermark, `not in a tx`);
+				if (backfill?.request.table.metadata && key !== null) {
+					for (const col of Object.keys(backfill.request.table.metadata.rowKey)) if (key[col] !== row[col]) {
+						backfill.minWatermark = txWatermark;
+						this.#lc.info?.(`key for row as changed (col: ${col}). backfill data must not predate ${backfill.minWatermark}`);
+						break;
+					}
+				}
+				break;
+			}
+			case "backfill-completed": {
+				const { relation, columns } = change;
+				const backfillRequest = this.#requiredBackfills.get(relation);
+				assert(backfillRequest, () => `No BackfillRequest completed backfill ${stringify(change)}`);
+				const remaining = Object.entries(backfillRequest.columns).filter(([col]) => !(columns.includes(col) || relation.rowKey.columns.includes(col)));
+				if (remaining.length === 0) this.#deleteRequiredBackfill(tag, relation);
+				else this.#setRequiredBackfill(tag, {
+					...backfillRequest,
+					columns: Object.fromEntries(remaining)
+				});
+				this.#stopRunningBackfill();
+				break;
+			}
+		}
+	}
+	cancel() {
+		this.#stopRunningBackfill(`change stream canceled`);
+		clearTimeout(this.#backfillRetryTimer);
+	}
+};
+var BackfillStreamError = class extends Error {
+	constructor(bf, msg, cause) {
+		super(`Cannot backfill ${bf.table.schema}.${bf.table.name}[${Object.keys(bf.columns).join(",")}]: ${msg}`, { cause });
+	}
+};
+/**
+* Background: The zero-cache supports replication of tables without a
+* PRIMARY KEY to facilitate the onboarding process. These rows can be
+* INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs
+* on such tables because the rows cannot be identified by a key. Supporting
+* this mode of replication allows the user to "fix" the setup by adding the
+* primary key, after which the table can be published downstream without
+* requiring a resync of the data.
+*
+* In terms of backfill, however, non-empty tables without a row key **cannot**
+* be backfilled, because backfill retries would result in writing duplicating
+* rows. (Empty tables, on the other hand, are fine because there is no data
+* to be deduped.)
+*
+* The MissingRowKeyError is used to signal that the table cannot be backfilled
+* in its current state. For simplicity, it is handled like runtime errors and
+* retried with backoff, with which it can eventually succeed if (1) a primary
+* key is added or (2) the table is emptied, e.g. via a TRUNCATE.
+*/
+var MissingRowKeyError = class extends BackfillStreamError {
+	name = "MissingRowKeyError";
+	constructor(bf, cause) {
+		super(bf, `"${bf.table.name}" is missing a PRIMARY KEY`, cause);
+	}
+};
+/**
+* Error type for backfill stream implementations to throw indicating that
+* the backfill request failed due to a schema incompatibility error. This
+* type of error does not need exponential backoff, as the retry happens
+* naturally once the invalidating schema change is processed and committed.
+*/
+var SchemaIncompatibilityError = class extends BackfillStreamError {
+	name = "SchemaIncompatibilityError";
+	constructor(bf, msg, cause) {
+		super(bf, msg, cause);
+	}
 };
-//# sourceMappingURL=backfill-manager.js.map
+//#endregion
+export { BackfillManager, SchemaIncompatibilityError };
+
+//# sourceMappingURL=backfill-manager.js.map