@rocicorp/zero 0.25.0 → 0.26.0-canary.0

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -1,368 +1,261 @@
-import { LogContext } from '@rocicorp/logger';
-import { resolver } from '@rocicorp/resolver';
+import { resolver } from "@rocicorp/resolver";
 import { unreachable } from "../../../../shared/src/asserts.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
-import { min, } from "../../types/lexi-version.js";
+import { min } from "../../types/lexi-version.js";
 import { Subscription } from "../../types/subscription.js";
-import {} from "../change-source/protocol/current/downstream.js";
-import { DEFAULT_MAX_RETRY_DELAY_MS, RunningState, UnrecoverableError, } from "../running-state.js";
-import {} from "./change-streamer.js";
-import * as ErrorType from "./error-type-enum.js";
+import "../change-source/protocol/current/downstream.js";
+import { publishReplicationError } from "../replicator/replication-status.js";
+import { RunningState, UnrecoverableError, DEFAULT_MAX_RETRY_DELAY_MS } from "../running-state.js";
+import "./change-streamer.js";
+import { WrongReplicaVersion } from "./error-type-enum.js";
 import { Forwarder } from "./forwarder.js";
 import { initChangeStreamerSchema } from "./schema/init.js";
-import { AutoResetSignal, ensureReplicationConfig, markResetRequired, } from "./schema/tables.js";
+import { ensureReplicationConfig, markResetRequired, AutoResetSignal } from "./schema/tables.js";
 import { Storer } from "./storer.js";
 import { Subscriber } from "./subscriber.js";
-/**
- * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.
- */
-export async function initializeStreamer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, changeSource, subscriptionState, autoReset, setTimeoutFn = setTimeout) {
-    // Make sure the ChangeLog DB is set up.
-    await initChangeStreamerSchema(lc, changeDB, shard);
-    await ensureReplicationConfig(lc, changeDB, subscriptionState, shard, autoReset);
-    const { replicaVersion } = subscriptionState;
-    return new ChangeStreamerImpl(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, changeSource, autoReset, setTimeoutFn);
+async function initializeStreamer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, changeSource, subscriptionState, autoReset, setTimeoutFn = setTimeout) {
+  await initChangeStreamerSchema(lc, changeDB, shard);
+  await ensureReplicationConfig(
+    lc,
+    changeDB,
+    subscriptionState,
+    shard,
+    autoReset
+  );
+  const { replicaVersion } = subscriptionState;
+  return new ChangeStreamerImpl(
+    lc,
+    shard,
+    taskID,
+    discoveryAddress,
+    discoveryProtocol,
+    changeDB,
+    replicaVersion,
+    changeSource,
+    autoReset,
+    setTimeoutFn
+  );
 }
-/**
- * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a
- * {@link Forwarder} and {@link Storer} to execute the forward-store-ack
- * procedure described in {@link ChangeStreamer}.
- *
- * ### Subscriber Catchup
- *
- * Connecting clients first need to be "caught up" to the current watermark
- * (from stored change log entries) before new entries are forwarded to
- * them. This is non-trivial because the replication stream may be in the
- * middle of a pending streamed Transaction for which some entries have
- * already been forwarded but are not yet committed to the store.
- *
- *
- * ```
- * ------------------------------- - - - - - - - - - - - - - - - - - - -
- * | Historic changes in storage |  Pending (streamed) tx  |   Next tx
- * ------------------------------- - - - - - - - - - - - - - - - - - - -
- *                                           Replication stream
- *                                           >  >  >  >  >  >  >  >  >
- *           ^  ---> required catchup --->   ^
- * Subscriber watermark               Subscription begins
- * ```
- *
- * Preemptively buffering the changes of every pending transaction
- * would be wasteful and consume too much memory for large transactions.
- *
- * Instead, the streamer synchronously dispatches changes and subscriptions
- * to the {@link Forwarder} and the {@link Storer} such that the two
- * components are aligned as to where in the stream the subscription started.
- * The two components then coordinate catchup and handoff via the
- * {@link Subscriber} object with the following algorithm:
- *
- * * If the streamer is in the middle of a pending Transaction, the
- *   Subscriber is "queued" on both the Forwarder and the Storer. In this
- *   state, new changes are *not* forwarded to the Subscriber, and catchup
- *   is not yet executed.
- * * Once the commit message for the pending Transaction is processed
- *   by the Storer, it begins catchup on the Subscriber (with a READONLY
- *   snapshot so that it does not block subsequent storage operations).
- *   This catchup is thus guaranteed to load the change log entries of
- *   that last Transaction.
- * * When the Forwarder processes that same commit message, it moves the
- *   Subscriber from the "queued" to the "active" set of clients such that
- *   the Subscriber begins receiving new changes, starting from the next
- *   Transaction.
- * * The Subscriber does not forward those changes, however, if its catchup
- *   is not complete. Until then, it buffers the changes in memory.
- * * Once catchup is complete, the buffered changes are immediately sent
- *   and the Subscriber henceforth forwards changes as they are received.
- *
- * In the (common) case where the streamer is not in the middle of a pending
- * transaction when a subscription begins, the Storer begins catchup
- * immediately and the Forwarder directly adds the Subscriber to its active
- * set. However, the Subscriber still buffers any forwarded messages until
- * its catchup is complete.
- *
- * ### Watermarks and ordering
- *
- * The ChangeStreamerService depends on its {@link ChangeSource} to send
- * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences
- * in commit order. This follows Postgres's Logical Replication Protocol
- * Message Flow:
- *
- * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW
- *
- * > The logical replication protocol sends individual transactions one by one.
- * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.
- *
- * In order to correctly replay (new) and filter (old) messages to subscribers
- * at different points in the replication stream, these changes must be assigned
- * watermarks such that they preserve the order in which they were received
- * from the ChangeSource.
- *
- * A previous implementation incorrectly derived these watermarks from the Postgres
- * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,
- * non-conflicting transactions can overlap, which can result in a `begin` message
- * with an earlier LSN arriving after a `commit` message. For example, the
- * changes for these transactions:
- *
- * ```
- * LSN:   1     2     3  4    5   6   7     8   9      10
- * tx1: begin  data     data     data     commit
- * tx2:             begin    data    data      data  commit
- * ```
- *
- * will arrive as:
- *
- * ```
- * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10
- * ```
- *
- * Thus, LSN of non-commit messages are not suitable for tracking the sorting
- * order of the replication stream.
- *
- * Instead, the ChangeStreamer uses the following algorithm for deterministic
- * catchup and filtering of changes:
- *
- * * A `commit` message is assigned to a watermark corresponding to its LSN.
- *   These are guaranteed to be in commit order by definition.
- *
- * * `begin` and `data` messages are assigned to the watermark of the
- *   preceding `commit` (the previous transaction, or the replication
- *   slot's starting LSN) plus 1. This guarantees that they will be sorted
- *   after the previously commit transaction even if their LSNs came before it.
- *   This is referred to as the `preCommitWatermark`.
- *
- * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is
- *   the position of the message within its transaction, with the `begin` message
- *   starting at `0`. This guarantees that `begin` and `data` messages will be
- *   fetched in the original ChangeSource order during catchup.
- *
- * `begin` and `data` messages share the same watermark, but this is sufficient for
- * Subscriber filtering because subscribers only know about the `commit` watermarks
- * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares
- * the internal watermarks of the incoming messages against the commit watermark of
- * the caller, updating the watermark at every `Commit` message that is forwarded.
- *
- * ### Cleanup
- *
- * As mentioned in the {@link ChangeStreamer} documentation: "the ChangeStreamer
- * uses a combination of [the "initial", i.e. backup-derived watermark and] ACK
- * responses from connected subscribers to determine the watermark up
- * to which it is safe to purge old change log entries."
- *
- * More concretely:
- *
- * * The `initial`, backup-derived watermark is the earliest to which cleanup
- *   should ever happen.
- *
- * * However, it is possible for the replica backup to be *ahead* of a connected
- *   subscriber; and if a network error causes that subscriber to retry from its
- *   last watermark, the change streamer must support it.
- *
- * Thus, before cleaning up to an `initial` backup-derived watermark, the change
- * streamer first confirms that all connected subscribers have also passed
- * that watermark.
- */
 class ChangeStreamerImpl {
-    id;
-    #lc;
-    #shard;
-    #changeDB;
-    #replicaVersion;
-    #source;
-    #storer;
-    #forwarder;
-    #autoReset;
-    #state;
-    #initialWatermarks = new Set();
-    // Starting the (Postgres) ChangeStream results in killing the previous
-    // Postgres subscriber, potentially creating a gap in which the old
-    // change-streamer has shut down and the new change-streamer has not yet
-    // been recognized as "healthy" (and thus does not get any requests).
-    //
-    // To minimize this gap, delay starting the ChangeStream until the first
-    // request from a `serving` replicator, indicating that higher level
-    // load-balancing / routing logic has begun routing requests to this task.
-    #serving = resolver();
-    #txCounter = getOrCreateCounter('replication', 'transactions', 'Count of replicated transactions');
-    #stream;
-    constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, source, autoReset, setTimeoutFn = setTimeout) {
-        this.id = `change-streamer`;
-        this.#lc = lc.withContext('component', 'change-streamer');
-        this.#shard = shard;
-        this.#changeDB = changeDB;
-        this.#replicaVersion = replicaVersion;
-        this.#source = source;
-        this.#storer = new Storer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]), err => this.stop(err));
-        this.#forwarder = new Forwarder();
-        this.#autoReset = autoReset;
-        this.#state = new RunningState(this.id, undefined, setTimeoutFn);
-    }
-    async run() {
-        this.#lc.info?.('starting change stream');
-        // Once this change-streamer acquires "ownership" of the change DB,
-        // it is safe to start the storer.
-        await this.#storer.assumeOwnership();
-        // The storer will, in turn, detect changes to ownership and stop
-        // the change-streamer appropriately.
-        this.#storer.run().catch(e => this.stop(e));
-        while (this.#state.shouldRun()) {
-            let err;
-            let watermark = null;
-            try {
-                const startAfter = await this.#storer.getLastWatermarkToStartStream();
-                const stream = await this.#source.startStream(startAfter);
-                this.#stream = stream;
-                this.#state.resetBackoff();
-                watermark = null;
-                for await (const change of stream.changes) {
-                    const [type, msg] = change;
-                    switch (type) {
-                        case 'status':
-                            this.#storer.status(change); // storer acks once it gets through its queue
-                            continue;
-                        case 'control':
-                            await this.#handleControlMessage(msg);
-                            continue; // control messages are not stored/forwarded
-                        case 'begin':
-                            watermark = change[2].commitWatermark;
-                            break;
-                        case 'commit':
-                            if (watermark !== change[2].watermark) {
-                                throw new UnrecoverableError(`commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`);
-                            }
-                            this.#txCounter.add(1);
-                            break;
-                        default:
-                            if (watermark === null) {
-                                throw new UnrecoverableError(`${type} change (${msg.tag}) received before 'begin' message`);
-                            }
-                            break;
-                    }
-                    this.#storer.store([watermark, change]);
-                    this.#forwarder.forward([watermark, change]);
-                    if (type === 'commit') {
-                        watermark = null;
-                    }
-                    // Allow the storer to exert back pressure.
-                    const readyForMore = this.#storer.readyForMore();
-                    if (readyForMore) {
-                        await readyForMore;
-                    }
-                }
-            }
-            catch (e) {
-                err = e;
-            }
-            finally {
-                this.#stream?.changes.cancel();
-                this.#stream = undefined;
-            }
-            // When the change stream is interrupted, abort any pending transaction.
-            if (watermark) {
-                this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);
-                this.#storer.abort();
-                this.#forwarder.forward([watermark, ['rollback', { tag: 'rollback' }]]);
-            }
-            await this.#state.backoff(this.#lc, err);
-        }
-        this.#lc.info?.('ChangeStreamer stopped');
-    }
-    async #handleControlMessage(msg) {
-        this.#lc.info?.('received control message', msg);
-        const { tag } = msg;
-        switch (tag) {
-            case 'reset-required':
-                await markResetRequired(this.#changeDB, this.#shard);
-                if (this.#autoReset) {
-                    this.#lc.warn?.('shutting down for auto-reset');
-                    await this.stop(new AutoResetSignal());
-                }
-                break;
+  id;
+  #lc;
+  #shard;
+  #changeDB;
+  #replicaVersion;
+  #source;
+  #storer;
+  #forwarder;
+  #autoReset;
+  #state;
+  #initialWatermarks = /* @__PURE__ */ new Set();
+  // Starting the (Postgres) ChangeStream results in killing the previous
+  // Postgres subscriber, potentially creating a gap in which the old
+  // change-streamer has shut down and the new change-streamer has not yet
+  // been recognized as "healthy" (and thus does not get any requests).
+  //
+  // To minimize this gap, delay starting the ChangeStream until the first
+  // request from a `serving` replicator, indicating that higher level
+  // load-balancing / routing logic has begun routing requests to this task.
+  #serving = resolver();
+  #txCounter = getOrCreateCounter(
+    "replication",
+    "transactions",
+    "Count of replicated transactions"
+  );
+  #stream;
+  constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, source, autoReset, setTimeoutFn = setTimeout) {
+    this.id = `change-streamer`;
+    this.#lc = lc.withContext("component", "change-streamer");
+    this.#shard = shard;
+    this.#changeDB = changeDB;
+    this.#replicaVersion = replicaVersion;
+    this.#source = source;
+    this.#storer = new Storer(
+      lc,
+      shard,
+      taskID,
+      discoveryAddress,
+      discoveryProtocol,
+      changeDB,
+      replicaVersion,
+      (consumed) => this.#stream?.acks.push(["status", consumed[1], consumed[2]]),
+      (err) => this.stop(err)
+    );
+    this.#forwarder = new Forwarder();
+    this.#autoReset = autoReset;
+    this.#state = new RunningState(this.id, void 0, setTimeoutFn);
+  }
+  async run() {
+    this.#lc.info?.("starting change stream");
+    await this.#storer.assumeOwnership();
+    this.#storer.run().then(() => this.stop()).catch((e) => this.stop(e));
+    while (this.#state.shouldRun()) {
+      let err;
+      let watermark = null;
+      try {
+        const startAfter = await this.#storer.getLastWatermarkToStartStream();
+        const stream = await this.#source.startStream(startAfter);
+        this.#stream = stream;
+        this.#state.resetBackoff();
+        watermark = null;
+        for await (const change of stream.changes) {
+          const [type, msg] = change;
+          switch (type) {
+            case "status":
+              this.#storer.status(change);
+              continue;
+            case "control":
+              await this.#handleControlMessage(msg);
+              continue;
+            // control messages are not stored/forwarded
+            case "begin":
+              watermark = change[2].commitWatermark;
+              break;
+            case "commit":
+              if (watermark !== change[2].watermark) {
+                throw new UnrecoverableError(
+                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`
+                );
+              }
+              this.#txCounter.add(1);
+              break;
             default:
-                unreachable(tag);
+              if (watermark === null) {
+                throw new UnrecoverableError(
+                  `${type} change (${msg.tag}) received before 'begin' message`
+                );
+              }
+              break;
+          }
+          this.#storer.store([watermark, change]);
+          this.#forwarder.forward([watermark, change]);
+          if (type === "commit") {
+            watermark = null;
+          }
+          const readyForMore = this.#storer.readyForMore();
+          if (readyForMore) {
+            await readyForMore;
+          }
         }
+      } catch (e) {
+        err = e;
+      } finally {
+        this.#stream?.changes.cancel();
+        this.#stream = void 0;
+      }
+      if (watermark) {
+        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);
+        this.#storer.abort();
+        this.#forwarder.forward([watermark, ["rollback", { tag: "rollback" }]]);
+      }
+      await this.#state.backoff(this.#lc, err);
     }
-    subscribe(ctx) {
-        const { protocolVersion, id, mode, replicaVersion, watermark, initial } = ctx;
-        if (mode === 'serving') {
-            this.#serving.resolve();
+    this.#lc.info?.("ChangeStreamer stopped");
+  }
+  async #handleControlMessage(msg) {
+    this.#lc.info?.("received control message", msg);
+    const { tag } = msg;
+    switch (tag) {
+      case "reset-required":
+        await markResetRequired(this.#changeDB, this.#shard);
+        await publishReplicationError(
+          this.#lc,
+          "Replicating",
+          msg.message ?? "Resync required",
+          msg.errorDetails
+        );
+        if (this.#autoReset) {
+          this.#lc.warn?.("shutting down for auto-reset");
+          await this.stop(new AutoResetSignal());
         }
-        const downstream = Subscription.create({
-            cleanup: () => this.#forwarder.remove(subscriber),
-        });
-        const subscriber = new Subscriber(protocolVersion, id, watermark, downstream);
-        if (replicaVersion !== this.#replicaVersion) {
-            this.#lc.warn?.(`rejecting subscriber at replica version ${replicaVersion}`);
-            subscriber.close(ErrorType.WrongReplicaVersion, `current replica version is ${this.#replicaVersion} (requested ${replicaVersion})`);
-        }
-        else {
-            this.#lc.debug?.(`adding subscriber ${subscriber.id}`);
-            this.#forwarder.add(subscriber);
-            this.#storer.catchup(subscriber, mode);
-            if (initial) {
-                this.scheduleCleanup(watermark);
-            }
-        }
-        return Promise.resolve(downstream);
+        break;
+      default:
+        unreachable();
     }
-    scheduleCleanup(watermark) {
-        const origSize = this.#initialWatermarks.size;
-        this.#initialWatermarks.add(watermark);
-        if (origSize === 0) {
-            this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);
-        }
+  }
+  subscribe(ctx) {
+    const { protocolVersion, id, mode, replicaVersion, watermark, initial } = ctx;
+    if (mode === "serving") {
+      this.#serving.resolve();
     }
-    async #purgeOldChanges() {
-        const initial = [...this.#initialWatermarks];
-        if (initial.length === 0) {
-            this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.
-            return;
-        }
-        const current = [...this.#forwarder.getAcks()];
-        if (current.length === 0) {
-            // Also not expected, but possible (e.g. subscriber connects, then disconnects).
-            // Bail to be safe.
-            this.#lc.warn?.('No subscribers to confirm cleanup');
-            return;
-        }
-        try {
-            const earliestInitial = min(...initial);
-            const earliestCurrent = min(...current);
-            if (earliestCurrent < earliestInitial) {
-                this.#lc.info?.(`At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`);
-            }
-            else {
-                const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);
-                this.#lc.info?.(`Purged ${deleted} changes before ${earliestInitial}`);
-                this.#initialWatermarks.delete(earliestInitial);
-            }
-        }
-        finally {
-            if (this.#initialWatermarks.size) {
-                // If there are unpurged watermarks to check, schedule the next purge.
-                this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);
-            }
-        }
+    const downstream = Subscription.create({
+      cleanup: () => this.#forwarder.remove(subscriber)
+    });
+    const subscriber = new Subscriber(
+      protocolVersion,
+      id,
+      watermark,
+      downstream
+    );
+    if (replicaVersion !== this.#replicaVersion) {
+      this.#lc.warn?.(
+        `rejecting subscriber at replica version ${replicaVersion}`
+      );
+      subscriber.close(
+        WrongReplicaVersion,
+        `current replica version is ${this.#replicaVersion} (requested ${replicaVersion})`
+      );
+    } else {
+      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);
+      this.#forwarder.add(subscriber);
+      this.#storer.catchup(subscriber, mode);
+      if (initial) {
+        this.scheduleCleanup(watermark);
+      }
     }
-    async stop(err) {
-        this.#state.stop(this.#lc, err);
-        this.#stream?.changes.cancel();
-        await this.#storer.stop();
+    return Promise.resolve(downstream);
+  }
+  scheduleCleanup(watermark) {
+    const origSize = this.#initialWatermarks.size;
+    this.#initialWatermarks.add(watermark);
+    if (origSize === 0) {
+      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);
+    }
+  }
+  async getChangeLogState() {
+    const minWatermark = await this.#storer.getMinWatermarkForCatchup();
+    return {
+      replicaVersion: this.#replicaVersion,
+      minWatermark: minWatermark ?? this.#replicaVersion
+    };
+  }
+  async #purgeOldChanges() {
+    const initial = [...this.#initialWatermarks];
+    if (initial.length === 0) {
+      this.#lc.warn?.("No initial watermarks to check for cleanup");
+      return;
+    }
+    const current = [...this.#forwarder.getAcks()];
+    if (current.length === 0) {
+      this.#lc.warn?.("No subscribers to confirm cleanup");
+      return;
+    }
+    try {
+      const earliestInitial = min(...initial);
+      const earliestCurrent = min(...current);
+      if (earliestCurrent < earliestInitial) {
+        this.#lc.info?.(
+          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`
+        );
+      } else {
+        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);
+        this.#lc.info?.(`Purged ${deleted} changes before ${earliestInitial}`);
+        this.#initialWatermarks.delete(earliestInitial);
+      }
+    } finally {
+      if (this.#initialWatermarks.size) {
+        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);
+      }
     }
+  }
+  async stop(err) {
+    this.#state.stop(this.#lc, err);
+    this.#stream?.changes.cancel();
+    await this.#storer.stop();
+  }
 }
-// The delay between receiving an initial, backup-based watermark
-// and performing a check of whether to purge records before it.
-// This delay should be long enough to handle situations like the following:
-//
-// 1. `litestream restore` downloads a backup for the `replication-manager`
-// 2. `replication-manager` starts up and runs this `change-streamer`
-// 3. `zero-cache`s that are running on a different replica connect to this
-//    `change-streamer` after exponential backoff retries.
-//
-// It is possible for a `zero-cache`[3] to be behind the backup restored [1].
-// This cleanup delay (30 seconds) is thus set to be a value comfortably
-// longer than the max delay for exponential backoff (10 seconds) in
-// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect
-// so that the `change-streamer` can track its progress and know when it has
-// surpassed the initial watermark of the backup [1].
 const CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;
-//# sourceMappingURL=change-streamer-service.js.map
+export {
+  initializeStreamer
+};
+//# sourceMappingURL=change-streamer-service.js.map