@rocicorp/zero 0.26.0-canary.14 → 0.26.0-canary.18

package/out/zero/package.json.js

@@ -1,4 +1,4 @@
-const version = "0.26.0-canary.14";
+const version = "0.26.0-canary.18";
 const packageJson = {
   version
 };

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -91,7 +91,6 @@ class ChangeStreamerImpl {
   async run() {
     this.#lc.info?.("starting change stream");
     await this.#storer.assumeOwnership();
-    this.#storer.run().then(() => this.stop()).catch((e) => this.stop(e));
     const flushBytesThreshold = getDefaultHighWaterMark(false);
     while (this.#state.shouldRun()) {
       let err;
@@ -103,6 +102,7 @@ class ChangeStreamerImpl {
           lastWatermark,
           backfillRequests
         );
+        this.#storer.run().catch((e) => stream.changes.cancel(e));
         this.#stream = stream;
         this.#state.resetBackoff();
         watermark = null;
@@ -165,7 +165,10 @@ class ChangeStreamerImpl {
           ["rollback", { tag: "rollback" }]
         ]);
       }
-      await this.#state.backoff(this.#lc, err);
+      await Promise.all([
+        this.#storer.stop(),
+        this.#state.backoff(this.#lc, err)
+      ]);
     }
     this.#lc.info?.("ChangeStreamer stopped");
   }

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.js","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getDefaultHighWaterMark} from 'node:stream';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport {publishReplicationError} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  subscriptionState: SubscriptionState,\n  autoReset: boolean,\n  backPressureLimitHeapProportion: number,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    autoReset,\n    backPressureLimitHeapProportion,\n    setTimeoutFn,\n  );\n}\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n */\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    autoReset: boolean,\n    backPressureLimitHeapProportion: number,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      backPressureLimitHeapProportion,\n    );\n    this.#forwarder = new Forwarder();\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership();\n    // The storer will, in turn, detect changes to ownership and stop\n    // the change-streamer appropriately.\n    this.#storer\n      .run()\n      .then(() => this.stop())\n      .catch(e => this.stop(e));\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#stream = stream;\n        this.#state.resetBackoff();\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          unflushedBytes += this.#storer.store([watermark, change]);\n          const sent = this.#forwarder.forward([watermark, change]);\n          if (unflushedBytes >= flushBytesThreshold) {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            await sent;\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        await this.#forwarder.forward([\n          watermark,\n          ['rollback', {tag: 'rollback'}],\n        ]);\n      }\n\n      await this.#state.backoff(this.#lc, err);\n    }\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n\n      if (initial) {\n        this.scheduleCleanup(watermark);\n      }\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n"],"names":["ErrorType.WrongReplicaVersion"],"mappings":";;;;;;;;;;;;;;;;AAgDA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,mBACA,WACA,iCACA,eAAe,YACiB;AAEhC,QAAM,yBAAyB,IAAI,UAAU,KAAK;AAClD,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,EAAC,mBAAkB;AACzB,SAAO,IAAI;AAAA,IACT;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;AAwJA,MAAM,mBAAoD;AAAA,EAC/C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,yCAAyB,IAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUzB,WAAW,SAAA;AAAA,EAEX,aAAa;AAAA,IACpB;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGF;AAAA,EAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,WACA,iCACA,eAAe,YACf;AACA,SAAK,KAAK;AACV,SAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;AACxD,SAAK,SAAS;AACd,SAAK,YAAY;AACjB,SAAK,kBAAkB;AACvB,SAAK,UAAU;AACf,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,CAAA,aAAY,KAAK,SAAS,KAAK,KAAK,CAAC,UAAU,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;AAAA,MACxE,CAAA,QAAO,KAAK,KAAK,GAAG;AAAA,MACpB;AAAA,IAAA;AAEF,SAAK,aAAa,IAAI,UAAA;AACtB,SAAK,aAAa;AAClB,SAAK,SAAS,IAAI,aAAa,KAAK,IAAI,QAAW,YAAY;AAAA,EACjE;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,IAAI,OAAO,wBAAwB;AAIxC,UAAM,KAAK,QAAQ,gBAAA;AAGnB,SAAK,QACF,IAAA,EACA,KAAK,MAAM,KAAK,KAAA,CAAM,EACtB,MAAM,CAAA,MAAK,KAAK,KAAK,CAAC,CAAC;AAI1B,UAAM,sBAAsB,wBAAwB,KAAK;AAEzD,WAAO,KAAK,OAAO,aAAa;AAC9B,UAAI;AACJ,UAAI,YAA2B;AAC/B,UAAI,iBAAiB;AACrB,UAAI;AACF,cAAM,EAAC,eAAe,iBAAA,IACpB,MAAM,KAAK,QAAQ,uCAAA;AACrB,cAAM,SAAS,MAAM,KAAK,QAAQ;AAAA,UAChC;AAAA,UACA;AAAA,QAAA;AAEF,aAAK,UAAU;AACf,aAAK,OAAO,aAAA;AACZ,oBAAY;AAEZ,yBAAiB,UAAU,OAAO,SAAS;AACzC,gBAAM,CAAC,MAAM,GAAG,IAAI;AACpB,kBAAQ,MAAA;AAAA,YACN,KAAK;AACH,kBAAI,IAAI,KAAK;AACX,qBAAK,QAAQ,OAAO,MAAM;AAAA,cAC5B;AACA;AAAA,YACF,KAAK;AACH,oBAAM,KAAK,sBAAsB,GAAG;AACpC;AAAA;AAAA,YACF,KAAK;AACH,0BAAY,OAAO,CAAC,EAAE;AACtB;AAAA,YACF,KAAK;AACH,kBAAI,cAAc,OAAO,CAAC,EAAE,WAAW;AACrC,sBAAM,IAAI;AAAA,kBACR,oBAAoB,OAAO,CAAC,EAAE,SAAS,qCAAqC,SAAS;AAAA,gBAAA;AAAA,cAEzF;AACA,mBAAK,WAAW,IAAI,CAAC;AACrB;AAAA,YACF;AACE,kBAAI,cAAc,MAAM;AACtB,sBAAM,IAAI;AAAA,kBACR,GAAG,IAAI,YAAY,IAAI,GAAG;AAAA,gBAAA;AAAA,cAE9B;AACA;AAAA,UAAA;AAGJ,4BAAkB,KAAK,QAAQ,MAAM,CAAC,WAAW,MAAM,CAAC;AACxD,gBAAM,OAAO,KAAK,WAAW,QAAQ,CAAC,WAAW,MAAM,CAAC;AACxD,cAAI,kBAAkB,qBAAqB;AAQzC,kBAAM;AACN,6BAAiB;AAAA,UACnB;AAEA,cAAI,SAAS,YAAY,SAAS,YAAY;AAC5C,wBAAY;AAAA,UACd;AAGA,gBAAM,eAAe,KAAK,QAAQ,aAAA;AAClC,cAAI,cAAc;AAChB,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR,UAAA;AACE,aAAK,SAAS,QAAQ,OAAA;AACtB,aAAK,UAAU;AAAA,MACjB;AAGA,UAAI,WAAW;AACb,aAAK,IAAI,OAAO,oCAAoC,SAAS,EAAE;AAC/D,aAAK,QAAQ,MAAA;AACb,cAAM,KAAK,WAAW,QAAQ;AAAA,UAC5B;AAAA,UACA,CAAC,YAAY,EAAC,KAAK,YAAW;AAAA,QAAA,CAC/B;AAAA,MACH;AAEA,YAAM,KAAK,OAAO,QAAQ,KAAK,KAAK,GAAG;AAAA,IACzC;AACA,SAAK,IAAI,OAAO,wBAAwB;AAAA,EAC1C;AAAA,EAEA,MAAM,sBAAsB,KAA6B;AACvD,SAAK,IAAI,OAAO,4BAA4B,GAAG;AAC/C,UAAM,EAAC,QAAO;AAEd,YAAQ,KAAA;AAAA,MACN,KAAK;AACH,cAAM,kBAAkB,KAAK,WAAW,KAAK,MAAM;AACnD,cAAM;AAAA,UACJ,KAAK;AAAA,UACL;AAAA,UACA,IAAI,WAAW;AAAA,UACf,IAAI;AAAA,QAAA;AAEN,YAAI,KAAK,YAAY;AACnB,eAAK,IAAI,OAAO,8BAA8B;AAC9C,gBAAM,KAAK,KAAK,IAAI,iBAAiB;AAAA,QACvC;AACA;AAAA,MACF;AACE,oBAAe;AAAA,IAAA;AAAA,EAErB;AAAA,EAEA,UAAU,KAAqD;AAC7D,UAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,QAAI,SAAS,WAAW;AACtB,WAAK,SAAS,QAAA;AAAA,IAChB;AACA,UAAM,aAAa,aAAa,OAAmB;AAAA,MACjD,SAAS,MAAM,KAAK,WAAW,OAAO,UAAU;AAAA,IAAA,CACjD;AACD,UAAM,aAAa,IAAI;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI,mBAAmB,KAAK,iBAAiB;AAC3C,WAAK,IAAI;AAAA,QACP,2CAA2C,cAAc;AAAA,MAAA;AAE3D,iBAAW;AAAA,QACTA;AAAAA,QACA,8BACE,KAAK,eACP,eAAe,cAAc;AAAA,MAAA;AAAA,IAEjC,OAAO;AACL,WAAK,IAAI,QAAQ,qBAAqB,WAAW,EAAE,EAAE;AAErD,WAAK,WAAW,IAAI,UAAU;AAC9B,WAAK,QAAQ,QAAQ,YAAY,IAAI;AAErC,UAAI,SAAS;AACX,aAAK,gBAAgB,SAAS;AAAA,MAChC;AAAA,IACF;AACA,WAAO,QAAQ,QAAQ,UAAU;AAAA,EACnC;AAAA,EAEA,gBAAgB,WAAmB;AACjC,UAAM,WAAW,KAAK,mBAAmB;AACzC,SAAK,mBAAmB,IAAI,SAAS;AAErC,QAAI,aAAa,GAAG;AAClB,WAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,IACxE;AAAA,EACF;AAAA,EAEA,MAAM,oBAGH;AACD,UAAM,eAAe,MAAM,KAAK,QAAQ,0BAAA;AACxC,QAAI,CAAC,cAAc;AACjB,WAAK,IAAI;AAAA,QACP;AAAA,MAAA;AAAA,IAEJ;AACA,WAAO;AAAA,MACL,gBAAgB,KAAK;AAAA,MACrB,cAAc,gBAAgB,KAAK;AAAA,IAAA;AAAA,EAEvC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,mBAAkC;AACtC,UAAM,UAAU,CAAC,GAAG,KAAK,kBAAkB;AAC3C,QAAI,QAAQ,WAAW,GAAG;AACxB,WAAK,IAAI,OAAO,4CAA4C;AAC5D;AAAA,IACF;AACA,UAAM,UAAU,CAAC,GAAG,KAAK,WAAW,SAAS;AAC7C,QAAI,QAAQ,WAAW,GAAG;AAGxB,WAAK,IAAI,OAAO,mCAAmC;AACnD;AAAA,IACF;AACA,QAAI;AACF,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,UAAI,kBAAkB,iBAAiB;AACrC,aAAK,IAAI;AAAA,UACP,yCAAyC,eAAe,MAAM,eAAe;AAAA,QAAA;AAAA,MAEjF,OAAO;AACL,aAAK,IAAI,OAAO,0BAA0B,eAAe,MAAM;AAC/D,cAAM,QAAQ,YAAY,IAAA;AAC1B,cAAM,UAAU,MAAM,KAAK,QAAQ,mBAAmB,eAAe;AACrE,cAAM,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACrD,aAAK,IAAI;AAAA,UACP,UAAU,OAAO,mBAAmB,eAAe,KAAK,OAAO;AAAA,QAAA;AAEjE,aAAK,mBAAmB,OAAO,eAAe;AAAA,MAChD;AAAA,IACF,SAAS,GAAG;AACV,WAAK,IAAI,OAAO,4BAA4B,CAAC;AAAA,IAC/C,UAAA;AACE,UAAI,KAAK,mBAAmB,MAAM;AAEhC,aAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,KAAK,KAAe;AACxB,SAAK,OAAO,KAAK,KAAK,KAAK,GAAG;AAC9B,SAAK,SAAS,QAAQ,OAAA;AACtB,UAAM,KAAK,QAAQ,KAAA;AAAA,EACrB;AACF;AAiBA,MAAM,mBAAmB,6BAA6B;"}
+{"version":3,"file":"change-streamer-service.js","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getDefaultHighWaterMark} from 'node:stream';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport {publishReplicationError} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  subscriptionState: SubscriptionState,\n  autoReset: boolean,\n  backPressureLimitHeapProportion: number,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    autoReset,\n    backPressureLimitHeapProportion,\n    setTimeoutFn,\n  );\n}\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n */\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    autoReset: boolean,\n    backPressureLimitHeapProportion: number,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      backPressureLimitHeapProportion,\n    );\n    this.#forwarder = new Forwarder();\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership();\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        this.#state.resetBackoff();\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          unflushedBytes += this.#storer.store([watermark, change]);\n          const sent = this.#forwarder.forward([watermark, change]);\n          if (unflushedBytes >= flushBytesThreshold) {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            await sent;\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        await this.#forwarder.forward([\n          watermark,\n          ['rollback', {tag: 'rollback'}],\n        ]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n      ]);\n    }\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n\n      if (initial) {\n        this.scheduleCleanup(watermark);\n      }\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n"],"names":["ErrorType.WrongReplicaVersion"],"mappings":";;;;;;;;;;;;;;;;AAgDA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,mBACA,WACA,iCACA,eAAe,YACiB;AAEhC,QAAM,yBAAyB,IAAI,UAAU,KAAK;AAClD,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,EAAC,mBAAkB;AACzB,SAAO,IAAI;AAAA,IACT;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;AAwJA,MAAM,mBAAoD;AAAA,EAC/C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,yCAAyB,IAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUzB,WAAW,SAAA;AAAA,EAEX,aAAa;AAAA,IACpB;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGF;AAAA,EAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,WACA,iCACA,eAAe,YACf;AACA,SAAK,KAAK;AACV,SAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;AACxD,SAAK,SAAS;AACd,SAAK,YAAY;AACjB,SAAK,kBAAkB;AACvB,SAAK,UAAU;AACf,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,CAAA,aAAY,KAAK,SAAS,KAAK,KAAK,CAAC,UAAU,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;AAAA,MACxE,CAAA,QAAO,KAAK,KAAK,GAAG;AAAA,MACpB;AAAA,IAAA;AAEF,SAAK,aAAa,IAAI,UAAA;AACtB,SAAK,aAAa;AAClB,SAAK,SAAS,IAAI,aAAa,KAAK,IAAI,QAAW,YAAY;AAAA,EACjE;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,IAAI,OAAO,wBAAwB;AAIxC,UAAM,KAAK,QAAQ,gBAAA;AAInB,UAAM,sBAAsB,wBAAwB,KAAK;AAEzD,WAAO,KAAK,OAAO,aAAa;AAC9B,UAAI;AACJ,UAAI,YAA2B;AAC/B,UAAI,iBAAiB;AACrB,UAAI;AACF,cAAM,EAAC,eAAe,iBAAA,IACpB,MAAM,KAAK,QAAQ,uCAAA;AACrB,cAAM,SAAS,MAAM,KAAK,QAAQ;AAAA,UAChC;AAAA,UACA;AAAA,QAAA;AAEF,aAAK,QAAQ,MAAM,MAAM,OAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;AAEtD,aAAK,UAAU;AACf,aAAK,OAAO,aAAA;AACZ,oBAAY;AAEZ,yBAAiB,UAAU,OAAO,SAAS;AACzC,gBAAM,CAAC,MAAM,GAAG,IAAI;AACpB,kBAAQ,MAAA;AAAA,YACN,KAAK;AACH,kBAAI,IAAI,KAAK;AACX,qBAAK,QAAQ,OAAO,MAAM;AAAA,cAC5B;AACA;AAAA,YACF,KAAK;AACH,oBAAM,KAAK,sBAAsB,GAAG;AACpC;AAAA;AAAA,YACF,KAAK;AACH,0BAAY,OAAO,CAAC,EAAE;AACtB;AAAA,YACF,KAAK;AACH,kBAAI,cAAc,OAAO,CAAC,EAAE,WAAW;AACrC,sBAAM,IAAI;AAAA,kBACR,oBAAoB,OAAO,CAAC,EAAE,SAAS,qCAAqC,SAAS;AAAA,gBAAA;AAAA,cAEzF;AACA,mBAAK,WAAW,IAAI,CAAC;AACrB;AAAA,YACF;AACE,kBAAI,cAAc,MAAM;AACtB,sBAAM,IAAI;AAAA,kBACR,GAAG,IAAI,YAAY,IAAI,GAAG;AAAA,gBAAA;AAAA,cAE9B;AACA;AAAA,UAAA;AAGJ,4BAAkB,KAAK,QAAQ,MAAM,CAAC,WAAW,MAAM,CAAC;AACxD,gBAAM,OAAO,KAAK,WAAW,QAAQ,CAAC,WAAW,MAAM,CAAC;AACxD,cAAI,kBAAkB,qBAAqB;AAQzC,kBAAM;AACN,6BAAiB;AAAA,UACnB;AAEA,cAAI,SAAS,YAAY,SAAS,YAAY;AAC5C,wBAAY;AAAA,UACd;AAGA,gBAAM,eAAe,KAAK,QAAQ,aAAA;AAClC,cAAI,cAAc;AAChB,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR,UAAA;AACE,aAAK,SAAS,QAAQ,OAAA;AACtB,aAAK,UAAU;AAAA,MACjB;AAGA,UAAI,WAAW;AACb,aAAK,IAAI,OAAO,oCAAoC,SAAS,EAAE;AAC/D,aAAK,QAAQ,MAAA;AACb,cAAM,KAAK,WAAW,QAAQ;AAAA,UAC5B;AAAA,UACA,CAAC,YAAY,EAAC,KAAK,YAAW;AAAA,QAAA,CAC/B;AAAA,MACH;AAGA,YAAM,QAAQ,IAAI;AAAA,QAChB,KAAK,QAAQ,KAAA;AAAA,QACb,KAAK,OAAO,QAAQ,KAAK,KAAK,GAAG;AAAA,MAAA,CAClC;AAAA,IACH;AACA,SAAK,IAAI,OAAO,wBAAwB;AAAA,EAC1C;AAAA,EAEA,MAAM,sBAAsB,KAA6B;AACvD,SAAK,IAAI,OAAO,4BAA4B,GAAG;AAC/C,UAAM,EAAC,QAAO;AAEd,YAAQ,KAAA;AAAA,MACN,KAAK;AACH,cAAM,kBAAkB,KAAK,WAAW,KAAK,MAAM;AACnD,cAAM;AAAA,UACJ,KAAK;AAAA,UACL;AAAA,UACA,IAAI,WAAW;AAAA,UACf,IAAI;AAAA,QAAA;AAEN,YAAI,KAAK,YAAY;AACnB,eAAK,IAAI,OAAO,8BAA8B;AAC9C,gBAAM,KAAK,KAAK,IAAI,iBAAiB;AAAA,QACvC;AACA;AAAA,MACF;AACE,oBAAe;AAAA,IAAA;AAAA,EAErB;AAAA,EAEA,UAAU,KAAqD;AAC7D,UAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,QAAI,SAAS,WAAW;AACtB,WAAK,SAAS,QAAA;AAAA,IAChB;AACA,UAAM,aAAa,aAAa,OAAmB;AAAA,MACjD,SAAS,MAAM,KAAK,WAAW,OAAO,UAAU;AAAA,IAAA,CACjD;AACD,UAAM,aAAa,IAAI;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI,mBAAmB,KAAK,iBAAiB;AAC3C,WAAK,IAAI;AAAA,QACP,2CAA2C,cAAc;AAAA,MAAA;AAE3D,iBAAW;AAAA,QACTA;AAAAA,QACA,8BACE,KAAK,eACP,eAAe,cAAc;AAAA,MAAA;AAAA,IAEjC,OAAO;AACL,WAAK,IAAI,QAAQ,qBAAqB,WAAW,EAAE,EAAE;AAErD,WAAK,WAAW,IAAI,UAAU;AAC9B,WAAK,QAAQ,QAAQ,YAAY,IAAI;AAErC,UAAI,SAAS;AACX,aAAK,gBAAgB,SAAS;AAAA,MAChC;AAAA,IACF;AACA,WAAO,QAAQ,QAAQ,UAAU;AAAA,EACnC;AAAA,EAEA,gBAAgB,WAAmB;AACjC,UAAM,WAAW,KAAK,mBAAmB;AACzC,SAAK,mBAAmB,IAAI,SAAS;AAErC,QAAI,aAAa,GAAG;AAClB,WAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,IACxE;AAAA,EACF;AAAA,EAEA,MAAM,oBAGH;AACD,UAAM,eAAe,MAAM,KAAK,QAAQ,0BAAA;AACxC,QAAI,CAAC,cAAc;AACjB,WAAK,IAAI;AAAA,QACP;AAAA,MAAA;AAAA,IAEJ;AACA,WAAO;AAAA,MACL,gBAAgB,KAAK;AAAA,MACrB,cAAc,gBAAgB,KAAK;AAAA,IAAA;AAAA,EAEvC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,mBAAkC;AACtC,UAAM,UAAU,CAAC,GAAG,KAAK,kBAAkB;AAC3C,QAAI,QAAQ,WAAW,GAAG;AACxB,WAAK,IAAI,OAAO,4CAA4C;AAC5D;AAAA,IACF;AACA,UAAM,UAAU,CAAC,GAAG,KAAK,WAAW,SAAS;AAC7C,QAAI,QAAQ,WAAW,GAAG;AAGxB,WAAK,IAAI,OAAO,mCAAmC;AACnD;AAAA,IACF;AACA,QAAI;AACF,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,UAAI,kBAAkB,iBAAiB;AACrC,aAAK,IAAI;AAAA,UACP,yCAAyC,eAAe,MAAM,eAAe;AAAA,QAAA;AAAA,MAEjF,OAAO;AACL,aAAK,IAAI,OAAO,0BAA0B,eAAe,MAAM;AAC/D,cAAM,QAAQ,YAAY,IAAA;AAC1B,cAAM,UAAU,MAAM,KAAK,QAAQ,mBAAmB,eAAe;AACrE,cAAM,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACrD,aAAK,IAAI;AAAA,UACP,UAAU,OAAO,mBAAmB,eAAe,KAAK,OAAO;AAAA,QAAA;AAEjE,aAAK,mBAAmB,OAAO,eAAe;AAAA,MAChD;AAAA,IACF,SAAS,GAAG;AACV,WAAK,IAAI,OAAO,4BAA4B,CAAC;AAAA,IAC/C,UAAA;AACE,UAAI,KAAK,mBAAmB,MAAM;AAEhC,aAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,KAAK,KAAe;AACxB,SAAK,OAAO,KAAK,KAAK,KAAK,GAAG;AAC9B,SAAK,SAAS,QAAQ,OAAA;AACtB,UAAM,KAAK,QAAQ,KAAA;AAAA,EACrB;AACF;AAiBA,MAAM,mBAAmB,6BAA6B;"}

package/out/zero-cache/src/services/change-streamer/storer.d.ts

@@ -57,7 +57,16 @@ export declare class Storer implements Service {
     status(s: DownstreamStatusMessage): void;
     catchup(subscriber: Subscriber, mode: ReplicatorMode): void;
     readyForMore(): Promise<void> | undefined;
+    /**
+     * Runs the storer loop until {@link stop()} is called, or an error is thrown.
+     * Once {@link run()} completes, it can be called again.
+     */
     run(): Promise<void>;
+    /**
+     * Waits until all currently queued entries have been processed.
+     * This is only used in tests.
+     */
+    allProcessed(): Promise<void>;
     stop(): Promise<void>;
 }
 //# sourceMappingURL=storer.d.ts.map

package/out/zero-cache/src/services/change-streamer/storer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAajD,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,iDAAiD,CAAC;AAC5E,OAAO,KAAK,EACV,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AAUpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AA8BhD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAiBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACvD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,+BAA+B,EAAE,MAAM;IA8BnC,eAAe;IAmBf,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;KACrC,CAAC;IAyCI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAQzD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAsBtD;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,iBAAiB;IAsB9B,KAAK;IAIL,MAAM,CAAC,CAAC,EAAE,uBAAuB;IAIjC,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAMpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IAwCnC,GAAG;IA0YT,IAAI;CAIL"}
+{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAajD,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,iDAAiD,CAAC;AAC5E,OAAO,KAAK,EACV,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AAUpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AA8BhD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAiBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACvD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,+BAA+B,EAAE,MAAM;IA8BnC,eAAe;IAmBf,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;KACrC,CAAC;IAkCI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAQzD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAsBtD;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,iBAAiB;IAsB9B,KAAK;IAIL,MAAM,CAAC,CAAC,EAAE,uBAAuB;IAIjC,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAMpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IA0CzC;;;OAGG;IACG,GAAG;IAuZT;;;OAGG;IACG,YAAY;IAQlB,IAAI;CAOL"}

package/out/zero-cache/src/services/change-streamer/storer.js

@@ -74,9 +74,6 @@ class Storer {
     );
   }
   async getStartStreamInitializationParameters() {
-    const { promise: ready, resolve } = resolver();
-    this.#queue.enqueue(["ready", resolve]);
-    await ready;
     const [[{ lastWatermark }], result] = await runTx(
       this.#db,
       (sql) => [
@@ -187,16 +184,32 @@ To inspect changeLog backlog in your change DB:
       this.#readyForMore = null;
     }
   }
+  #stopped = promiseVoid;
+  /**
+   * Runs the storer loop until {@link stop()} is called, or an error is thrown.
+   * Once {@link run()} completes, it can be called again.
+   */
   async run() {
+    assert(!this.#running, `storer is already running`);
+    const { promise: stopped, resolve: signalStopped } = resolver();
     this.#running = true;
+    this.#stopped = stopped;
+    this.#lc.info?.("starting storer");
     try {
       await this.#processQueue();
     } finally {
-      this.#running = false;
       if (this.#readyForMore !== null) {
         this.#readyForMore.resolve();
         this.#readyForMore = null;
       }
+      const unprocessed = this.#queue.drain();
+      if (unprocessed.length) {
+        this.#lc.warn?.(
+          `dropped ${unprocessed.length} entries from the changeLog queue`
+        );
+      }
+      this.#running = false;
+      signalStopped();
       this.#lc.info?.("storer stopped");
     }
   }
@@ -501,9 +514,23 @@ To inspect changeLog backlog in your change DB:
           DO UPDATE SET ${sql(row)};
     `;
   }
+  /**
+   * Waits until all currently queued entries have been processed.
+   * This is only used in tests.
+   */
+  async allProcessed() {
+    if (this.#running) {
+      const { promise, resolve } = resolver();
+      this.#queue.enqueue(["ready", resolve]);
+      await promise;
+    }
+  }
   stop() {
-    this.#queue.enqueue("stop");
-    return promiseVoid;
+    if (this.#running) {
+      this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);
+      this.#queue.enqueue("stop");
+    }
+    return this.#stopped;
   }
 }
 function toDownstream(entry) {

package/out/zero-cache/src/services/change-streamer/storer.js.map

@@ -1 +1 @@
-{"version":3,"file":"storer.js","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {getHeapStatistics} from 'node:v8';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {\n  backfillRequestSchema,\n  isDataChange,\n  isSchemaChange,\n  type BackfillID,\n  type BackfillRequest,\n  type Change,\n  type DataChange,\n  type Identifier,\n  type SchemaChange,\n  type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {type Commit} from '../change-source/protocol/current/downstream.ts';\nimport type {\n  DownstreamStatusMessage,\n  UpstreamStatusMessage,\n} from '../change-source/protocol/current/status.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type ChangeEntry} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n  AutoResetSignal,\n  markResetRequired,\n  type BackfillingColumn,\n  type ReplicationState,\n  type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n  subscriber: Subscriber;\n  mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n  | [\n      'change',\n      watermark: string,\n      json: string,\n      orig: Exclude<Change, DataChange> | null, // null for DataChanges\n    ]\n  | ['ready', callback: () => void]\n  | ['subscriber', SubscriberAndMode]\n  | DownstreamStatusMessage\n  | ['abort']\n  | 'stop';\n\ntype PendingTransaction = {\n  pool: TransactionPool;\n  preCommitWatermark: string;\n  pos: number;\n  startingReplicationState: Promise<ReplicationState>;\n  ack: boolean;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n *   `replicaVersion`, which is the version at which initial sync\n *   was performed, and any attempts to catchup from a different\n *   point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n *   a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n *   subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n  readonly id = 'storer';\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #taskID: string;\n  readonly #discoveryAddress: string;\n  readonly #discoveryProtocol: string;\n  readonly #db: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #onConsumed: (c: Commit | UpstreamStatusMessage) => void;\n  readonly #onFatal: (err: Error) => void;\n  readonly #queue = new Queue<QueueEntry>();\n  readonly #backPressureThresholdBytes: number;\n\n  #approximateQueuedBytes = 0;\n  #running = false;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    db: PostgresDB,\n    replicaVersion: string,\n    onConsumed: (c: Commit | UpstreamStatusMessage) => void,\n    onFatal: (err: Error) => void,\n    backPressureLimitHeapProportion: number,\n  ) {\n    this.#lc = lc.withContext('component', 'change-log');\n    this.#shard = shard;\n    this.#taskID = taskID;\n    this.#discoveryAddress = discoveryAddress;\n    this.#discoveryProtocol = discoveryProtocol;\n    this.#db = db;\n    this.#replicaVersion = replicaVersion;\n    this.#onConsumed = onConsumed;\n    this.#onFatal = onFatal;\n\n    const heapStats = getHeapStatistics();\n    this.#backPressureThresholdBytes =\n      (heapStats.heap_size_limit - heapStats.used_heap_size) *\n      backPressureLimitHeapProportion;\n\n    this.#lc.info?.(\n      `Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of ` +\n        `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) ` +\n        `to absorb upstream spikes`,\n      {heapStats},\n    );\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async assumeOwnership() {\n    const db = this.#db;\n    const owner = this.#taskID;\n    const ownerAddress = this.#discoveryAddress;\n    const ownerProtocol = this.#discoveryProtocol;\n    // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n    const addressWithProtocol =\n      ownerProtocol === 'ws'\n        ? ownerAddress\n        : `${ownerProtocol}://${ownerAddress}`;\n    this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n    const start = performance.now();\n    await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n    const elapsed = (performance.now() - start).toFixed(2);\n    this.#lc.info?.(\n      `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n    );\n  }\n\n  async getStartStreamInitializationParameters(): Promise<{\n    lastWatermark: string;\n    backfillRequests: BackfillRequest[];\n  }> {\n    // Before starting or restarting a stream from the change source,\n    // wait for all queued changes to be processed so that we pick up\n    // from the right spot.\n    const {promise: ready, resolve} = resolver();\n    this.#queue.enqueue(['ready', resolve]);\n    await ready;\n\n    const [[{lastWatermark}], result] = await runTx(\n      this.#db,\n      sql => [\n        sql<{lastWatermark: string}[]>`\n        SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n        // Formats a BackfillRequest using json_object_agg() to construct the\n        // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n        // to make it optional and possibly `null`.\n        sql`\n        SELECT \n            json_build_object(\n              'schema', b.\"schema\",\n              'name', b.\"table\",\n              'metadata', t.\"metadata\"\n            ) as \"table\",\n            json_object_agg(b.\"column\", b.\"backfill\") \n              as \"columns\"\n          FROM ${this.#cdc('backfilling')} as b\n          LEFT JOIN ${this.#cdc('tableMetadata')} as t\n          ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n          GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n        `,\n      ],\n      {mode: Mode.READONLY},\n    );\n\n    return {\n      lastWatermark,\n      backfillRequests: v.parse(result, backfillRequestsSchema),\n    };\n  }\n\n  async getMinWatermarkForCatchup(): Promise<string | null> {\n    const [{minWatermark}] = await this.#db<\n      {minWatermark: string | null}[]\n    > /*sql*/ `\n      SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n    return minWatermark;\n  }\n\n  purgeRecordsBefore(watermark: string): Promise<number> {\n    return runTx(this.#db, async sql => {\n      const [{deleted}] = await sql<{deleted: bigint}[]>`\n        WITH purged AS (\n          DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n            RETURNING watermark, pos\n        ) SELECT COUNT(*) as deleted FROM purged;`;\n\n      // Before committing the purge, check that this process is still the\n      // owner. This is done after the DELETE to minimize the amount of time\n      // that writes to the changeLog are delayed.\n      const [{owner}] = await sql<ReplicationState[]>`\n        SELECT * FROM ${this.#cdc('replicationState')} FOR SHARE`;\n      if (owner !== this.#taskID) {\n        throw new AbortError(\n          `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n        );\n      }\n      return Number(deleted);\n    });\n  }\n\n  /**\n   * @returns The size of the serialized entry, for memory / I/O estimations.\n   */\n  store(entry: WatermarkedChange) {\n    const [watermark, [_tag, change]] = entry;\n    // Eagerly stringify the JSON object so that the memory usage can be\n    // more accurately measured (i.e. without an extra object traversal and\n    // ad hoc memory counting heuristics).\n    //\n    // This essentially moves the stringify() computation out of the pg client,\n    // which is instead configured to pass `string` objects directly as JSON\n    // strings for JSON-valued columns (see TypeOptions.sendStringAsJson).\n    const json = BigIntJSON.stringify(change);\n    this.#approximateQueuedBytes += json.length;\n\n    this.#queue.enqueue([\n      'change',\n      watermark,\n      json,\n      isDataChange(change) ? null : change, // drop DataChanges to save memory\n    ]);\n\n    return json.length;\n  }\n\n  abort() {\n    this.#queue.enqueue(['abort']);\n  }\n\n  status(s: DownstreamStatusMessage) {\n    this.#queue.enqueue(s);\n  }\n\n  catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n    this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n  }\n\n  #readyForMore: Resolver<void> | null = null;\n\n  readyForMore(): Promise<void> | undefined {\n    if (!this.#running) {\n      return undefined;\n    }\n    if (\n      this.#readyForMore === null &&\n      this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n    ) {\n      this.#lc.warn?.(\n        `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\\n` +\n          `\\n` +\n          `To inspect changeLog backlog in your change DB:\\n` +\n          `  SELECT\\n` +\n          `    (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\\n` +\n          `    change->>'tag' AS operation,\\n` +\n          `    COUNT(*) AS count\\n` +\n          `  FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n          `  GROUP BY 1, 2\\n` +\n          `  ORDER BY 3 DESC\\n` +\n          `  LIMIT 20;`,\n      );\n      this.#readyForMore = resolver();\n    }\n    return this.#readyForMore?.promise;\n  }\n\n  #maybeReleaseBackPressure() {\n    if (\n      this.#readyForMore !== null &&\n      // Wait for at least 20% of the threshold to free up.\n      this.#approximateQueuedBytes < this.#backPressureThresholdBytes * 0.8\n    ) {\n      this.#lc.info?.(\n        `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`,\n      );\n      this.#readyForMore.resolve();\n      this.#readyForMore = null;\n    }\n  }\n\n  async run() {\n    this.#running = true;\n    try {\n      await this.#processQueue();\n    } finally {\n      this.#running = false;\n      // Release any pending backpressure so the upstream can proceed\n      if (this.#readyForMore !== null) {\n        this.#readyForMore.resolve();\n        this.#readyForMore = null;\n      }\n      this.#lc.info?.('storer stopped');\n    }\n  }\n\n  async #processQueue() {\n    let tx: PendingTransaction | null = null;\n    let msg: QueueEntry | false;\n\n    const catchupQueue: SubscriberAndMode[] = [];\n    while ((msg = await this.#queue.dequeue()) !== 'stop') {\n      const [msgType] = msg;\n      switch (msgType) {\n        case 'ready': {\n          const signalReady = msg[1];\n          signalReady();\n          continue;\n        }\n        case 'subscriber': {\n          const subscriber = msg[1];\n          if (tx) {\n            catchupQueue.push(subscriber); // Wait for the current tx to complete.\n          } else {\n            await this.#startCatchup([subscriber]); // Catch up immediately.\n          }\n          continue;\n        }\n        case 'status':\n          this.#onConsumed(msg);\n          continue;\n        case 'abort': {\n          if (tx) {\n            tx.pool.abort();\n            await tx.pool.done();\n            tx = null;\n          }\n          continue;\n        }\n      }\n      // msgType === 'change'\n      const [_, watermark, json, change] = msg;\n      const tag = change?.tag;\n      this.#approximateQueuedBytes -= json.length;\n\n      if (tag === 'begin') {\n        assert(!tx, 'received BEGIN in the middle of a transaction');\n        const {promise, resolve, reject} = resolver<ReplicationState>();\n        tx = {\n          pool: new TransactionPool(\n            this.#lc.withContext('watermark', watermark),\n            Mode.READ_COMMITTED,\n          ),\n          preCommitWatermark: watermark,\n          pos: 0,\n          startingReplicationState: promise,\n          ack: !change.skipAck,\n        };\n        tx.pool.run(this.#db);\n        // Acquire a lock on the replicationState row to detect and/or prevent\n        // a concurrent ownership change.\n        void tx.pool.process(tx => {\n          tx<ReplicationState[]> /*sql*/ `\n          SELECT * FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n            ([result]) => resolve(result),\n            reject,\n          );\n          return [];\n        });\n      } else {\n        assert(tx, () => `received change outside of transaction: ${json}`);\n        tx.pos++;\n      }\n\n      const entry = {\n        watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n        precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n        pos: tx.pos,\n        change: json,\n      };\n\n      const processed = tx.pool.process(sql => [\n        sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n        ...(change !== null && isSchemaChange(change)\n          ? this.#trackBackfillMetadata(sql, change)\n          : []),\n      ]);\n\n      if (tx.pos % 100 === 0) {\n        // Backpressure is exerted on commit when awaiting tx.pool.done().\n        // However, backpressure checks need to be regularly done for\n        // very large transactions in order to avoid memory blowup.\n        await processed;\n      }\n      this.#maybeReleaseBackPressure();\n\n      if (tag === 'commit') {\n        const {owner} = await tx.startingReplicationState;\n        if (owner !== this.#taskID) {\n          // Ownership change reflected in the replicationState read in 'begin'.\n          tx.pool.fail(\n            new AbortError(`changeLog ownership has been assumed by ${owner}`),\n          );\n        } else {\n          // Update the replication state.\n          const lastWatermark = watermark;\n          void tx.pool.process(tx => [\n            tx`\n            UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n          ]);\n          tx.pool.setDone();\n        }\n\n        await tx.pool.done();\n\n        // ACK the LSN to the upstream Postgres.\n        if (tx.ack) {\n          this.#onConsumed(['commit', change, {watermark}]);\n        }\n        tx = null;\n\n        // Before beginning the next transaction, open a READONLY snapshot to\n        // concurrently catchup any queued subscribers.\n        await this.#startCatchup(catchupQueue.splice(0));\n      } else if (tag === 'rollback') {\n        // Aborted transactions are not stored in the changeLog. Abort the current tx\n        // and process catchup of subscribers that were waiting for it to end.\n        tx.pool.abort();\n        await tx.pool.done();\n        tx = null;\n\n        await this.#startCatchup(catchupQueue.splice(0));\n      }\n    }\n  }\n\n  async #startCatchup(subs: SubscriberAndMode[]) {\n    if (subs.length === 0) {\n      return;\n    }\n\n    const reader = new TransactionPool(\n      this.#lc.withContext('pool', 'catchup'),\n      Mode.READONLY,\n    );\n    reader.run(this.#db);\n\n    // Ensure that the transaction has started (and is thus holding a snapshot\n    // of the database) before continuing on to commit more changes. This is\n    // done by performing a single read on the db, which determines the\n    // snapshot for the REPEATABLE_READ transaction.\n    const [{lastWatermark}] = await reader.processReadTask(\n      sql => sql<ReplicationState[]>`\n        SELECT * FROM ${this.#cdc('replicationState')}\n      `,\n    );\n\n    // Run the actual catchup queries in the background. Errors are handled in\n    // #catchup() by disconnecting the associated subscriber.\n    void Promise.all(\n      subs.map(sub => this.#catchup(sub, lastWatermark, reader)),\n    ).finally(() => reader.setDone());\n  }\n\n  async #catchup(\n    {subscriber: sub, mode}: SubscriberAndMode,\n    lastWatermark: string,\n    reader: TransactionPool,\n  ) {\n    try {\n      await reader.processReadTask(async tx => {\n        const start = Date.now();\n\n        // When starting from initial-sync, there won't be a change with a watermark\n        // equal to the replica version. This is the empty changeLog scenario.\n        let watermarkFound = sub.watermark === this.#replicaVersion;\n        let count = 0;\n        let lastBatchConsumed: Promise<unknown> | undefined;\n\n        for await (const entries of tx<ChangeEntry[]> /*sql*/ `\n          SELECT watermark, change FROM ${this.#cdc('changeLog')}\n           WHERE watermark >= ${sub.watermark}\n             AND watermark <= ${lastWatermark}\n           ORDER BY watermark, pos`.cursor(2000)) {\n          // Wait for the last batch of entries to be consumed by the\n          // subscriber before sending down the current batch. This pipelining\n          // allows one batch of changes to be received from the change-db\n          // while the previous batch of changes are sent to the subscriber,\n          // resulting in flow control that caps the number of changes\n          // referenced in memory to 2 * batch-size.\n          const start = performance.now();\n          await lastBatchConsumed;\n          const elapsed = performance.now() - start;\n          if (lastBatchConsumed) {\n            (elapsed > 100 ? this.#lc.info : this.#lc.debug)?.(\n              `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n            );\n          }\n\n          for (const entry of entries) {\n            if (entry.watermark === sub.watermark) {\n              // This should be the first entry.\n              // Catchup starts from *after* the watermark.\n              watermarkFound = true;\n            } else if (watermarkFound) {\n              lastBatchConsumed = sub.catchup(toDownstream(entry));\n              count++;\n            } else if (mode === 'backup') {\n              throw new AutoResetSignal(\n                `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n              );\n            } else {\n              this.#lc.warn?.(\n                `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n              );\n              sub.close(\n                ErrorType.WatermarkTooOld,\n                `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n              );\n              return;\n            }\n          }\n        }\n        if (watermarkFound) {\n          await lastBatchConsumed;\n          this.#lc.info?.(\n            `caught up ${sub.id} with ${count} changes (${\n              Date.now() - start\n            } ms)`,\n          );\n        } else {\n          this.#lc.warn?.(\n            `subscriber at watermark ${sub.watermark} is ahead of latest watermark`,\n          );\n        }\n        // Flushes the backlog of messages buffered during catchup and\n        // allows the subscription to forward subsequent messages immediately.\n        sub.setCaughtUp();\n      });\n    } catch (err) {\n      this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n      if (err instanceof AutoResetSignal) {\n        await markResetRequired(this.#db, this.#shard);\n        this.#onFatal(err);\n      }\n      sub.fail(err);\n    }\n  }\n\n  /**\n   * Returns the db statements necessary to track backfill and table metadata\n   * presented in the `change`, if any.\n   */\n  #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n    const stmts: PendingQuery<Row[]>[] = [];\n\n    switch (change.tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        stmts.push(this.#upsertTableMetadataStmt(sql, table, metadata));\n        break;\n      }\n\n      case 'create-table': {\n        const {spec, metadata, backfill} = change;\n        if (metadata) {\n          stmts.push(this.#upsertTableMetadataStmt(sql, spec, metadata));\n        }\n        if (backfill) {\n          Object.entries(backfill).forEach(([col, backfill]) => {\n            stmts.push(\n              this.#upsertColumnBackfillStmt(sql, spec, col, backfill),\n            );\n          });\n        }\n        break;\n      }\n\n      case 'rename-table': {\n        const {old} = change;\n        const row = {schema: change.new.schema, table: change.new.name};\n        stmts.push(\n          sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n          sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n        );\n        break;\n      }\n\n      case 'drop-table': {\n        const {\n          id: {schema, name},\n        } = change;\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('tableMetadata')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n        );\n        break;\n      }\n\n      case 'add-column': {\n        const {table, tableMetadata, column, backfill} = change;\n        if (tableMetadata) {\n          stmts.push(this.#upsertTableMetadataStmt(sql, table, tableMetadata));\n        }\n        if (backfill) {\n          stmts.push(\n            this.#upsertColumnBackfillStmt(sql, table, column.name, backfill),\n          );\n        }\n        break;\n      }\n\n      case 'update-column': {\n        const {\n          table: {schema, name: table},\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          stmts.push(\n            sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${newName}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${oldName}`,\n          );\n        }\n        break;\n      }\n\n      case 'drop-column': {\n        const {\n          table: {schema, name},\n          column,\n        } = change;\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${column}`,\n        );\n        break;\n      }\n\n      case 'backfill-completed': {\n        const {\n          relation: {schema, name: table, rowKey},\n          columns,\n        } = change;\n        const cols = [...rowKey.columns, ...columns];\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql(cols)}`,\n        );\n      }\n    }\n    return stmts;\n  }\n\n  #upsertTableMetadataStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    metadata: TableMetadata,\n  ) {\n    const row: TableMetadataRow = {schema, table, metadata};\n    return sql`\n        INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\") \n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  #upsertColumnBackfillStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    column: string,\n    backfill: BackfillID,\n  ) {\n    const row: BackfillingColumn = {schema, table, column, backfill};\n    return sql`\n        INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\", \"column\") \n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  stop() {\n    this.#queue.enqueue('stop');\n    return promiseVoid;\n  }\n}\n\nfunction toDownstream(entry: ChangeEntry): WatermarkedChange {\n  const {watermark, change} = entry;\n  switch (change.tag) {\n    case 'begin':\n      return [watermark, ['begin', change, {commitWatermark: watermark}]];\n    case 'commit':\n      return [watermark, ['commit', change, {watermark}]];\n    case 'rollback':\n      return [watermark, ['rollback', change]];\n    default:\n      return [watermark, ['data', change]];\n  }\n}\n"],"names":["v.array","Mode.READONLY","v.parse","Mode.READ_COMMITTED","tx","start","ErrorType.WatermarkTooOld","backfill"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAwEA,MAAM,yBAAyBA,MAAQ,qBAAqB;AAgCrD,MAAM,OAA0B;AAAA,EAC5B,KAAK;AAAA,EACL;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS,IAAI,MAAA;AAAA,EACb;AAAA,EAET,0BAA0B;AAAA,EAC1B,WAAW;AAAA,EAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,YACA,SACA,iCACA;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,YAAY;AACnD,SAAK,SAAS;AACd,SAAK,UAAU;AACf,SAAK,oBAAoB;AACzB,SAAK,qBAAqB;AAC1B,SAAK,MAAM;AACX,SAAK,kBAAkB;AACvB,SAAK,cAAc;AACnB,SAAK,WAAW;AAEhB,UAAM,YAAY,kBAAA;AAClB,SAAK,+BACF,UAAU,kBAAkB,UAAU,kBACvC;AAEF,SAAK,IAAI;AAAA,MACP,gBAAgB,KAAK,8BAA8B,QAAQ,GAAG,QAAQ,CAAC,CAAC,kCAC3C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,CAAC,CAAC;AAAA,MAE9E,EAAC,UAAA;AAAA,IAAS;AAAA,EAEd;AAAA;AAAA,EAGA,KAAK,OAAe;AAClB,WAAO,KAAK,IAAI,GAAG,UAAU,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE;AAAA,EACtD;AAAA,EAEA,MAAM,kBAAkB;AACtB,UAAM,KAAK,KAAK;AAChB,UAAM,QAAQ,KAAK;AACnB,UAAM,eAAe,KAAK;AAC1B,UAAM,gBAAgB,KAAK;AAE3B,UAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,aAAa,MAAM,YAAY;AACxC,SAAK,IAAI,OAAO,yBAAyB,mBAAmB,EAAE;AAC9D,UAAM,QAAQ,YAAY,IAAA;AAC1B,UAAM,YAAY,KAAK,KAAK,kBAAkB,CAAC,QAAQ,GAAG,EAAC,OAAO,cAAc,oBAAA,CAAoB,CAAC;AACrG,UAAM,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACrD,SAAK,IAAI;AAAA,MACP,wBAAwB,mBAAmB,KAAK,OAAO;AAAA,IAAA;AAAA,EAE3D;AAAA,EAEA,MAAM,yCAGH;AAID,UAAM,EAAC,SAAS,OAAO,QAAA,IAAW,SAAA;AAClC,SAAK,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;AACtC,UAAM;AAEN,UAAM,CAAC,CAAC,EAAC,cAAA,CAAc,GAAG,MAAM,IAAI,MAAM;AAAA,MACxC,KAAK;AAAA,MACL,CAAA,QAAO;AAAA,QACL;AAAA,sCAC8B,KAAK,KAAK,kBAAkB,CAAC;AAAA;AAAA;AAAA;AAAA,QAK3D;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBASS,KAAK,KAAK,aAAa,CAAC;AAAA,sBACnB,KAAK,KAAK,eAAe,CAAC;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,MAK1C,EAAC,MAAMC,SAAK;AAAA,IAAQ;AAGtB,WAAO;AAAA,MACL;AAAA,MACA,kBAAkBC,MAAQ,QAAQ,sBAAsB;AAAA,IAAA;AAAA,EAE5D;AAAA,EAEA,MAAM,4BAAoD;AACxD,UAAM,CAAC,EAAC,aAAA,CAAa,IAAI,MAAM,KAAK;AAAA,qDAGa,KAAK,KAAK,WAAW,CAAC;AACvE,WAAO;AAAA,EACT;AAAA,EAEA,mBAAmB,WAAoC;AACrD,WAAO,MAAM,KAAK,KAAK,OAAM,QAAO;AAClC,YAAM,CAAC,EAAC,SAAQ,IAAI,MAAM;AAAA;AAAA,wBAER,KAAK,KAAK,WAAW,CAAC,sBAAsB,SAAS;AAAA;AAAA;AAOvE,YAAM,CAAC,EAAC,OAAM,IAAI,MAAM;AAAA,wBACN,KAAK,KAAK,kBAAkB,CAAC;AAC/C,UAAI,UAAU,KAAK,SAAS;AAC1B,cAAM,IAAI;AAAA,UACR,+BAA+B,SAAS,wCAAwC,KAAK;AAAA,QAAA;AAAA,MAEzF;AACA,aAAO,OAAO,OAAO;AAAA,IACvB,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,OAA0B;AAC9B,UAAM,CAAC,WAAW,CAAC,MAAM,MAAM,CAAC,IAAI;AAQpC,UAAM,OAAO,WAAW,UAAU,MAAM;AACxC,SAAK,2BAA2B,KAAK;AAErC,SAAK,OAAO,QAAQ;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,MAAM,IAAI,OAAO;AAAA;AAAA,IAAA,CAC/B;AAED,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,QAAQ;AACN,SAAK,OAAO,QAAQ,CAAC,OAAO,CAAC;AAAA,EAC/B;AAAA,EAEA,OAAO,GAA4B;AACjC,SAAK,OAAO,QAAQ,CAAC;AAAA,EACvB;AAAA,EAEA,QAAQ,YAAwB,MAAsB;AACpD,SAAK,OAAO,QAAQ,CAAC,cAAc,EAAC,YAAY,KAAA,CAAK,CAAC;AAAA,EACxD;AAAA,EAEA,gBAAuC;AAAA,EAEvC,eAA0C;AACxC,QAAI,CAAC,KAAK,UAAU;AAClB,aAAO;AAAA,IACT;AACA,QACE,KAAK,kBAAkB,QACvB,KAAK,0BAA0B,KAAK,6BACpC;AACA,WAAK,IAAI;AAAA,QACP,+BAA+B,KAAK,OAAO,KAAA,CAAM,sBAAsB,KAAK,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA;AAY7H,WAAK,gBAAgB,SAAA;AAAA,IACvB;AACA,WAAO,KAAK,eAAe;AAAA,EAC7B;AAAA,EAEA,4BAA4B;AAC1B,QACE,KAAK,kBAAkB;AAAA,IAEvB,KAAK,0BAA0B,KAAK,8BAA8B,KAClE;AACA,WAAK,IAAI;AAAA,QACP,gCAAgC,KAAK,OAAO,KAAA,CAAM,sBAAsB,KAAK,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,CAAC;AAAA,MAAA;AAE9H,WAAK,cAAc,QAAA;AACnB,WAAK,gBAAgB;AAAA,IACvB;AAAA,EACF;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,WAAW;AAChB,QAAI;AACF,YAAM,KAAK,cAAA;AAAA,IACb,UAAA;AACE,WAAK,WAAW;AAEhB,UAAI,KAAK,kBAAkB,MAAM;AAC/B,aAAK,cAAc,QAAA;AACnB,aAAK,gBAAgB;AAAA,MACvB;AACA,WAAK,IAAI,OAAO,gBAAgB;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,MAAM,gBAAgB;AACpB,QAAI,KAAgC;AACpC,QAAI;AAEJ,UAAM,eAAoC,CAAA;AAC1C,YAAQ,MAAM,MAAM,KAAK,OAAO,QAAA,OAAe,QAAQ;AACrD,YAAM,CAAC,OAAO,IAAI;AAClB,cAAQ,SAAA;AAAA,QACN,KAAK,SAAS;AACZ,gBAAM,cAAc,IAAI,CAAC;AACzB,sBAAA;AACA;AAAA,QACF;AAAA,QACA,KAAK,cAAc;AACjB,gBAAM,aAAa,IAAI,CAAC;AACxB,cAAI,IAAI;AACN,yBAAa,KAAK,UAAU;AAAA,UAC9B,OAAO;AACL,kBAAM,KAAK,cAAc,CAAC,UAAU,CAAC;AAAA,UACvC;AACA;AAAA,QACF;AAAA,QACA,KAAK;AACH,eAAK,YAAY,GAAG;AACpB;AAAA,QACF,KAAK,SAAS;AACZ,cAAI,IAAI;AACN,eAAG,KAAK,MAAA;AACR,kBAAM,GAAG,KAAK,KAAA;AACd,iBAAK;AAAA,UACP;AACA;AAAA,QACF;AAAA,MAAA;AAGF,YAAM,CAAC,GAAG,WAAW,MAAM,MAAM,IAAI;AACrC,YAAM,MAAM,QAAQ;AACpB,WAAK,2BAA2B,KAAK;AAErC,UAAI,QAAQ,SAAS;AACnB,eAAO,CAAC,IAAI,+CAA+C;AAC3D,cAAM,EAAC,SAAS,SAAS,OAAA,IAAU,SAAA;AACnC,aAAK;AAAA,UACH,MAAM,IAAI;AAAA,YACR,KAAK,IAAI,YAAY,aAAa,SAAS;AAAA,YAC3CC;AAAAA,UAAK;AAAA,UAEP,oBAAoB;AAAA,UACpB,KAAK;AAAA,UACL,0BAA0B;AAAA,UAC1B,KAAK,CAAC,OAAO;AAAA,QAAA;AAEf,WAAG,KAAK,IAAI,KAAK,GAAG;AAGpB,aAAK,GAAG,KAAK,QAAQ,CAAAC,QAAM;AACzBA;AAAAA,0BACgB,KAAK,KAAK,kBAAkB,CAAC,cAAc;AAAA,YACzD,CAAC,CAAC,MAAM,MAAM,QAAQ,MAAM;AAAA,YAC5B;AAAA,UAAA;AAEF,iBAAO,CAAA;AAAA,QACT,CAAC;AAAA,MACH,OAAO;AACL,eAAO,IAAI,MAAM,2CAA2C,IAAI,EAAE;AAClE,WAAG;AAAA,MACL;AAEA,YAAM,QAAQ;AAAA,QACZ,WAAW,QAAQ,WAAW,YAAY,GAAG;AAAA,QAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;AAAA,QACtD,KAAK,GAAG;AAAA,QACR,QAAQ;AAAA,MAAA;AAGV,YAAM,YAAY,GAAG,KAAK,QAAQ,CAAA,QAAO;AAAA,QACvC,kBAAkB,KAAK,KAAK,WAAW,CAAC,IAAI,IAAI,KAAK,CAAC;AAAA,QACtD,GAAI,WAAW,QAAQ,eAAe,MAAM,IACxC,KAAK,uBAAuB,KAAK,MAAM,IACvC,CAAA;AAAA,MAAC,CACN;AAED,UAAI,GAAG,MAAM,QAAQ,GAAG;AAItB,cAAM;AAAA,MACR;AACA,WAAK,0BAAA;AAEL,UAAI,QAAQ,UAAU;AACpB,cAAM,EAAC,MAAA,IAAS,MAAM,GAAG;AACzB,YAAI,UAAU,KAAK,SAAS;AAE1B,aAAG,KAAK;AAAA,YACN,IAAI,WAAW,2CAA2C,KAAK,EAAE;AAAA,UAAA;AAAA,QAErE,OAAO;AAEL,gBAAM,gBAAgB;AACtB,eAAK,GAAG,KAAK,QAAQ,CAAAA,QAAM;AAAA,YACzBA;AAAAA,qBACS,KAAK,KAAK,kBAAkB,CAAC,QAAQA,IAAG,EAAC,cAAA,CAAc,CAAC;AAAA,UAAA,CAClE;AACD,aAAG,KAAK,QAAA;AAAA,QACV;AAEA,cAAM,GAAG,KAAK,KAAA;AAGd,YAAI,GAAG,KAAK;AACV,eAAK,YAAY,CAAC,UAAU,QAAQ,EAAC,UAAA,CAAU,CAAC;AAAA,QAClD;AACA,aAAK;AAIL,cAAM,KAAK,cAAc,aAAa,OAAO,CAAC,CAAC;AAAA,MACjD,WAAW,QAAQ,YAAY;AAG7B,WAAG,KAAK,MAAA;AACR,cAAM,GAAG,KAAK,KAAA;AACd,aAAK;AAEL,cAAM,KAAK,cAAc,aAAa,OAAO,CAAC,CAAC;AAAA,MACjD;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,MAA2B;AAC7C,QAAI,KAAK,WAAW,GAAG;AACrB;AAAA,IACF;AAEA,UAAM,SAAS,IAAI;AAAA,MACjB,KAAK,IAAI,YAAY,QAAQ,SAAS;AAAA,MACtCH;AAAAA,IAAK;AAEP,WAAO,IAAI,KAAK,GAAG;AAMnB,UAAM,CAAC,EAAC,cAAA,CAAc,IAAI,MAAM,OAAO;AAAA,MACrC,CAAA,QAAO;AAAA,wBACW,KAAK,KAAK,kBAAkB,CAAC;AAAA;AAAA,IAAA;AAMjD,SAAK,QAAQ;AAAA,MACX,KAAK,IAAI,CAAA,QAAO,KAAK,SAAS,KAAK,eAAe,MAAM,CAAC;AAAA,IAAA,EACzD,QAAQ,MAAM,OAAO,SAAS;AAAA,EAClC;AAAA,EAEA,MAAM,SACJ,EAAC,YAAY,KAAK,KAAA,GAClB,eACA,QACA;AACA,QAAI;AACF,YAAM,OAAO,gBAAgB,OAAM,OAAM;AACvC,cAAM,QAAQ,KAAK,IAAA;AAInB,YAAI,iBAAiB,IAAI,cAAc,KAAK;AAC5C,YAAI,QAAQ;AACZ,YAAI;AAEJ,yBAAiB,WAAW;AAAA,0CACM,KAAK,KAAK,WAAW,CAAC;AAAA,gCAChC,IAAI,SAAS;AAAA,gCACb,aAAa;AAAA,oCACT,OAAO,GAAI,GAAG;AAOxC,gBAAMI,SAAQ,YAAY,IAAA;AAC1B,gBAAM;AACN,gBAAM,UAAU,YAAY,IAAA,IAAQA;AACpC,cAAI,mBAAmB;AACrB,aAAC,UAAU,MAAM,KAAK,IAAI,OAAO,KAAK,IAAI;AAAA,cACxC,UAAU,QAAQ,QAAQ,CAAC,CAAC,WAAW,IAAI,EAAE;AAAA,YAAA;AAAA,UAEjD;AAEA,qBAAW,SAAS,SAAS;AAC3B,gBAAI,MAAM,cAAc,IAAI,WAAW;AAGrC,+BAAiB;AAAA,YACnB,WAAW,gBAAgB;AACzB,kCAAoB,IAAI,QAAQ,aAAa,KAAK,CAAC;AACnD;AAAA,YACF,WAAW,SAAS,UAAU;AAC5B,oBAAM,IAAI;AAAA,gBACR,+BAA+B,IAAI,SAAS,yBAAyB,MAAM,SAAS;AAAA,cAAA;AAAA,YAExF,OAAO;AACL,mBAAK,IAAI;AAAA,gBACP,qCAAqC,IAAI,SAAS,yBAAyB,MAAM,SAAS;AAAA,cAAA;AAE5F,kBAAI;AAAA,gBACFC;AAAAA,gBACA,mCAAmC,MAAM,SAAS,eAAe,IAAI,SAAS;AAAA,cAAA;AAEhF;AAAA,YACF;AAAA,UACF;AAAA,QACF;AACA,YAAI,gBAAgB;AAClB,gBAAM;AACN,eAAK,IAAI;AAAA,YACP,aAAa,IAAI,EAAE,SAAS,KAAK,aAC/B,KAAK,QAAQ,KACf;AAAA,UAAA;AAAA,QAEJ,OAAO;AACL,eAAK,IAAI;AAAA,YACP,2BAA2B,IAAI,SAAS;AAAA,UAAA;AAAA,QAE5C;AAGA,YAAI,YAAA;AAAA,MACN,CAAC;AAAA,IACH,SAAS,KAAK;AACZ,WAAK,IAAI,QAAQ,sCAAsC,IAAI,EAAE,IAAI,GAAG;AACpE,UAAI,eAAe,iBAAiB;AAClC,cAAM,kBAAkB,KAAK,KAAK,KAAK,MAAM;AAC7C,aAAK,SAAS,GAAG;AAAA,MACnB;AACA,UAAI,KAAK,GAAG;AAAA,IACd;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,uBAAuB,KAA0B,QAAsB;AACrE,UAAM,QAA+B,CAAA;AAErC,YAAQ,OAAO,KAAA;AAAA,MACb,KAAK,yBAAyB;AAC5B,cAAM,EAAC,OAAO,KAAK,SAAA,IAAY;AAC/B,cAAM,KAAK,KAAK,yBAAyB,KAAK,OAAO,QAAQ,CAAC;AAC9D;AAAA,MACF;AAAA,MAEA,KAAK,gBAAgB;AACnB,cAAM,EAAC,MAAM,UAAU,SAAA,IAAY;AACnC,YAAI,UAAU;AACZ,gBAAM,KAAK,KAAK,yBAAyB,KAAK,MAAM,QAAQ,CAAC;AAAA,QAC/D;AACA,YAAI,UAAU;AACZ,iBAAO,QAAQ,QAAQ,EAAE,QAAQ,CAAC,CAAC,KAAKC,SAAQ,MAAM;AACpD,kBAAM;AAAA,cACJ,KAAK,0BAA0B,KAAK,MAAM,KAAKA,SAAQ;AAAA,YAAA;AAAA,UAE3D,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAAA,MAEA,KAAK,gBAAgB;AACnB,cAAM,EAAC,QAAO;AACd,cAAM,MAAM,EAAC,QAAQ,OAAO,IAAI,QAAQ,OAAO,OAAO,IAAI,KAAA;AAC1D,cAAM;AAAA,UACJ,aAAa,KAAK,KAAK,eAAe,CAAC,QAAQ,IAAI,GAAG,CAAC;AAAA,mCAC9B,IAAI,MAAM,kBAAkB,IAAI,IAAI;AAAA,UAC7D,aAAa,KAAK,KAAK,aAAa,CAAC,QAAQ,IAAI,GAAG,CAAC;AAAA,mCAC5B,IAAI,MAAM,kBAAkB,IAAI,IAAI;AAAA,QAAA;AAE/D;AAAA,MACF;AAAA,MAEA,KAAK,cAAc;AACjB,cAAM;AAAA,UACJ,IAAI,EAAC,QAAQ,KAAA;AAAA,QAAI,IACf;AACJ,cAAM;AAAA,UACJ,kBAAkB,KAAK,KAAK,eAAe,CAAC;AAAA,mCACnB,MAAM,kBAAkB,IAAI;AAAA,UACrD,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAAA,mCACjB,MAAM,kBAAkB,IAAI;AAAA,QAAA;AAEvD;AAAA,MACF;AAAA,MAEA,KAAK,cAAc;AACjB,cAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;AACjD,YAAI,eAAe;AACjB,gBAAM,KAAK,KAAK,yBAAyB,KAAK,OAAO,aAAa,CAAC;AAAA,QACrE;AACA,YAAI,UAAU;AACZ,gBAAM;AAAA,YACJ,KAAK,0BAA0B,KAAK,OAAO,OAAO,MAAM,QAAQ;AAAA,UAAA;AAAA,QAEpE;AACA;AAAA,MACF;AAAA,MAEA,KAAK,iBAAiB;AACpB,cAAM;AAAA,UACJ,OAAO,EAAC,QAAQ,MAAM,MAAA;AAAA,UACtB,KAAK,EAAC,MAAM,QAAA;AAAA,UACZ,KAAK,EAAC,MAAM,QAAA;AAAA,QAAO,IACjB;AACJ,YAAI,YAAY,SAAS;AACvB,gBAAM;AAAA,YACJ,aAAa,KAAK,KAAK,aAAa,CAAC,mBAAmB,OAAO;AAAA,mCACxC,MAAM,kBAAkB,KAAK,mBAAmB,OAAO;AAAA,UAAA;AAAA,QAElF;AACA;AAAA,MACF;AAAA,MAEA,KAAK,eAAe;AAClB,cAAM;AAAA,UACJ,OAAO,EAAC,QAAQ,KAAA;AAAA,UAChB;AAAA,QAAA,IACE;AACJ,cAAM;AAAA,UACJ,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAAA,mCACjB,MAAM,kBAAkB,IAAI,mBAAmB,MAAM;AAAA,QAAA;AAEhF;AAAA,MACF;AAAA,MAEA,KAAK,sBAAsB;AACzB,cAAM;AAAA,UACJ,UAAU,EAAC,QAAQ,MAAM,OAAO,OAAA;AAAA,UAChC;AAAA,QAAA,IACE;AACJ,cAAM,OAAO,CAAC,GAAG,OAAO,SAAS,GAAG,OAAO;AAC3C,cAAM;AAAA,UACJ,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAAA,mCACjB,MAAM,kBAAkB,KAAK,oBAAoB,IAAI,IAAI,CAAC;AAAA,QAAA;AAAA,MAEvF;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEA,yBACE,KACA,EAAC,QAAQ,MAAM,MAAA,GACf,UACA;AACA,UAAM,MAAwB,EAAC,QAAQ,OAAO,SAAA;AAC9C,WAAO;AAAA,sBACW,KAAK,KAAK,eAAe,CAAC,IAAI,IAAI,GAAG,CAAC;AAAA;AAAA,0BAElC,IAAI,GAAG,CAAC;AAAA;AAAA,EAEhC;AAAA,EAEA,0BACE,KACA,EAAC,QAAQ,MAAM,MAAA,GACf,QACA,UACA;AACA,UAAM,MAAyB,EAAC,QAAQ,OAAO,QAAQ,SAAA;AACvD,WAAO;AAAA,sBACW,KAAK,KAAK,aAAa,CAAC,IAAI,IAAI,GAAG,CAAC;AAAA;AAAA,0BAEhC,IAAI,GAAG,CAAC;AAAA;AAAA,EAEhC;AAAA,EAEA,OAAO;AACL,SAAK,OAAO,QAAQ,MAAM;AAC1B,WAAO;AAAA,EACT;AACF;AAEA,SAAS,aAAa,OAAuC;AAC3D,QAAM,EAAC,WAAW,OAAA,IAAU;AAC5B,UAAQ,OAAO,KAAA;AAAA,IACb,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,SAAS,QAAQ,EAAC,iBAAiB,UAAA,CAAU,CAAC;AAAA,IACpE,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,UAAU,QAAQ,EAAC,UAAA,CAAU,CAAC;AAAA,IACpD,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,YAAY,MAAM,CAAC;AAAA,IACzC;AACE,aAAO,CAAC,WAAW,CAAC,QAAQ,MAAM,CAAC;AAAA,EAAA;AAEzC;"}
+{"version":3,"file":"storer.js","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {getHeapStatistics} from 'node:v8';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {\n  backfillRequestSchema,\n  isDataChange,\n  isSchemaChange,\n  type BackfillID,\n  type BackfillRequest,\n  type Change,\n  type DataChange,\n  type Identifier,\n  type SchemaChange,\n  type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {type Commit} from '../change-source/protocol/current/downstream.ts';\nimport type {\n  DownstreamStatusMessage,\n  UpstreamStatusMessage,\n} from '../change-source/protocol/current/status.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type ChangeEntry} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n  AutoResetSignal,\n  markResetRequired,\n  type BackfillingColumn,\n  type ReplicationState,\n  type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n  subscriber: Subscriber;\n  mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n  | [\n      'change',\n      watermark: string,\n      json: string,\n      orig: Exclude<Change, DataChange> | null, // null for DataChanges\n    ]\n  | ['ready', callback: () => void]\n  | ['subscriber', SubscriberAndMode]\n  | DownstreamStatusMessage\n  | ['abort']\n  | 'stop';\n\ntype PendingTransaction = {\n  pool: TransactionPool;\n  preCommitWatermark: string;\n  pos: number;\n  startingReplicationState: Promise<ReplicationState>;\n  ack: boolean;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n *   `replicaVersion`, which is the version at which initial sync\n *   was performed, and any attempts to catchup from a different\n *   point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n *   a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n *   subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n  readonly id = 'storer';\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #taskID: string;\n  readonly #discoveryAddress: string;\n  readonly #discoveryProtocol: string;\n  readonly #db: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #onConsumed: (c: Commit | UpstreamStatusMessage) => void;\n  readonly #onFatal: (err: Error) => void;\n  readonly #queue = new Queue<QueueEntry>();\n  readonly #backPressureThresholdBytes: number;\n\n  #approximateQueuedBytes = 0;\n  #running = false;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    db: PostgresDB,\n    replicaVersion: string,\n    onConsumed: (c: Commit | UpstreamStatusMessage) => void,\n    onFatal: (err: Error) => void,\n    backPressureLimitHeapProportion: number,\n  ) {\n    this.#lc = lc.withContext('component', 'change-log');\n    this.#shard = shard;\n    this.#taskID = taskID;\n    this.#discoveryAddress = discoveryAddress;\n    this.#discoveryProtocol = discoveryProtocol;\n    this.#db = db;\n    this.#replicaVersion = replicaVersion;\n    this.#onConsumed = onConsumed;\n    this.#onFatal = onFatal;\n\n    const heapStats = getHeapStatistics();\n    this.#backPressureThresholdBytes =\n      (heapStats.heap_size_limit - heapStats.used_heap_size) *\n      backPressureLimitHeapProportion;\n\n    this.#lc.info?.(\n      `Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of ` +\n        `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) ` +\n        `to absorb upstream spikes`,\n      {heapStats},\n    );\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async assumeOwnership() {\n    const db = this.#db;\n    const owner = this.#taskID;\n    const ownerAddress = this.#discoveryAddress;\n    const ownerProtocol = this.#discoveryProtocol;\n    // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n    const addressWithProtocol =\n      ownerProtocol === 'ws'\n        ? ownerAddress\n        : `${ownerProtocol}://${ownerAddress}`;\n    this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n    const start = performance.now();\n    await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n    const elapsed = (performance.now() - start).toFixed(2);\n    this.#lc.info?.(\n      `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n    );\n  }\n\n  async getStartStreamInitializationParameters(): Promise<{\n    lastWatermark: string;\n    backfillRequests: BackfillRequest[];\n  }> {\n    const [[{lastWatermark}], result] = await runTx(\n      this.#db,\n      sql => [\n        sql<{lastWatermark: string}[]>`\n        SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n        // Formats a BackfillRequest using json_object_agg() to construct the\n        // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n        // to make it optional and possibly `null`.\n        sql`\n        SELECT \n            json_build_object(\n              'schema', b.\"schema\",\n              'name', b.\"table\",\n              'metadata', t.\"metadata\"\n            ) as \"table\",\n            json_object_agg(b.\"column\", b.\"backfill\") \n              as \"columns\"\n          FROM ${this.#cdc('backfilling')} as b\n          LEFT JOIN ${this.#cdc('tableMetadata')} as t\n          ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n          GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n        `,\n      ],\n      {mode: Mode.READONLY},\n    );\n\n    return {\n      lastWatermark,\n      backfillRequests: v.parse(result, backfillRequestsSchema),\n    };\n  }\n\n  async getMinWatermarkForCatchup(): Promise<string | null> {\n    const [{minWatermark}] = await this.#db<\n      {minWatermark: string | null}[]\n    > /*sql*/ `\n      SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n    return minWatermark;\n  }\n\n  purgeRecordsBefore(watermark: string): Promise<number> {\n    return runTx(this.#db, async sql => {\n      const [{deleted}] = await sql<{deleted: bigint}[]>`\n        WITH purged AS (\n          DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n            RETURNING watermark, pos\n        ) SELECT COUNT(*) as deleted FROM purged;`;\n\n      // Before committing the purge, check that this process is still the\n      // owner. This is done after the DELETE to minimize the amount of time\n      // that writes to the changeLog are delayed.\n      const [{owner}] = await sql<ReplicationState[]>`\n        SELECT * FROM ${this.#cdc('replicationState')} FOR SHARE`;\n      if (owner !== this.#taskID) {\n        throw new AbortError(\n          `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n        );\n      }\n      return Number(deleted);\n    });\n  }\n\n  /**\n   * @returns The size of the serialized entry, for memory / I/O estimations.\n   */\n  store(entry: WatermarkedChange) {\n    const [watermark, [_tag, change]] = entry;\n    // Eagerly stringify the JSON object so that the memory usage can be\n    // more accurately measured (i.e. without an extra object traversal and\n    // ad hoc memory counting heuristics).\n    //\n    // This essentially moves the stringify() computation out of the pg client,\n    // which is instead configured to pass `string` objects directly as JSON\n    // strings for JSON-valued columns (see TypeOptions.sendStringAsJson).\n    const json = BigIntJSON.stringify(change);\n    this.#approximateQueuedBytes += json.length;\n\n    this.#queue.enqueue([\n      'change',\n      watermark,\n      json,\n      isDataChange(change) ? null : change, // drop DataChanges to save memory\n    ]);\n\n    return json.length;\n  }\n\n  abort() {\n    this.#queue.enqueue(['abort']);\n  }\n\n  status(s: DownstreamStatusMessage) {\n    this.#queue.enqueue(s);\n  }\n\n  catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n    this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n  }\n\n  #readyForMore: Resolver<void> | null = null;\n\n  readyForMore(): Promise<void> | undefined {\n    if (!this.#running) {\n      return undefined;\n    }\n    if (\n      this.#readyForMore === null &&\n      this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n    ) {\n      this.#lc.warn?.(\n        `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\\n` +\n          `\\n` +\n          `To inspect changeLog backlog in your change DB:\\n` +\n          `  SELECT\\n` +\n          `    (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\\n` +\n          `    change->>'tag' AS operation,\\n` +\n          `    COUNT(*) AS count\\n` +\n          `  FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n          `  GROUP BY 1, 2\\n` +\n          `  ORDER BY 3 DESC\\n` +\n          `  LIMIT 20;`,\n      );\n      this.#readyForMore = resolver();\n    }\n    return this.#readyForMore?.promise;\n  }\n\n  #maybeReleaseBackPressure() {\n    if (\n      this.#readyForMore !== null &&\n      // Wait for at least 20% of the threshold to free up.\n      this.#approximateQueuedBytes < this.#backPressureThresholdBytes * 0.8\n    ) {\n      this.#lc.info?.(\n        `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`,\n      );\n      this.#readyForMore.resolve();\n      this.#readyForMore = null;\n    }\n  }\n\n  #stopped = promiseVoid;\n\n  /**\n   * Runs the storer loop until {@link stop()} is called, or an error is thrown.\n   * Once {@link run()} completes, it can be called again.\n   */\n  async run() {\n    assert(!this.#running, `storer is already running`);\n\n    const {promise: stopped, resolve: signalStopped} = resolver();\n    this.#running = true;\n    this.#stopped = stopped;\n\n    this.#lc.info?.('starting storer');\n    try {\n      await this.#processQueue();\n    } finally {\n      // Release any pending backpressure so the upstream can proceed\n      if (this.#readyForMore !== null) {\n        this.#readyForMore.resolve();\n        this.#readyForMore = null;\n      }\n      const unprocessed = this.#queue.drain();\n      if (unprocessed.length) {\n        this.#lc.warn?.(\n          `dropped ${unprocessed.length} entries from the changeLog queue`,\n        );\n      }\n      this.#running = false;\n      signalStopped();\n      this.#lc.info?.('storer stopped');\n    }\n  }\n\n  async #processQueue() {\n    let tx: PendingTransaction | null = null;\n    let msg: QueueEntry | false;\n\n    const catchupQueue: SubscriberAndMode[] = [];\n    while ((msg = await this.#queue.dequeue()) !== 'stop') {\n      const [msgType] = msg;\n      switch (msgType) {\n        case 'ready': {\n          const signalReady = msg[1];\n          signalReady();\n          continue;\n        }\n        case 'subscriber': {\n          const subscriber = msg[1];\n          if (tx) {\n            catchupQueue.push(subscriber); // Wait for the current tx to complete.\n          } else {\n            await this.#startCatchup([subscriber]); // Catch up immediately.\n          }\n          continue;\n        }\n        case 'status':\n          this.#onConsumed(msg);\n          continue;\n        case 'abort': {\n          if (tx) {\n            tx.pool.abort();\n            await tx.pool.done();\n            tx = null;\n          }\n          continue;\n        }\n      }\n      // msgType === 'change'\n      const [_, watermark, json, change] = msg;\n      const tag = change?.tag;\n      this.#approximateQueuedBytes -= json.length;\n\n      if (tag === 'begin') {\n        assert(!tx, 'received BEGIN in the middle of a transaction');\n        const {promise, resolve, reject} = resolver<ReplicationState>();\n        tx = {\n          pool: new TransactionPool(\n            this.#lc.withContext('watermark', watermark),\n            Mode.READ_COMMITTED,\n          ),\n          preCommitWatermark: watermark,\n          pos: 0,\n          startingReplicationState: promise,\n          ack: !change.skipAck,\n        };\n        tx.pool.run(this.#db);\n        // Acquire a lock on the replicationState row to detect and/or prevent\n        // a concurrent ownership change.\n        void tx.pool.process(tx => {\n          tx<ReplicationState[]> /*sql*/ `\n          SELECT * FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n            ([result]) => resolve(result),\n            reject,\n          );\n          return [];\n        });\n      } else {\n        assert(tx, () => `received change outside of transaction: ${json}`);\n        tx.pos++;\n      }\n\n      const entry = {\n        watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n        precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n        pos: tx.pos,\n        change: json,\n      };\n\n      const processed = tx.pool.process(sql => [\n        sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n        ...(change !== null && isSchemaChange(change)\n          ? this.#trackBackfillMetadata(sql, change)\n          : []),\n      ]);\n\n      if (tx.pos % 100 === 0) {\n        // Backpressure is exerted on commit when awaiting tx.pool.done().\n        // However, backpressure checks need to be regularly done for\n        // very large transactions in order to avoid memory blowup.\n        await processed;\n      }\n      this.#maybeReleaseBackPressure();\n\n      if (tag === 'commit') {\n        const {owner} = await tx.startingReplicationState;\n        if (owner !== this.#taskID) {\n          // Ownership change reflected in the replicationState read in 'begin'.\n          tx.pool.fail(\n            new AbortError(`changeLog ownership has been assumed by ${owner}`),\n          );\n        } else {\n          // Update the replication state.\n          const lastWatermark = watermark;\n          void tx.pool.process(tx => [\n            tx`\n            UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n          ]);\n          tx.pool.setDone();\n        }\n\n        await tx.pool.done();\n\n        // ACK the LSN to the upstream Postgres.\n        if (tx.ack) {\n          this.#onConsumed(['commit', change, {watermark}]);\n        }\n        tx = null;\n\n        // Before beginning the next transaction, open a READONLY snapshot to\n        // concurrently catchup any queued subscribers.\n        await this.#startCatchup(catchupQueue.splice(0));\n      } else if (tag === 'rollback') {\n        // Aborted transactions are not stored in the changeLog. Abort the current tx\n        // and process catchup of subscribers that were waiting for it to end.\n        tx.pool.abort();\n        await tx.pool.done();\n        tx = null;\n\n        await this.#startCatchup(catchupQueue.splice(0));\n      }\n    }\n  }\n\n  async #startCatchup(subs: SubscriberAndMode[]) {\n    if (subs.length === 0) {\n      return;\n    }\n\n    const reader = new TransactionPool(\n      this.#lc.withContext('pool', 'catchup'),\n      Mode.READONLY,\n    );\n    reader.run(this.#db);\n\n    // Ensure that the transaction has started (and is thus holding a snapshot\n    // of the database) before continuing on to commit more changes. This is\n    // done by performing a single read on the db, which determines the\n    // snapshot for the REPEATABLE_READ transaction.\n    const [{lastWatermark}] = await reader.processReadTask(\n      sql => sql<ReplicationState[]>`\n        SELECT * FROM ${this.#cdc('replicationState')}\n      `,\n    );\n\n    // Run the actual catchup queries in the background. Errors are handled in\n    // #catchup() by disconnecting the associated subscriber.\n    void Promise.all(\n      subs.map(sub => this.#catchup(sub, lastWatermark, reader)),\n    ).finally(() => reader.setDone());\n  }\n\n  async #catchup(\n    {subscriber: sub, mode}: SubscriberAndMode,\n    lastWatermark: string,\n    reader: TransactionPool,\n  ) {\n    try {\n      await reader.processReadTask(async tx => {\n        const start = Date.now();\n\n        // When starting from initial-sync, there won't be a change with a watermark\n        // equal to the replica version. This is the empty changeLog scenario.\n        let watermarkFound = sub.watermark === this.#replicaVersion;\n        let count = 0;\n        let lastBatchConsumed: Promise<unknown> | undefined;\n\n        for await (const entries of tx<ChangeEntry[]> /*sql*/ `\n          SELECT watermark, change FROM ${this.#cdc('changeLog')}\n           WHERE watermark >= ${sub.watermark}\n             AND watermark <= ${lastWatermark}\n           ORDER BY watermark, pos`.cursor(2000)) {\n          // Wait for the last batch of entries to be consumed by the\n          // subscriber before sending down the current batch. This pipelining\n          // allows one batch of changes to be received from the change-db\n          // while the previous batch of changes are sent to the subscriber,\n          // resulting in flow control that caps the number of changes\n          // referenced in memory to 2 * batch-size.\n          const start = performance.now();\n          await lastBatchConsumed;\n          const elapsed = performance.now() - start;\n          if (lastBatchConsumed) {\n            (elapsed > 100 ? this.#lc.info : this.#lc.debug)?.(\n              `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n            );\n          }\n\n          for (const entry of entries) {\n            if (entry.watermark === sub.watermark) {\n              // This should be the first entry.\n              // Catchup starts from *after* the watermark.\n              watermarkFound = true;\n            } else if (watermarkFound) {\n              lastBatchConsumed = sub.catchup(toDownstream(entry));\n              count++;\n            } else if (mode === 'backup') {\n              throw new AutoResetSignal(\n                `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n              );\n            } else {\n              this.#lc.warn?.(\n                `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n              );\n              sub.close(\n                ErrorType.WatermarkTooOld,\n                `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n              );\n              return;\n            }\n          }\n        }\n        if (watermarkFound) {\n          await lastBatchConsumed;\n          this.#lc.info?.(\n            `caught up ${sub.id} with ${count} changes (${\n              Date.now() - start\n            } ms)`,\n          );\n        } else {\n          this.#lc.warn?.(\n            `subscriber at watermark ${sub.watermark} is ahead of latest watermark`,\n          );\n        }\n        // Flushes the backlog of messages buffered during catchup and\n        // allows the subscription to forward subsequent messages immediately.\n        sub.setCaughtUp();\n      });\n    } catch (err) {\n      this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n      if (err instanceof AutoResetSignal) {\n        await markResetRequired(this.#db, this.#shard);\n        this.#onFatal(err);\n      }\n      sub.fail(err);\n    }\n  }\n\n  /**\n   * Returns the db statements necessary to track backfill and table metadata\n   * presented in the `change`, if any.\n   */\n  #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n    const stmts: PendingQuery<Row[]>[] = [];\n\n    switch (change.tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        stmts.push(this.#upsertTableMetadataStmt(sql, table, metadata));\n        break;\n      }\n\n      case 'create-table': {\n        const {spec, metadata, backfill} = change;\n        if (metadata) {\n          stmts.push(this.#upsertTableMetadataStmt(sql, spec, metadata));\n        }\n        if (backfill) {\n          Object.entries(backfill).forEach(([col, backfill]) => {\n            stmts.push(\n              this.#upsertColumnBackfillStmt(sql, spec, col, backfill),\n            );\n          });\n        }\n        break;\n      }\n\n      case 'rename-table': {\n        const {old} = change;\n        const row = {schema: change.new.schema, table: change.new.name};\n        stmts.push(\n          sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n          sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n        );\n        break;\n      }\n\n      case 'drop-table': {\n        const {\n          id: {schema, name},\n        } = change;\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('tableMetadata')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n        );\n        break;\n      }\n\n      case 'add-column': {\n        const {table, tableMetadata, column, backfill} = change;\n        if (tableMetadata) {\n          stmts.push(this.#upsertTableMetadataStmt(sql, table, tableMetadata));\n        }\n        if (backfill) {\n          stmts.push(\n            this.#upsertColumnBackfillStmt(sql, table, column.name, backfill),\n          );\n        }\n        break;\n      }\n\n      case 'update-column': {\n        const {\n          table: {schema, name: table},\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          stmts.push(\n            sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${newName}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${oldName}`,\n          );\n        }\n        break;\n      }\n\n      case 'drop-column': {\n        const {\n          table: {schema, name},\n          column,\n        } = change;\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${column}`,\n        );\n        break;\n      }\n\n      case 'backfill-completed': {\n        const {\n          relation: {schema, name: table, rowKey},\n          columns,\n        } = change;\n        const cols = [...rowKey.columns, ...columns];\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql(cols)}`,\n        );\n      }\n    }\n    return stmts;\n  }\n\n  #upsertTableMetadataStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    metadata: TableMetadata,\n  ) {\n    const row: TableMetadataRow = {schema, table, metadata};\n    return sql`\n        INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\") \n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  #upsertColumnBackfillStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    column: string,\n    backfill: BackfillID,\n  ) {\n    const row: BackfillingColumn = {schema, table, column, backfill};\n    return sql`\n        INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\", \"column\") \n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  /**\n   * Waits until all currently queued entries have been processed.\n   * This is only used in tests.\n   */\n  async allProcessed() {\n    if (this.#running) {\n      const {promise, resolve} = resolver();\n      this.#queue.enqueue(['ready', resolve]);\n      await promise;\n    }\n  }\n\n  stop() {\n    if (this.#running) {\n      this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n      this.#queue.enqueue('stop');\n    }\n    return this.#stopped;\n  }\n}\n\nfunction toDownstream(entry: ChangeEntry): WatermarkedChange {\n  const {watermark, change} = entry;\n  switch (change.tag) {\n    case 'begin':\n      return [watermark, ['begin', change, {commitWatermark: watermark}]];\n    case 'commit':\n      return [watermark, ['commit', change, {watermark}]];\n    case 'rollback':\n      return [watermark, ['rollback', change]];\n    default:\n      return [watermark, ['data', change]];\n  }\n}\n"],"names":["v.array","Mode.READONLY","v.parse","Mode.READ_COMMITTED","tx","start","ErrorType.WatermarkTooOld","backfill"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAwEA,MAAM,yBAAyBA,MAAQ,qBAAqB;AAgCrD,MAAM,OAA0B;AAAA,EAC5B,KAAK;AAAA,EACL;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS,IAAI,MAAA;AAAA,EACb;AAAA,EAET,0BAA0B;AAAA,EAC1B,WAAW;AAAA,EAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,YACA,SACA,iCACA;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,YAAY;AACnD,SAAK,SAAS;AACd,SAAK,UAAU;AACf,SAAK,oBAAoB;AACzB,SAAK,qBAAqB;AAC1B,SAAK,MAAM;AACX,SAAK,kBAAkB;AACvB,SAAK,cAAc;AACnB,SAAK,WAAW;AAEhB,UAAM,YAAY,kBAAA;AAClB,SAAK,+BACF,UAAU,kBAAkB,UAAU,kBACvC;AAEF,SAAK,IAAI;AAAA,MACP,gBAAgB,KAAK,8BAA8B,QAAQ,GAAG,QAAQ,CAAC,CAAC,kCAC3C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,CAAC,CAAC;AAAA,MAE9E,EAAC,UAAA;AAAA,IAAS;AAAA,EAEd;AAAA;AAAA,EAGA,KAAK,OAAe;AAClB,WAAO,KAAK,IAAI,GAAG,UAAU,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE;AAAA,EACtD;AAAA,EAEA,MAAM,kBAAkB;AACtB,UAAM,KAAK,KAAK;AAChB,UAAM,QAAQ,KAAK;AACnB,UAAM,eAAe,KAAK;AAC1B,UAAM,gBAAgB,KAAK;AAE3B,UAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,aAAa,MAAM,YAAY;AACxC,SAAK,IAAI,OAAO,yBAAyB,mBAAmB,EAAE;AAC9D,UAAM,QAAQ,YAAY,IAAA;AAC1B,UAAM,YAAY,KAAK,KAAK,kBAAkB,CAAC,QAAQ,GAAG,EAAC,OAAO,cAAc,oBAAA,CAAoB,CAAC;AACrG,UAAM,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACrD,SAAK,IAAI;AAAA,MACP,wBAAwB,mBAAmB,KAAK,OAAO;AAAA,IAAA;AAAA,EAE3D;AAAA,EAEA,MAAM,yCAGH;AACD,UAAM,CAAC,CAAC,EAAC,cAAA,CAAc,GAAG,MAAM,IAAI,MAAM;AAAA,MACxC,KAAK;AAAA,MACL,CAAA,QAAO;AAAA,QACL;AAAA,sCAC8B,KAAK,KAAK,kBAAkB,CAAC;AAAA;AAAA;AAAA;AAAA,QAK3D;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBASS,KAAK,KAAK,aAAa,CAAC;AAAA,sBACnB,KAAK,KAAK,eAAe,CAAC;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,MAK1C,EAAC,MAAMC,SAAK;AAAA,IAAQ;AAGtB,WAAO;AAAA,MACL;AAAA,MACA,kBAAkBC,MAAQ,QAAQ,sBAAsB;AAAA,IAAA;AAAA,EAE5D;AAAA,EAEA,MAAM,4BAAoD;AACxD,UAAM,CAAC,EAAC,aAAA,CAAa,IAAI,MAAM,KAAK;AAAA,qDAGa,KAAK,KAAK,WAAW,CAAC;AACvE,WAAO;AAAA,EACT;AAAA,EAEA,mBAAmB,WAAoC;AACrD,WAAO,MAAM,KAAK,KAAK,OAAM,QAAO;AAClC,YAAM,CAAC,EAAC,SAAQ,IAAI,MAAM;AAAA;AAAA,wBAER,KAAK,KAAK,WAAW,CAAC,sBAAsB,SAAS;AAAA;AAAA;AAOvE,YAAM,CAAC,EAAC,OAAM,IAAI,MAAM;AAAA,wBACN,KAAK,KAAK,kBAAkB,CAAC;AAC/C,UAAI,UAAU,KAAK,SAAS;AAC1B,cAAM,IAAI;AAAA,UACR,+BAA+B,SAAS,wCAAwC,KAAK;AAAA,QAAA;AAAA,MAEzF;AACA,aAAO,OAAO,OAAO;AAAA,IACvB,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,OAA0B;AAC9B,UAAM,CAAC,WAAW,CAAC,MAAM,MAAM,CAAC,IAAI;AAQpC,UAAM,OAAO,WAAW,UAAU,MAAM;AACxC,SAAK,2BAA2B,KAAK;AAErC,SAAK,OAAO,QAAQ;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,MAAM,IAAI,OAAO;AAAA;AAAA,IAAA,CAC/B;AAED,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,QAAQ;AACN,SAAK,OAAO,QAAQ,CAAC,OAAO,CAAC;AAAA,EAC/B;AAAA,EAEA,OAAO,GAA4B;AACjC,SAAK,OAAO,QAAQ,CAAC;AAAA,EACvB;AAAA,EAEA,QAAQ,YAAwB,MAAsB;AACpD,SAAK,OAAO,QAAQ,CAAC,cAAc,EAAC,YAAY,KAAA,CAAK,CAAC;AAAA,EACxD;AAAA,EAEA,gBAAuC;AAAA,EAEvC,eAA0C;AACxC,QAAI,CAAC,KAAK,UAAU;AAClB,aAAO;AAAA,IACT;AACA,QACE,KAAK,kBAAkB,QACvB,KAAK,0BAA0B,KAAK,6BACpC;AACA,WAAK,IAAI;AAAA,QACP,+BAA+B,KAAK,OAAO,KAAA,CAAM,sBAAsB,KAAK,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA;AAY7H,WAAK,gBAAgB,SAAA;AAAA,IACvB;AACA,WAAO,KAAK,eAAe;AAAA,EAC7B;AAAA,EAEA,4BAA4B;AAC1B,QACE,KAAK,kBAAkB;AAAA,IAEvB,KAAK,0BAA0B,KAAK,8BAA8B,KAClE;AACA,WAAK,IAAI;AAAA,QACP,gCAAgC,KAAK,OAAO,KAAA,CAAM,sBAAsB,KAAK,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,CAAC;AAAA,MAAA;AAE9H,WAAK,cAAc,QAAA;AACnB,WAAK,gBAAgB;AAAA,IACvB;AAAA,EACF;AAAA,EAEA,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA,EAMX,MAAM,MAAM;AACV,WAAO,CAAC,KAAK,UAAU,2BAA2B;AAElD,UAAM,EAAC,SAAS,SAAS,SAAS,cAAA,IAAiB,SAAA;AACnD,SAAK,WAAW;AAChB,SAAK,WAAW;AAEhB,SAAK,IAAI,OAAO,iBAAiB;AACjC,QAAI;AACF,YAAM,KAAK,cAAA;AAAA,IACb,UAAA;AAEE,UAAI,KAAK,kBAAkB,MAAM;AAC/B,aAAK,cAAc,QAAA;AACnB,aAAK,gBAAgB;AAAA,MACvB;AACA,YAAM,cAAc,KAAK,OAAO,MAAA;AAChC,UAAI,YAAY,QAAQ;AACtB,aAAK,IAAI;AAAA,UACP,WAAW,YAAY,MAAM;AAAA,QAAA;AAAA,MAEjC;AACA,WAAK,WAAW;AAChB,oBAAA;AACA,WAAK,IAAI,OAAO,gBAAgB;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,MAAM,gBAAgB;AACpB,QAAI,KAAgC;AACpC,QAAI;AAEJ,UAAM,eAAoC,CAAA;AAC1C,YAAQ,MAAM,MAAM,KAAK,OAAO,QAAA,OAAe,QAAQ;AACrD,YAAM,CAAC,OAAO,IAAI;AAClB,cAAQ,SAAA;AAAA,QACN,KAAK,SAAS;AACZ,gBAAM,cAAc,IAAI,CAAC;AACzB,sBAAA;AACA;AAAA,QACF;AAAA,QACA,KAAK,cAAc;AACjB,gBAAM,aAAa,IAAI,CAAC;AACxB,cAAI,IAAI;AACN,yBAAa,KAAK,UAAU;AAAA,UAC9B,OAAO;AACL,kBAAM,KAAK,cAAc,CAAC,UAAU,CAAC;AAAA,UACvC;AACA;AAAA,QACF;AAAA,QACA,KAAK;AACH,eAAK,YAAY,GAAG;AACpB;AAAA,QACF,KAAK,SAAS;AACZ,cAAI,IAAI;AACN,eAAG,KAAK,MAAA;AACR,kBAAM,GAAG,KAAK,KAAA;AACd,iBAAK;AAAA,UACP;AACA;AAAA,QACF;AAAA,MAAA;AAGF,YAAM,CAAC,GAAG,WAAW,MAAM,MAAM,IAAI;AACrC,YAAM,MAAM,QAAQ;AACpB,WAAK,2BAA2B,KAAK;AAErC,UAAI,QAAQ,SAAS;AACnB,eAAO,CAAC,IAAI,+CAA+C;AAC3D,cAAM,EAAC,SAAS,SAAS,OAAA,IAAU,SAAA;AACnC,aAAK;AAAA,UACH,MAAM,IAAI;AAAA,YACR,KAAK,IAAI,YAAY,aAAa,SAAS;AAAA,YAC3CC;AAAAA,UAAK;AAAA,UAEP,oBAAoB;AAAA,UACpB,KAAK;AAAA,UACL,0BAA0B;AAAA,UAC1B,KAAK,CAAC,OAAO;AAAA,QAAA;AAEf,WAAG,KAAK,IAAI,KAAK,GAAG;AAGpB,aAAK,GAAG,KAAK,QAAQ,CAAAC,QAAM;AACzBA;AAAAA,0BACgB,KAAK,KAAK,kBAAkB,CAAC,cAAc;AAAA,YACzD,CAAC,CAAC,MAAM,MAAM,QAAQ,MAAM;AAAA,YAC5B;AAAA,UAAA;AAEF,iBAAO,CAAA;AAAA,QACT,CAAC;AAAA,MACH,OAAO;AACL,eAAO,IAAI,MAAM,2CAA2C,IAAI,EAAE;AAClE,WAAG;AAAA,MACL;AAEA,YAAM,QAAQ;AAAA,QACZ,WAAW,QAAQ,WAAW,YAAY,GAAG;AAAA,QAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;AAAA,QACtD,KAAK,GAAG;AAAA,QACR,QAAQ;AAAA,MAAA;AAGV,YAAM,YAAY,GAAG,KAAK,QAAQ,CAAA,QAAO;AAAA,QACvC,kBAAkB,KAAK,KAAK,WAAW,CAAC,IAAI,IAAI,KAAK,CAAC;AAAA,QACtD,GAAI,WAAW,QAAQ,eAAe,MAAM,IACxC,KAAK,uBAAuB,KAAK,MAAM,IACvC,CAAA;AAAA,MAAC,CACN;AAED,UAAI,GAAG,MAAM,QAAQ,GAAG;AAItB,cAAM;AAAA,MACR;AACA,WAAK,0BAAA;AAEL,UAAI,QAAQ,UAAU;AACpB,cAAM,EAAC,MAAA,IAAS,MAAM,GAAG;AACzB,YAAI,UAAU,KAAK,SAAS;AAE1B,aAAG,KAAK;AAAA,YACN,IAAI,WAAW,2CAA2C,KAAK,EAAE;AAAA,UAAA;AAAA,QAErE,OAAO;AAEL,gBAAM,gBAAgB;AACtB,eAAK,GAAG,KAAK,QAAQ,CAAAA,QAAM;AAAA,YACzBA;AAAAA,qBACS,KAAK,KAAK,kBAAkB,CAAC,QAAQA,IAAG,EAAC,cAAA,CAAc,CAAC;AAAA,UAAA,CAClE;AACD,aAAG,KAAK,QAAA;AAAA,QACV;AAEA,cAAM,GAAG,KAAK,KAAA;AAGd,YAAI,GAAG,KAAK;AACV,eAAK,YAAY,CAAC,UAAU,QAAQ,EAAC,UAAA,CAAU,CAAC;AAAA,QAClD;AACA,aAAK;AAIL,cAAM,KAAK,cAAc,aAAa,OAAO,CAAC,CAAC;AAAA,MACjD,WAAW,QAAQ,YAAY;AAG7B,WAAG,KAAK,MAAA;AACR,cAAM,GAAG,KAAK,KAAA;AACd,aAAK;AAEL,cAAM,KAAK,cAAc,aAAa,OAAO,CAAC,CAAC;AAAA,MACjD;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,MAA2B;AAC7C,QAAI,KAAK,WAAW,GAAG;AACrB;AAAA,IACF;AAEA,UAAM,SAAS,IAAI;AAAA,MACjB,KAAK,IAAI,YAAY,QAAQ,SAAS;AAAA,MACtCH;AAAAA,IAAK;AAEP,WAAO,IAAI,KAAK,GAAG;AAMnB,UAAM,CAAC,EAAC,cAAA,CAAc,IAAI,MAAM,OAAO;AAAA,MACrC,CAAA,QAAO;AAAA,wBACW,KAAK,KAAK,kBAAkB,CAAC;AAAA;AAAA,IAAA;AAMjD,SAAK,QAAQ;AAAA,MACX,KAAK,IAAI,CAAA,QAAO,KAAK,SAAS,KAAK,eAAe,MAAM,CAAC;AAAA,IAAA,EACzD,QAAQ,MAAM,OAAO,SAAS;AAAA,EAClC;AAAA,EAEA,MAAM,SACJ,EAAC,YAAY,KAAK,KAAA,GAClB,eACA,QACA;AACA,QAAI;AACF,YAAM,OAAO,gBAAgB,OAAM,OAAM;AACvC,cAAM,QAAQ,KAAK,IAAA;AAInB,YAAI,iBAAiB,IAAI,cAAc,KAAK;AAC5C,YAAI,QAAQ;AACZ,YAAI;AAEJ,yBAAiB,WAAW;AAAA,0CACM,KAAK,KAAK,WAAW,CAAC;AAAA,gCAChC,IAAI,SAAS;AAAA,gCACb,aAAa;AAAA,oCACT,OAAO,GAAI,GAAG;AAOxC,gBAAMI,SAAQ,YAAY,IAAA;AAC1B,gBAAM;AACN,gBAAM,UAAU,YAAY,IAAA,IAAQA;AACpC,cAAI,mBAAmB;AACrB,aAAC,UAAU,MAAM,KAAK,IAAI,OAAO,KAAK,IAAI;AAAA,cACxC,UAAU,QAAQ,QAAQ,CAAC,CAAC,WAAW,IAAI,EAAE;AAAA,YAAA;AAAA,UAEjD;AAEA,qBAAW,SAAS,SAAS;AAC3B,gBAAI,MAAM,cAAc,IAAI,WAAW;AAGrC,+BAAiB;AAAA,YACnB,WAAW,gBAAgB;AACzB,kCAAoB,IAAI,QAAQ,aAAa,KAAK,CAAC;AACnD;AAAA,YACF,WAAW,SAAS,UAAU;AAC5B,oBAAM,IAAI;AAAA,gBACR,+BAA+B,IAAI,SAAS,yBAAyB,MAAM,SAAS;AAAA,cAAA;AAAA,YAExF,OAAO;AACL,mBAAK,IAAI;AAAA,gBACP,qCAAqC,IAAI,SAAS,yBAAyB,MAAM,SAAS;AAAA,cAAA;AAE5F,kBAAI;AAAA,gBACFC;AAAAA,gBACA,mCAAmC,MAAM,SAAS,eAAe,IAAI,SAAS;AAAA,cAAA;AAEhF;AAAA,YACF;AAAA,UACF;AAAA,QACF;AACA,YAAI,gBAAgB;AAClB,gBAAM;AACN,eAAK,IAAI;AAAA,YACP,aAAa,IAAI,EAAE,SAAS,KAAK,aAC/B,KAAK,QAAQ,KACf;AAAA,UAAA;AAAA,QAEJ,OAAO;AACL,eAAK,IAAI;AAAA,YACP,2BAA2B,IAAI,SAAS;AAAA,UAAA;AAAA,QAE5C;AAGA,YAAI,YAAA;AAAA,MACN,CAAC;AAAA,IACH,SAAS,KAAK;AACZ,WAAK,IAAI,QAAQ,sCAAsC,IAAI,EAAE,IAAI,GAAG;AACpE,UAAI,eAAe,iBAAiB;AAClC,cAAM,kBAAkB,KAAK,KAAK,KAAK,MAAM;AAC7C,aAAK,SAAS,GAAG;AAAA,MACnB;AACA,UAAI,KAAK,GAAG;AAAA,IACd;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,uBAAuB,KAA0B,QAAsB;AACrE,UAAM,QAA+B,CAAA;AAErC,YAAQ,OAAO,KAAA;AAAA,MACb,KAAK,yBAAyB;AAC5B,cAAM,EAAC,OAAO,KAAK,SAAA,IAAY;AAC/B,cAAM,KAAK,KAAK,yBAAyB,KAAK,OAAO,QAAQ,CAAC;AAC9D;AAAA,MACF;AAAA,MAEA,KAAK,gBAAgB;AACnB,cAAM,EAAC,MAAM,UAAU,SAAA,IAAY;AACnC,YAAI,UAAU;AACZ,gBAAM,KAAK,KAAK,yBAAyB,KAAK,MAAM,QAAQ,CAAC;AAAA,QAC/D;AACA,YAAI,UAAU;AACZ,iBAAO,QAAQ,QAAQ,EAAE,QAAQ,CAAC,CAAC,KAAKC,SAAQ,MAAM;AACpD,kBAAM;AAAA,cACJ,KAAK,0BAA0B,KAAK,MAAM,KAAKA,SAAQ;AAAA,YAAA;AAAA,UAE3D,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAAA,MAEA,KAAK,gBAAgB;AACnB,cAAM,EAAC,QAAO;AACd,cAAM,MAAM,EAAC,QAAQ,OAAO,IAAI,QAAQ,OAAO,OAAO,IAAI,KAAA;AAC1D,cAAM;AAAA,UACJ,aAAa,KAAK,KAAK,eAAe,CAAC,QAAQ,IAAI,GAAG,CAAC;AAAA,mCAC9B,IAAI,MAAM,kBAAkB,IAAI,IAAI;AAAA,UAC7D,aAAa,KAAK,KAAK,aAAa,CAAC,QAAQ,IAAI,GAAG,CAAC;AAAA,mCAC5B,IAAI,MAAM,kBAAkB,IAAI,IAAI;AAAA,QAAA;AAE/D;AAAA,MACF;AAAA,MAEA,KAAK,cAAc;AACjB,cAAM;AAAA,UACJ,IAAI,EAAC,QAAQ,KAAA;AAAA,QAAI,IACf;AACJ,cAAM;AAAA,UACJ,kBAAkB,KAAK,KAAK,eAAe,CAAC;AAAA,mCACnB,MAAM,kBAAkB,IAAI;AAAA,UACrD,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAAA,mCACjB,MAAM,kBAAkB,IAAI;AAAA,QAAA;AAEvD;AAAA,MACF;AAAA,MAEA,KAAK,cAAc;AACjB,cAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;AACjD,YAAI,eAAe;AACjB,gBAAM,KAAK,KAAK,yBAAyB,KAAK,OAAO,aAAa,CAAC;AAAA,QACrE;AACA,YAAI,UAAU;AACZ,gBAAM;AAAA,YACJ,KAAK,0BAA0B,KAAK,OAAO,OAAO,MAAM,QAAQ;AAAA,UAAA;AAAA,QAEpE;AACA;AAAA,MACF;AAAA,MAEA,KAAK,iBAAiB;AACpB,cAAM;AAAA,UACJ,OAAO,EAAC,QAAQ,MAAM,MAAA;AAAA,UACtB,KAAK,EAAC,MAAM,QAAA;AAAA,UACZ,KAAK,EAAC,MAAM,QAAA;AAAA,QAAO,IACjB;AACJ,YAAI,YAAY,SAAS;AACvB,gBAAM;AAAA,YACJ,aAAa,KAAK,KAAK,aAAa,CAAC,mBAAmB,OAAO;AAAA,mCACxC,MAAM,kBAAkB,KAAK,mBAAmB,OAAO;AAAA,UAAA;AAAA,QAElF;AACA;AAAA,MACF;AAAA,MAEA,KAAK,eAAe;AAClB,cAAM;AAAA,UACJ,OAAO,EAAC,QAAQ,KAAA;AAAA,UAChB;AAAA,QAAA,IACE;AACJ,cAAM;AAAA,UACJ,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAAA,mCACjB,MAAM,kBAAkB,IAAI,mBAAmB,MAAM;AAAA,QAAA;AAEhF;AAAA,MACF;AAAA,MAEA,KAAK,sBAAsB;AACzB,cAAM;AAAA,UACJ,UAAU,EAAC,QAAQ,MAAM,OAAO,OAAA;AAAA,UAChC;AAAA,QAAA,IACE;AACJ,cAAM,OAAO,CAAC,GAAG,OAAO,SAAS,GAAG,OAAO;AAC3C,cAAM;AAAA,UACJ,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAAA,mCACjB,MAAM,kBAAkB,KAAK,oBAAoB,IAAI,IAAI,CAAC;AAAA,QAAA;AAAA,MAEvF;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEA,yBACE,KACA,EAAC,QAAQ,MAAM,MAAA,GACf,UACA;AACA,UAAM,MAAwB,EAAC,QAAQ,OAAO,SAAA;AAC9C,WAAO;AAAA,sBACW,KAAK,KAAK,eAAe,CAAC,IAAI,IAAI,GAAG,CAAC;AAAA;AAAA,0BAElC,IAAI,GAAG,CAAC;AAAA;AAAA,EAEhC;AAAA,EAEA,0BACE,KACA,EAAC,QAAQ,MAAM,MAAA,GACf,QACA,UACA;AACA,UAAM,MAAyB,EAAC,QAAQ,OAAO,QAAQ,SAAA;AACvD,WAAO;AAAA,sBACW,KAAK,KAAK,aAAa,CAAC,IAAI,IAAI,GAAG,CAAC;AAAA;AAAA,0BAEhC,IAAI,GAAG,CAAC;AAAA;AAAA,EAEhC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,eAAe;AACnB,QAAI,KAAK,UAAU;AACjB,YAAM,EAAC,SAAS,QAAA,IAAW,SAAA;AAC3B,WAAK,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;AACtC,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,OAAO;AACL,QAAI,KAAK,UAAU;AACjB,WAAK,IAAI,OAAO,YAAY,KAAK,OAAO,MAAM,oBAAoB;AAClE,WAAK,OAAO,QAAQ,MAAM;AAAA,IAC5B;AACA,WAAO,KAAK;AAAA,EACd;AACF;AAEA,SAAS,aAAa,OAAuC;AAC3D,QAAM,EAAC,WAAW,OAAA,IAAU;AAC5B,UAAQ,OAAO,KAAA;AAAA,IACb,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,SAAS,QAAQ,EAAC,iBAAiB,UAAA,CAAU,CAAC;AAAA,IACpE,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,UAAU,QAAQ,EAAC,UAAA,CAAU,CAAC;AAAA,IACpD,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,YAAY,MAAM,CAAC;AAAA,IACzC;AACE,aAAO,CAAC,WAAW,CAAC,QAAQ,MAAM,CAAC;AAAA,EAAA;AAEzC;"}

package/out/zero-cache/src/types/streams.d.ts

@@ -9,8 +9,10 @@ export type Source<T> = AsyncIterable<T> & {
      * Immediately terminates all current iterations (i.e. {@link AsyncIterator.next next()})
      * will return `{value: undefined, done: true}`), and prevents any subsequent iterations
      * from yielding any values.
+     *
+     * @param err Terminate the iteration by throwing the `err` instead.
      */
-    cancel: () => void;
+    cancel: (err?: Error) => void;
     /**
      * The presence of a `pipeline` iterable allows the usual "consumed-on-iterate" semantics
      * to be overridden.

package/out/zero-cache/src/types/streams.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/streams.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAEL,QAAQ,EAGR,KAAK,aAAa,EACnB,MAAM,aAAa,CAAC;AACrB,OAAO,EAKL,KAAK,SAAS,EACf,MAAM,IAAI,CAAC;AAEZ,OAAO,EAAa,KAAK,SAAS,EAAC,MAAM,oCAAoC,CAAC;AAE9E,OAAO,KAAK,CAAC,MAAM,+BAA+B,CAAC;AACnD,OAAO,EAAC,YAAY,EAAE,KAAK,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAW7D,MAAM,MAAM,MAAM,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG;IACzC;;;;OAIG;IACH,MAAM,EAAE,MAAM,IAAI,CAAC;IAEnB;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,CAAC,CAAC;QAAC,QAAQ,EAAE,MAAM,IAAI,CAAA;KAAC,CAAC,GAAG,SAAS,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,IAAI,CAAC,CAAC,IAAI;IACpB,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC;CACxB,CAAC;AAEF;;;GAGG;AAIH,wBAAgB,MAAM,CAAC,EAAE,SAAS,SAAS,EAAE,GAAG,SAAS,SAAS,EAChE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EACpB,UAAU,GAAE,OAAO,CAAC,GAAG,CAAM,EAC7B,SAAS,GAAE,OAAO,CAAC,EAAE,CAAM,EAC3B,aAAa,GAAE,aAAkB,GAChC;IAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,CAAA;CAAC,CAwE9C;AAED,KAAK,WAAW,CAAC,CAAC,IAAI;IACpB,MAAM,EAAE,QAAQ,CAAC;IACjB,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;IACtB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC;IACpC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,wBAAgB,IAAI,CAAC,CAAC,EAAE,EAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,cAAc,EAAC,EAAE,WAAW,CAAC,CAAC,CAAC,QAsD5E;AAkBD,wBAAsB,SAAS,CAAC,CAAC,SAAS,SAAS,EACjD,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EACjB,IAAI,EAAE,SAAS,GACd,OAAO,CAAC,IAAI,CAAC,CA0Df;AAED,wBAAsB,QAAQ,CAAC,CAAC,SAAS,SAAS,EAChD,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAChB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAqCpB"}
+{"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/streams.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAEL,QAAQ,EAGR,KAAK,aAAa,EACnB,MAAM,aAAa,CAAC;AACrB,OAAO,EAKL,KAAK,SAAS,EACf,MAAM,IAAI,CAAC;AAEZ,OAAO,EAAa,KAAK,SAAS,EAAC,MAAM,oCAAoC,CAAC;AAE9E,OAAO,KAAK,CAAC,MAAM,+BAA+B,CAAC;AACnD,OAAO,EAAC,YAAY,EAAE,KAAK,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAW7D,MAAM,MAAM,MAAM,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG;IACzC;;;;;;OAMG;IACH,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC;IAE9B;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,CAAC,CAAC;QAAC,QAAQ,EAAE,MAAM,IAAI,CAAA;KAAC,CAAC,GAAG,SAAS,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,IAAI,CAAC,CAAC,IAAI;IACpB,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC;CACxB,CAAC;AAEF;;;GAGG;AAIH,wBAAgB,MAAM,CAAC,EAAE,SAAS,SAAS,EAAE,GAAG,SAAS,SAAS,EAChE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EACpB,UAAU,GAAE,OAAO,CAAC,GAAG,CAAM,EAC7B,SAAS,GAAE,OAAO,CAAC,EAAE,CAAM,EAC3B,aAAa,GAAE,aAAkB,GAChC;IAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,CAAA;CAAC,CAwE9C;AAED,KAAK,WAAW,CAAC,CAAC,IAAI;IACpB,MAAM,EAAE,QAAQ,CAAC;IACjB,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;IACtB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC;IACpC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,wBAAgB,IAAI,CAAC,CAAC,EAAE,EAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,cAAc,EAAC,EAAE,WAAW,CAAC,CAAC,CAAC,QAsD5E;AAkBD,wBAAsB,SAAS,CAAC,CAAC,SAAS,SAAS,EACjD,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EACjB,IAAI,EAAE,SAAS,GACd,OAAO,CAAC,IAAI,CAAC,CA0Df;AAED,wBAAsB,QAAQ,CAAC,CAAC,SAAS,SAAS,EAChD,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAChB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAqCpB"}

package/out/zero-cache/src/types/streams.js.map

@@ -1 +1 @@
-{"version":3,"file":"streams.js","sources":["../../../../../zero-cache/src/types/streams.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {\n  pipeline,\n  Readable,\n  Transform,\n  Writable,\n  type DuplexOptions,\n} from 'node:stream';\nimport {\n  createWebSocketStream,\n  type CloseEvent,\n  type ErrorEvent,\n  type MessageEvent,\n  type WebSocket,\n} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {BigIntJSON, type JSONValue} from '../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../shared/src/queue.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Subscription, type Options} from './subscription.ts';\nimport {\n  closeWithError,\n  expectPingsForLiveness,\n  sendPingsForLiveness,\n} from './ws.ts';\n\n// Consistent with Postgres keepalives, and shorter than the\n// commonly used default idle timeout of 1 minute.\nconst PING_INTERVAL_MS = 30_000;\n\nexport type Source<T> = AsyncIterable<T> & {\n  /**\n   * Immediately terminates all current iterations (i.e. {@link AsyncIterator.next next()})\n   * will return `{value: undefined, done: true}`), and prevents any subsequent iterations\n   * from yielding any values.\n   */\n  cancel: () => void;\n\n  /**\n   * The presence of a `pipeline` iterable allows the usual \"consumed-on-iterate\" semantics\n   * to be overridden.\n   *\n   * This is suitable for transport layers that serialize messages across processes, such\n   * as the {@link streamOut()} method; pipelining allows the transport to send messages\n   * as they arrive without waiting for the previous message to be acked, streaming\n   * them to the receiving process where they are presumably queued and processed without\n   * a per-message ack delay. The receiving end of the transport then responds with acks\n   * asynchronously as the receiving end processes the messages.\n   */\n  pipeline?: AsyncIterable<{value: T; consumed: () => void}> | undefined;\n};\n\nexport type Sink<T> = {\n  push(message: T): void;\n};\n\n/**\n * Back-pressure-aware transformation of a WebSocket into\n * upstream and downstream {@link Subscription} objects.\n */\n// TODO: Change {@link streamIn} and {@link streamOut} to use this\n//       under the covers so that internal communication is also\n//       responsive to backpressure.\nexport function stream<In extends JSONValue, Out extends JSONValue>(\n  lc: LogContext,\n  ws: WebSocket,\n  inSchema: v.Type<In>,\n  outOptions: Options<Out> = {},\n  inOptions: Options<In> = {},\n  streamOptions: DuplexOptions = {},\n): {outstream: Sink<Out>; instream: Source<In>} {\n  const endpoint = ws.url ?? 'client';\n  function close(err?: unknown) {\n    if (ws.readyState !== ws.CLOSED && ws.readyState !== ws.CLOSING) {\n      if (err) {\n        closeWithError(lc, ws, err);\n      } else {\n        lc.info?.(`closing connection to ${endpoint}`);\n        ws.close();\n      }\n    }\n  }\n\n  const instream = Subscription.create<In>({\n    ...inOptions,\n    cleanup: (unconsumed, err) => {\n      inOptions.cleanup?.(unconsumed, err);\n      close(err);\n    },\n  });\n  const outstream = Subscription.create<Out>({\n    ...outOptions,\n    cleanup: (unconsumed, err) => {\n      outOptions.cleanup?.(unconsumed, err);\n      close(err);\n    },\n  });\n\n  const duplex = createWebSocketStream(ws, {\n    ...streamOptions,\n    decodeStrings: false,\n  });\n\n  // Outgoing transform.\n  function streamOut() {\n    // Mainly used for verifying that back-pressure kicks in tests.\n    duplex.on('drain', () => lc.debug?.(`drained messages to ${endpoint}`));\n\n    pipeline(\n      Readable.from(outstream),\n      new Transform({\n        objectMode: true,\n        transform: (msg, _encoding, callback) =>\n          callback(null, BigIntJSON.stringify(msg)),\n      }),\n      duplex,\n      err => (err ? outstream.fail(err) : outstream.cancel()),\n    );\n  }\n\n  if (ws.readyState === ws.CONNECTING) {\n    ws.on('open', () => {\n      lc.info?.(`connected to ${endpoint}`);\n      streamOut();\n    });\n  } else {\n    streamOut();\n  }\n\n  // Incoming transform.\n  pipe({\n    source: duplex,\n    sink: instream,\n    parse: chunk => {\n      const json = BigIntJSON.parse(chunk.toString());\n      return v.parse(json, inSchema, 'passthrough');\n    },\n  });\n\n  sendPingsForLiveness(lc, ws, PING_INTERVAL_MS);\n\n  return {outstream, instream};\n}\n\ntype PipeOptions<T> = {\n  source: Readable;\n  sink: Subscription<T>;\n  parse: (buffer: Buffer) => T | null;\n  bufferMessages?: number;\n};\n\nexport function pipe<T>({source, sink, parse, bufferMessages}: PipeOptions<T>) {\n  bufferMessages ??= 0;\n  assert(bufferMessages >= 0);\n  const pending: Promise<unknown>[] = [];\n\n  pipeline(\n    source,\n    new Writable({\n      decodeStrings: false,\n      write: (chunk, _encoding, callback) => {\n        let msg: T | null;\n        try {\n          if ((msg = parse(chunk)) === null) {\n            callback();\n            return;\n          }\n        } catch (err) {\n          callback(ensureError(err));\n          return;\n        }\n        // Inbound backpressure is exerted by unconsumed messages in the\n        // subscription. A buffer can be used to allow messages to queue up in\n        // in the Subscription object, which allows the consumer to \"peek\" at\n        // whether there are more messages immediately available\n        // (via {@link Subscription.queued}.\n        const {result} = sink.push(msg);\n        pending.push(result);\n        void result.then(() => pending.shift());\n\n        if (pending.length <= bufferMessages) {\n          // immediately allow more messages\n          callback();\n        } else {\n          // wait for the oldest result in the pending queue\n          pending[0].then(\n            () => callback(),\n            err => callback(ensureError(err)),\n          );\n        }\n      },\n      destroy: (err, callback) => {\n        if (err) {\n          sink.fail(ensureError(err));\n        }\n        // Otherwise, final will handle the cancel.\n        callback();\n      },\n      final: callback => {\n        sink.cancel();\n        callback();\n      },\n    }),\n    err => (err ? sink.fail(err) : sink.cancel()),\n  );\n}\n\nfunction ensureError(err: unknown) {\n  return err instanceof Error ? err : new Error(String(err));\n}\n\nconst ackSchema = v.object({ack: v.number()});\n\ntype Ack = v.Infer<typeof ackSchema>;\n\ntype Streamed<T> = {\n  /** Application-level message. */\n  msg: T;\n\n  /** ID used for the Ack message. */\n  id: number;\n};\n\nexport async function streamOut<T extends JSONValue>(\n  lc: LogContext,\n  source: Source<T>,\n  sink: WebSocket,\n): Promise<void> {\n  sendPingsForLiveness(lc, sink, PING_INTERVAL_MS);\n\n  const closer = WebSocketCloser.forSource(lc, sink, source);\n\n  const acks = new Queue<Ack>();\n  sink.addEventListener('message', ({data}) => {\n    try {\n      if (typeof data !== 'string') {\n        throw new Error('Expected string message');\n      }\n      acks.enqueue(v.parse(JSON.parse(data), ackSchema));\n    } catch (e) {\n      lc.error?.(`error parsing ack`, e);\n      closer.close(e);\n    }\n  });\n\n  try {\n    let nextID = 0;\n    const {pipeline} = source;\n    if (pipeline) {\n      lc.debug?.(`started pipelined outbound stream`);\n      for await (const {value: msg, consumed} of pipeline) {\n        const id = ++nextID;\n        const data = BigIntJSON.stringify({msg, id} satisfies Streamed<T>);\n        // Enable for debugging. Otherwise too verbose.\n        // lc.debug?.(`pipelining`, data);\n        sink.send(data);\n\n        void (async () => {\n          const {ack} = await acks.dequeue();\n          // lc.debug?.(`received ack`, ack);\n          if (ack !== id) {\n            throw new Error(`Unexpected ack for ${id}: ${ack}`);\n          }\n          consumed();\n        })();\n      }\n    } else {\n      lc.debug?.(`started synchronous outbound stream`);\n      for await (const msg of source) {\n        const id = ++nextID;\n        const data = BigIntJSON.stringify({msg, id} satisfies Streamed<T>);\n        // Enable for debugging. Otherwise too verbose.\n        // lc.debug?.(`sending`, data);\n        sink.send(data);\n\n        const {ack} = await acks.dequeue();\n        if (ack !== id) {\n          throw new Error(`Unexpected ack for ${id}: ${ack}`);\n        }\n      }\n    }\n    closer.close();\n  } catch (e) {\n    closer.close(e);\n  }\n}\n\nexport async function streamIn<T extends JSONValue>(\n  lc: LogContext,\n  source: WebSocket,\n  schema: v.Type<T>,\n): Promise<Source<T>> {\n  expectPingsForLiveness(lc, source, PING_INTERVAL_MS);\n\n  const streamedSchema = v.object({\n    msg: schema,\n    id: v.number(),\n  });\n\n  const sink: Subscription<T, Streamed<T>> = new Subscription<T, Streamed<T>>(\n    {\n      consumed: ({id}) => source.send(JSON.stringify({ack: id} satisfies Ack)),\n      cleanup: () => closer.close(),\n    },\n    ({msg}) => msg,\n  );\n\n  const closer = WebSocketCloser.forSink(lc, source, sink, handleMessage);\n\n  function handleMessage(event: MessageEvent) {\n    const data = event.data.toString();\n    if (!sink.active) {\n      lc.warn?.('dropping ws message received after close', data);\n      return;\n    }\n    try {\n      const value = BigIntJSON.parse(data);\n      const msg = v.parse(value, streamedSchema, 'passthrough');\n      // Enable for debugging. Otherwise too verbose.\n      // lc.debug?.(`received`, data);\n      sink.push(msg);\n    } catch (e) {\n      closer.close(e);\n    }\n  }\n\n  await closer.connected;\n  return sink;\n}\n\nclass WebSocketCloser {\n  readonly #lc: LogContext;\n  readonly #ws: WebSocket;\n  readonly #closeStream: () => void;\n  readonly #messageHandler: ((e: MessageEvent) => void | undefined) | null;\n  readonly #connected = resolver();\n\n  get connected(): Promise<void> {\n    return this.#connected.promise;\n  }\n\n  static forSource<T>(lc: LogContext, ws: WebSocket, stream: Source<T>) {\n    // If the websocket is closed, call cancel() to notify the Source of\n    // any unconsumed messages.\n    return new WebSocketCloser(lc, ws, () => stream.cancel());\n  }\n\n  static forSink<T>(\n    lc: LogContext,\n    ws: WebSocket,\n    stream: Subscription<T, Streamed<T>>,\n    messageHandler: (e: MessageEvent) => void | undefined,\n  ) {\n    // If the websocket is closed, call end() to allow the downstream Sink\n    // to process any pending messages before closing the stream.\n    return new WebSocketCloser(lc, ws, () => stream.end(), messageHandler);\n  }\n\n  private constructor(\n    lc: LogContext,\n    ws: WebSocket,\n    closeStream: () => void,\n    messageHandler?: (e: MessageEvent) => void | undefined,\n  ) {\n    this.#lc = lc;\n    this.#ws = ws;\n    this.#closeStream = closeStream;\n    this.#messageHandler = messageHandler ?? null;\n\n    ws.addEventListener('open', this.#handleOpen);\n    ws.addEventListener('close', this.#handleClose);\n    ws.addEventListener('error', this.#handleError);\n    if (this.#messageHandler) {\n      ws.addEventListener('message', this.#messageHandler);\n    }\n\n    switch (ws.readyState) {\n      case ws.CONNECTING:\n        break; // expected for new connections. resolve or reject in handlers.\n      case ws.OPEN:\n        this.#connected.resolve();\n        break;\n      default:\n        this.#connected.reject(\n          new Error(`websocket already in state ${ws.readyState}`),\n        );\n        break;\n    }\n  }\n\n  get #conn(): string {\n    return 'connection' + (this.#ws.url ? ` to ${this.#ws.url}` : '');\n  }\n\n  #handleOpen = () => {\n    this.#lc.info?.(`${this.#conn} established`);\n    this.#connected.resolve();\n  };\n\n  #handleClose = (e: CloseEvent) => {\n    const {code, reason, wasClean} = e;\n    this.#lc.info?.(`${this.#conn} closed`, {\n      code,\n      reason,\n      wasClean,\n    });\n    this.close();\n    this.#connected.reject(`${this.#conn} closed with code ${code}`);\n  };\n\n  #handleError = ({message, error}: ErrorEvent) => {\n    if (this.#ws.readyState === this.#ws.OPEN) {\n      this.#lc.error?.(`error in ${this.#conn}`, message, error);\n    }\n    this.#connected.reject(error);\n  };\n\n  close(err?: unknown) {\n    if (err) {\n      this.#lc.error?.(`closing stream with error`, err);\n    }\n    this.#closeStream();\n    if (!this.closed()) {\n      this.#ws.close();\n    }\n  }\n\n  closed() {\n    return (\n      this.#ws.readyState === this.#ws.CLOSED ||\n      this.#ws.readyState === this.#ws.CLOSING\n    );\n  }\n}\n"],"names":["streamOut","v.parse","parse","v.object","v.number","pipeline","stream"],"mappings":";;;;;;;;;;AA6BA,MAAM,mBAAmB;AAmClB,SAAS,OACd,IACA,IACA,UACA,aAA2B,CAAA,GAC3B,YAAyB,CAAA,GACzB,gBAA+B,IACe;AAC9C,QAAM,WAAW,GAAG,OAAO;AAC3B,WAAS,MAAM,KAAe;AAC5B,QAAI,GAAG,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,SAAS;AAC/D,UAAI,KAAK;AACP,uBAAe,IAAI,IAAI,GAAG;AAAA,MAC5B,OAAO;AACL,WAAG,OAAO,yBAAyB,QAAQ,EAAE;AAC7C,WAAG,MAAA;AAAA,MACL;AAAA,IACF;AAAA,EACF;AAEA,QAAM,WAAW,aAAa,OAAW;AAAA,IACvC,GAAG;AAAA,IACH,SAAS,CAAC,YAAY,QAAQ;AAC5B,gBAAU,UAAU,YAAY,GAAG;AACnC,YAAM,GAAG;AAAA,IACX;AAAA,EAAA,CACD;AACD,QAAM,YAAY,aAAa,OAAY;AAAA,IACzC,GAAG;AAAA,IACH,SAAS,CAAC,YAAY,QAAQ;AAC5B,iBAAW,UAAU,YAAY,GAAG;AACpC,YAAM,GAAG;AAAA,IACX;AAAA,EAAA,CACD;AAED,QAAM,SAAS,sBAAsB,IAAI;AAAA,IACvC,GAAG;AAAA,IACH,eAAe;AAAA,EAAA,CAChB;AAGD,WAASA,aAAY;AAEnB,WAAO,GAAG,SAAS,MAAM,GAAG,QAAQ,uBAAuB,QAAQ,EAAE,CAAC;AAEtE;AAAA,MACE,SAAS,KAAK,SAAS;AAAA,MACvB,IAAI,UAAU;AAAA,QACZ,YAAY;AAAA,QACZ,WAAW,CAAC,KAAK,WAAW,aAC1B,SAAS,MAAM,WAAW,UAAU,GAAG,CAAC;AAAA,MAAA,CAC3C;AAAA,MACD;AAAA,MACA,SAAQ,MAAM,UAAU,KAAK,GAAG,IAAI,UAAU,OAAA;AAAA,IAAO;AAAA,EAEzD;AAEA,MAAI,GAAG,eAAe,GAAG,YAAY;AACnC,OAAG,GAAG,QAAQ,MAAM;AAClB,SAAG,OAAO,gBAAgB,QAAQ,EAAE;AACpCA,iBAAAA;AAAAA,IACF,CAAC;AAAA,EACH,OAAO;AACLA,eAAAA;AAAAA,EACF;AAGA,OAAK;AAAA,IACH,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,OAAO,CAAA,UAAS;AACd,YAAM,OAAO,WAAW,MAAM,MAAM,UAAU;AAC9C,aAAOC,MAAQ,MAAM,UAAU,aAAa;AAAA,IAC9C;AAAA,EAAA,CACD;AAED,uBAAqB,IAAI,IAAI,gBAAgB;AAE7C,SAAO,EAAC,WAAW,SAAA;AACrB;AASO,SAAS,KAAQ,EAAC,QAAQ,MAAM,OAAAC,QAAO,kBAAiC;AAC7E,qBAAmB;AACnB,SAAO,kBAAkB,CAAC;AAC1B,QAAM,UAA8B,CAAA;AAEpC;AAAA,IACE;AAAA,IACA,IAAI,SAAS;AAAA,MACX,eAAe;AAAA,MACf,OAAO,CAAC,OAAO,WAAW,aAAa;AACrC,YAAI;AACJ,YAAI;AACF,eAAK,MAAMA,OAAM,KAAK,OAAO,MAAM;AACjC,qBAAA;AACA;AAAA,UACF;AAAA,QACF,SAAS,KAAK;AACZ,mBAAS,YAAY,GAAG,CAAC;AACzB;AAAA,QACF;AAMA,cAAM,EAAC,OAAA,IAAU,KAAK,KAAK,GAAG;AAC9B,gBAAQ,KAAK,MAAM;AACnB,aAAK,OAAO,KAAK,MAAM,QAAQ,OAAO;AAEtC,YAAI,QAAQ,UAAU,gBAAgB;AAEpC,mBAAA;AAAA,QACF,OAAO;AAEL,kBAAQ,CAAC,EAAE;AAAA,YACT,MAAM,SAAA;AAAA,YACN,CAAA,QAAO,SAAS,YAAY,GAAG,CAAC;AAAA,UAAA;AAAA,QAEpC;AAAA,MACF;AAAA,MACA,SAAS,CAAC,KAAK,aAAa;AAC1B,YAAI,KAAK;AACP,eAAK,KAAK,YAAY,GAAG,CAAC;AAAA,QAC5B;AAEA,iBAAA;AAAA,MACF;AAAA,MACA,OAAO,CAAA,aAAY;AACjB,aAAK,OAAA;AACL,iBAAA;AAAA,MACF;AAAA,IAAA,CACD;AAAA,IACD,SAAQ,MAAM,KAAK,KAAK,GAAG,IAAI,KAAK,OAAA;AAAA,EAAO;AAE/C;AAEA,SAAS,YAAY,KAAc;AACjC,SAAO,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAC3D;AAEA,MAAM,YAAYC,OAAS,EAAC,KAAKC,OAAE,GAAS;AAY5C,eAAsB,UACpB,IACA,QACA,MACe;AACf,uBAAqB,IAAI,MAAM,gBAAgB;AAE/C,QAAM,SAAS,gBAAgB,UAAU,IAAI,MAAM,MAAM;AAEzD,QAAM,OAAO,IAAI,MAAA;AACjB,OAAK,iBAAiB,WAAW,CAAC,EAAC,WAAU;AAC3C,QAAI;AACF,UAAI,OAAO,SAAS,UAAU;AAC5B,cAAM,IAAI,MAAM,yBAAyB;AAAA,MAC3C;AACA,WAAK,QAAQH,MAAQ,KAAK,MAAM,IAAI,GAAG,SAAS,CAAC;AAAA,IACnD,SAAS,GAAG;AACV,SAAG,QAAQ,qBAAqB,CAAC;AACjC,aAAO,MAAM,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,MAAI;AACF,QAAI,SAAS;AACb,UAAM,EAAC,UAAAI,UAAAA,IAAY;AACnB,QAAIA,WAAU;AACZ,SAAG,QAAQ,mCAAmC;AAC9C,uBAAiB,EAAC,OAAO,KAAK,SAAA,KAAaA,WAAU;AACnD,cAAM,KAAK,EAAE;AACb,cAAM,OAAO,WAAW,UAAU,EAAC,KAAK,IAAyB;AAGjE,aAAK,KAAK,IAAI;AAEd,cAAM,YAAY;AAChB,gBAAM,EAAC,IAAA,IAAO,MAAM,KAAK,QAAA;AAEzB,cAAI,QAAQ,IAAI;AACd,kBAAM,IAAI,MAAM,sBAAsB,EAAE,KAAK,GAAG,EAAE;AAAA,UACpD;AACA,mBAAA;AAAA,QACF,GAAA;AAAA,MACF;AAAA,IACF,OAAO;AACL,SAAG,QAAQ,qCAAqC;AAChD,uBAAiB,OAAO,QAAQ;AAC9B,cAAM,KAAK,EAAE;AACb,cAAM,OAAO,WAAW,UAAU,EAAC,KAAK,IAAyB;AAGjE,aAAK,KAAK,IAAI;AAEd,cAAM,EAAC,IAAA,IAAO,MAAM,KAAK,QAAA;AACzB,YAAI,QAAQ,IAAI;AACd,gBAAM,IAAI,MAAM,sBAAsB,EAAE,KAAK,GAAG,EAAE;AAAA,QACpD;AAAA,MACF;AAAA,IACF;AACA,WAAO,MAAA;AAAA,EACT,SAAS,GAAG;AACV,WAAO,MAAM,CAAC;AAAA,EAChB;AACF;AAEA,eAAsB,SACpB,IACA,QACA,QACoB;AACpB,yBAAuB,IAAI,QAAQ,gBAAgB;AAEnD,QAAM,iBAAiBF,OAAS;AAAA,IAC9B,KAAK;AAAA,IACL,IAAIC,OAAE;AAAA,EAAO,CACd;AAED,QAAM,OAAqC,IAAI;AAAA,IAC7C;AAAA,MACE,UAAU,CAAC,EAAC,GAAA,MAAQ,OAAO,KAAK,KAAK,UAAU,EAAC,KAAK,GAAA,CAAiB,CAAC;AAAA,MACvE,SAAS,MAAM,OAAO,MAAA;AAAA,IAAM;AAAA,IAE9B,CAAC,EAAC,IAAA,MAAS;AAAA,EAAA;AAGb,QAAM,SAAS,gBAAgB,QAAQ,IAAI,QAAQ,MAAM,aAAa;AAEtE,WAAS,cAAc,OAAqB;AAC1C,UAAM,OAAO,MAAM,KAAK,SAAA;AACxB,QAAI,CAAC,KAAK,QAAQ;AAChB,SAAG,OAAO,4CAA4C,IAAI;AAC1D;AAAA,IACF;AACA,QAAI;AACF,YAAM,QAAQ,WAAW,MAAM,IAAI;AACnC,YAAM,MAAMH,MAAQ,OAAO,gBAAgB,aAAa;AAGxD,WAAK,KAAK,GAAG;AAAA,IACf,SAAS,GAAG;AACV,aAAO,MAAM,CAAC;AAAA,IAChB;AAAA,EACF;AAEA,QAAM,OAAO;AACb,SAAO;AACT;AAEA,MAAM,gBAAgB;AAAA,EACX;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAa,SAAA;AAAA,EAEtB,IAAI,YAA2B;AAC7B,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,OAAO,UAAa,IAAgB,IAAeK,SAAmB;AAGpE,WAAO,IAAI,gBAAgB,IAAI,IAAI,MAAMA,QAAO,QAAQ;AAAA,EAC1D;AAAA,EAEA,OAAO,QACL,IACA,IACAA,SACA,gBACA;AAGA,WAAO,IAAI,gBAAgB,IAAI,IAAI,MAAMA,QAAO,IAAA,GAAO,cAAc;AAAA,EACvE;AAAA,EAEQ,YACN,IACA,IACA,aACA,gBACA;AACA,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,eAAe;AACpB,SAAK,kBAAkB,kBAAkB;AAEzC,OAAG,iBAAiB,QAAQ,KAAK,WAAW;AAC5C,OAAG,iBAAiB,SAAS,KAAK,YAAY;AAC9C,OAAG,iBAAiB,SAAS,KAAK,YAAY;AAC9C,QAAI,KAAK,iBAAiB;AACxB,SAAG,iBAAiB,WAAW,KAAK,eAAe;AAAA,IACrD;AAEA,YAAQ,GAAG,YAAA;AAAA,MACT,KAAK,GAAG;AACN;AAAA;AAAA,MACF,KAAK,GAAG;AACN,aAAK,WAAW,QAAA;AAChB;AAAA,MACF;AACE,aAAK,WAAW;AAAA,UACd,IAAI,MAAM,8BAA8B,GAAG,UAAU,EAAE;AAAA,QAAA;AAEzD;AAAA,IAAA;AAAA,EAEN;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,gBAAgB,KAAK,IAAI,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK;AAAA,EAChE;AAAA,EAEA,cAAc,MAAM;AAClB,SAAK,IAAI,OAAO,GAAG,KAAK,KAAK,cAAc;AAC3C,SAAK,WAAW,QAAA;AAAA,EAClB;AAAA,EAEA,eAAe,CAAC,MAAkB;AAChC,UAAM,EAAC,MAAM,QAAQ,SAAA,IAAY;AACjC,SAAK,IAAI,OAAO,GAAG,KAAK,KAAK,WAAW;AAAA,MACtC;AAAA,MACA;AAAA,MACA;AAAA,IAAA,CACD;AACD,SAAK,MAAA;AACL,SAAK,WAAW,OAAO,GAAG,KAAK,KAAK,qBAAqB,IAAI,EAAE;AAAA,EACjE;AAAA,EAEA,eAAe,CAAC,EAAC,SAAS,YAAuB;AAC/C,QAAI,KAAK,IAAI,eAAe,KAAK,IAAI,MAAM;AACzC,WAAK,IAAI,QAAQ,YAAY,KAAK,KAAK,IAAI,SAAS,KAAK;AAAA,IAC3D;AACA,SAAK,WAAW,OAAO,KAAK;AAAA,EAC9B;AAAA,EAEA,MAAM,KAAe;AACnB,QAAI,KAAK;AACP,WAAK,IAAI,QAAQ,6BAA6B,GAAG;AAAA,IACnD;AACA,SAAK,aAAA;AACL,QAAI,CAAC,KAAK,UAAU;AAClB,WAAK,IAAI,MAAA;AAAA,IACX;AAAA,EACF;AAAA,EAEA,SAAS;AACP,WACE,KAAK,IAAI,eAAe,KAAK,IAAI,UACjC,KAAK,IAAI,eAAe,KAAK,IAAI;AAAA,EAErC;AACF;"}
+{"version":3,"file":"streams.js","sources":["../../../../../zero-cache/src/types/streams.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {\n  pipeline,\n  Readable,\n  Transform,\n  Writable,\n  type DuplexOptions,\n} from 'node:stream';\nimport {\n  createWebSocketStream,\n  type CloseEvent,\n  type ErrorEvent,\n  type MessageEvent,\n  type WebSocket,\n} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {BigIntJSON, type JSONValue} from '../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../shared/src/queue.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Subscription, type Options} from './subscription.ts';\nimport {\n  closeWithError,\n  expectPingsForLiveness,\n  sendPingsForLiveness,\n} from './ws.ts';\n\n// Consistent with Postgres keepalives, and shorter than the\n// commonly used default idle timeout of 1 minute.\nconst PING_INTERVAL_MS = 30_000;\n\nexport type Source<T> = AsyncIterable<T> & {\n  /**\n   * Immediately terminates all current iterations (i.e. {@link AsyncIterator.next next()})\n   * will return `{value: undefined, done: true}`), and prevents any subsequent iterations\n   * from yielding any values.\n   *\n   * @param err Terminate the iteration by throwing the `err` instead.\n   */\n  cancel: (err?: Error) => void;\n\n  /**\n   * The presence of a `pipeline` iterable allows the usual \"consumed-on-iterate\" semantics\n   * to be overridden.\n   *\n   * This is suitable for transport layers that serialize messages across processes, such\n   * as the {@link streamOut()} method; pipelining allows the transport to send messages\n   * as they arrive without waiting for the previous message to be acked, streaming\n   * them to the receiving process where they are presumably queued and processed without\n   * a per-message ack delay. The receiving end of the transport then responds with acks\n   * asynchronously as the receiving end processes the messages.\n   */\n  pipeline?: AsyncIterable<{value: T; consumed: () => void}> | undefined;\n};\n\nexport type Sink<T> = {\n  push(message: T): void;\n};\n\n/**\n * Back-pressure-aware transformation of a WebSocket into\n * upstream and downstream {@link Subscription} objects.\n */\n// TODO: Change {@link streamIn} and {@link streamOut} to use this\n//       under the covers so that internal communication is also\n//       responsive to backpressure.\nexport function stream<In extends JSONValue, Out extends JSONValue>(\n  lc: LogContext,\n  ws: WebSocket,\n  inSchema: v.Type<In>,\n  outOptions: Options<Out> = {},\n  inOptions: Options<In> = {},\n  streamOptions: DuplexOptions = {},\n): {outstream: Sink<Out>; instream: Source<In>} {\n  const endpoint = ws.url ?? 'client';\n  function close(err?: unknown) {\n    if (ws.readyState !== ws.CLOSED && ws.readyState !== ws.CLOSING) {\n      if (err) {\n        closeWithError(lc, ws, err);\n      } else {\n        lc.info?.(`closing connection to ${endpoint}`);\n        ws.close();\n      }\n    }\n  }\n\n  const instream = Subscription.create<In>({\n    ...inOptions,\n    cleanup: (unconsumed, err) => {\n      inOptions.cleanup?.(unconsumed, err);\n      close(err);\n    },\n  });\n  const outstream = Subscription.create<Out>({\n    ...outOptions,\n    cleanup: (unconsumed, err) => {\n      outOptions.cleanup?.(unconsumed, err);\n      close(err);\n    },\n  });\n\n  const duplex = createWebSocketStream(ws, {\n    ...streamOptions,\n    decodeStrings: false,\n  });\n\n  // Outgoing transform.\n  function streamOut() {\n    // Mainly used for verifying that back-pressure kicks in tests.\n    duplex.on('drain', () => lc.debug?.(`drained messages to ${endpoint}`));\n\n    pipeline(\n      Readable.from(outstream),\n      new Transform({\n        objectMode: true,\n        transform: (msg, _encoding, callback) =>\n          callback(null, BigIntJSON.stringify(msg)),\n      }),\n      duplex,\n      err => (err ? outstream.fail(err) : outstream.cancel()),\n    );\n  }\n\n  if (ws.readyState === ws.CONNECTING) {\n    ws.on('open', () => {\n      lc.info?.(`connected to ${endpoint}`);\n      streamOut();\n    });\n  } else {\n    streamOut();\n  }\n\n  // Incoming transform.\n  pipe({\n    source: duplex,\n    sink: instream,\n    parse: chunk => {\n      const json = BigIntJSON.parse(chunk.toString());\n      return v.parse(json, inSchema, 'passthrough');\n    },\n  });\n\n  sendPingsForLiveness(lc, ws, PING_INTERVAL_MS);\n\n  return {outstream, instream};\n}\n\ntype PipeOptions<T> = {\n  source: Readable;\n  sink: Subscription<T>;\n  parse: (buffer: Buffer) => T | null;\n  bufferMessages?: number;\n};\n\nexport function pipe<T>({source, sink, parse, bufferMessages}: PipeOptions<T>) {\n  bufferMessages ??= 0;\n  assert(bufferMessages >= 0);\n  const pending: Promise<unknown>[] = [];\n\n  pipeline(\n    source,\n    new Writable({\n      decodeStrings: false,\n      write: (chunk, _encoding, callback) => {\n        let msg: T | null;\n        try {\n          if ((msg = parse(chunk)) === null) {\n            callback();\n            return;\n          }\n        } catch (err) {\n          callback(ensureError(err));\n          return;\n        }\n        // Inbound backpressure is exerted by unconsumed messages in the\n        // subscription. A buffer can be used to allow messages to queue up in\n        // in the Subscription object, which allows the consumer to \"peek\" at\n        // whether there are more messages immediately available\n        // (via {@link Subscription.queued}.\n        const {result} = sink.push(msg);\n        pending.push(result);\n        void result.then(() => pending.shift());\n\n        if (pending.length <= bufferMessages) {\n          // immediately allow more messages\n          callback();\n        } else {\n          // wait for the oldest result in the pending queue\n          pending[0].then(\n            () => callback(),\n            err => callback(ensureError(err)),\n          );\n        }\n      },\n      destroy: (err, callback) => {\n        if (err) {\n          sink.fail(ensureError(err));\n        }\n        // Otherwise, final will handle the cancel.\n        callback();\n      },\n      final: callback => {\n        sink.cancel();\n        callback();\n      },\n    }),\n    err => (err ? sink.fail(err) : sink.cancel()),\n  );\n}\n\nfunction ensureError(err: unknown) {\n  return err instanceof Error ? err : new Error(String(err));\n}\n\nconst ackSchema = v.object({ack: v.number()});\n\ntype Ack = v.Infer<typeof ackSchema>;\n\ntype Streamed<T> = {\n  /** Application-level message. */\n  msg: T;\n\n  /** ID used for the Ack message. */\n  id: number;\n};\n\nexport async function streamOut<T extends JSONValue>(\n  lc: LogContext,\n  source: Source<T>,\n  sink: WebSocket,\n): Promise<void> {\n  sendPingsForLiveness(lc, sink, PING_INTERVAL_MS);\n\n  const closer = WebSocketCloser.forSource(lc, sink, source);\n\n  const acks = new Queue<Ack>();\n  sink.addEventListener('message', ({data}) => {\n    try {\n      if (typeof data !== 'string') {\n        throw new Error('Expected string message');\n      }\n      acks.enqueue(v.parse(JSON.parse(data), ackSchema));\n    } catch (e) {\n      lc.error?.(`error parsing ack`, e);\n      closer.close(e);\n    }\n  });\n\n  try {\n    let nextID = 0;\n    const {pipeline} = source;\n    if (pipeline) {\n      lc.debug?.(`started pipelined outbound stream`);\n      for await (const {value: msg, consumed} of pipeline) {\n        const id = ++nextID;\n        const data = BigIntJSON.stringify({msg, id} satisfies Streamed<T>);\n        // Enable for debugging. Otherwise too verbose.\n        // lc.debug?.(`pipelining`, data);\n        sink.send(data);\n\n        void (async () => {\n          const {ack} = await acks.dequeue();\n          // lc.debug?.(`received ack`, ack);\n          if (ack !== id) {\n            throw new Error(`Unexpected ack for ${id}: ${ack}`);\n          }\n          consumed();\n        })();\n      }\n    } else {\n      lc.debug?.(`started synchronous outbound stream`);\n      for await (const msg of source) {\n        const id = ++nextID;\n        const data = BigIntJSON.stringify({msg, id} satisfies Streamed<T>);\n        // Enable for debugging. Otherwise too verbose.\n        // lc.debug?.(`sending`, data);\n        sink.send(data);\n\n        const {ack} = await acks.dequeue();\n        if (ack !== id) {\n          throw new Error(`Unexpected ack for ${id}: ${ack}`);\n        }\n      }\n    }\n    closer.close();\n  } catch (e) {\n    closer.close(e);\n  }\n}\n\nexport async function streamIn<T extends JSONValue>(\n  lc: LogContext,\n  source: WebSocket,\n  schema: v.Type<T>,\n): Promise<Source<T>> {\n  expectPingsForLiveness(lc, source, PING_INTERVAL_MS);\n\n  const streamedSchema = v.object({\n    msg: schema,\n    id: v.number(),\n  });\n\n  const sink: Subscription<T, Streamed<T>> = new Subscription<T, Streamed<T>>(\n    {\n      consumed: ({id}) => source.send(JSON.stringify({ack: id} satisfies Ack)),\n      cleanup: () => closer.close(),\n    },\n    ({msg}) => msg,\n  );\n\n  const closer = WebSocketCloser.forSink(lc, source, sink, handleMessage);\n\n  function handleMessage(event: MessageEvent) {\n    const data = event.data.toString();\n    if (!sink.active) {\n      lc.warn?.('dropping ws message received after close', data);\n      return;\n    }\n    try {\n      const value = BigIntJSON.parse(data);\n      const msg = v.parse(value, streamedSchema, 'passthrough');\n      // Enable for debugging. Otherwise too verbose.\n      // lc.debug?.(`received`, data);\n      sink.push(msg);\n    } catch (e) {\n      closer.close(e);\n    }\n  }\n\n  await closer.connected;\n  return sink;\n}\n\nclass WebSocketCloser {\n  readonly #lc: LogContext;\n  readonly #ws: WebSocket;\n  readonly #closeStream: () => void;\n  readonly #messageHandler: ((e: MessageEvent) => void | undefined) | null;\n  readonly #connected = resolver();\n\n  get connected(): Promise<void> {\n    return this.#connected.promise;\n  }\n\n  static forSource<T>(lc: LogContext, ws: WebSocket, stream: Source<T>) {\n    // If the websocket is closed, call cancel() to notify the Source of\n    // any unconsumed messages.\n    return new WebSocketCloser(lc, ws, () => stream.cancel());\n  }\n\n  static forSink<T>(\n    lc: LogContext,\n    ws: WebSocket,\n    stream: Subscription<T, Streamed<T>>,\n    messageHandler: (e: MessageEvent) => void | undefined,\n  ) {\n    // If the websocket is closed, call end() to allow the downstream Sink\n    // to process any pending messages before closing the stream.\n    return new WebSocketCloser(lc, ws, () => stream.end(), messageHandler);\n  }\n\n  private constructor(\n    lc: LogContext,\n    ws: WebSocket,\n    closeStream: () => void,\n    messageHandler?: (e: MessageEvent) => void | undefined,\n  ) {\n    this.#lc = lc;\n    this.#ws = ws;\n    this.#closeStream = closeStream;\n    this.#messageHandler = messageHandler ?? null;\n\n    ws.addEventListener('open', this.#handleOpen);\n    ws.addEventListener('close', this.#handleClose);\n    ws.addEventListener('error', this.#handleError);\n    if (this.#messageHandler) {\n      ws.addEventListener('message', this.#messageHandler);\n    }\n\n    switch (ws.readyState) {\n      case ws.CONNECTING:\n        break; // expected for new connections. resolve or reject in handlers.\n      case ws.OPEN:\n        this.#connected.resolve();\n        break;\n      default:\n        this.#connected.reject(\n          new Error(`websocket already in state ${ws.readyState}`),\n        );\n        break;\n    }\n  }\n\n  get #conn(): string {\n    return 'connection' + (this.#ws.url ? ` to ${this.#ws.url}` : '');\n  }\n\n  #handleOpen = () => {\n    this.#lc.info?.(`${this.#conn} established`);\n    this.#connected.resolve();\n  };\n\n  #handleClose = (e: CloseEvent) => {\n    const {code, reason, wasClean} = e;\n    this.#lc.info?.(`${this.#conn} closed`, {\n      code,\n      reason,\n      wasClean,\n    });\n    this.close();\n    this.#connected.reject(`${this.#conn} closed with code ${code}`);\n  };\n\n  #handleError = ({message, error}: ErrorEvent) => {\n    if (this.#ws.readyState === this.#ws.OPEN) {\n      this.#lc.error?.(`error in ${this.#conn}`, message, error);\n    }\n    this.#connected.reject(error);\n  };\n\n  close(err?: unknown) {\n    if (err) {\n      this.#lc.error?.(`closing stream with error`, err);\n    }\n    this.#closeStream();\n    if (!this.closed()) {\n      this.#ws.close();\n    }\n  }\n\n  closed() {\n    return (\n      this.#ws.readyState === this.#ws.CLOSED ||\n      this.#ws.readyState === this.#ws.CLOSING\n    );\n  }\n}\n"],"names":["streamOut","v.parse","parse","v.object","v.number","pipeline","stream"],"mappings":";;;;;;;;;;AA6BA,MAAM,mBAAmB;AAqClB,SAAS,OACd,IACA,IACA,UACA,aAA2B,CAAA,GAC3B,YAAyB,CAAA,GACzB,gBAA+B,IACe;AAC9C,QAAM,WAAW,GAAG,OAAO;AAC3B,WAAS,MAAM,KAAe;AAC5B,QAAI,GAAG,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,SAAS;AAC/D,UAAI,KAAK;AACP,uBAAe,IAAI,IAAI,GAAG;AAAA,MAC5B,OAAO;AACL,WAAG,OAAO,yBAAyB,QAAQ,EAAE;AAC7C,WAAG,MAAA;AAAA,MACL;AAAA,IACF;AAAA,EACF;AAEA,QAAM,WAAW,aAAa,OAAW;AAAA,IACvC,GAAG;AAAA,IACH,SAAS,CAAC,YAAY,QAAQ;AAC5B,gBAAU,UAAU,YAAY,GAAG;AACnC,YAAM,GAAG;AAAA,IACX;AAAA,EAAA,CACD;AACD,QAAM,YAAY,aAAa,OAAY;AAAA,IACzC,GAAG;AAAA,IACH,SAAS,CAAC,YAAY,QAAQ;AAC5B,iBAAW,UAAU,YAAY,GAAG;AACpC,YAAM,GAAG;AAAA,IACX;AAAA,EAAA,CACD;AAED,QAAM,SAAS,sBAAsB,IAAI;AAAA,IACvC,GAAG;AAAA,IACH,eAAe;AAAA,EAAA,CAChB;AAGD,WAASA,aAAY;AAEnB,WAAO,GAAG,SAAS,MAAM,GAAG,QAAQ,uBAAuB,QAAQ,EAAE,CAAC;AAEtE;AAAA,MACE,SAAS,KAAK,SAAS;AAAA,MACvB,IAAI,UAAU;AAAA,QACZ,YAAY;AAAA,QACZ,WAAW,CAAC,KAAK,WAAW,aAC1B,SAAS,MAAM,WAAW,UAAU,GAAG,CAAC;AAAA,MAAA,CAC3C;AAAA,MACD;AAAA,MACA,SAAQ,MAAM,UAAU,KAAK,GAAG,IAAI,UAAU,OAAA;AAAA,IAAO;AAAA,EAEzD;AAEA,MAAI,GAAG,eAAe,GAAG,YAAY;AACnC,OAAG,GAAG,QAAQ,MAAM;AAClB,SAAG,OAAO,gBAAgB,QAAQ,EAAE;AACpCA,iBAAAA;AAAAA,IACF,CAAC;AAAA,EACH,OAAO;AACLA,eAAAA;AAAAA,EACF;AAGA,OAAK;AAAA,IACH,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,OAAO,CAAA,UAAS;AACd,YAAM,OAAO,WAAW,MAAM,MAAM,UAAU;AAC9C,aAAOC,MAAQ,MAAM,UAAU,aAAa;AAAA,IAC9C;AAAA,EAAA,CACD;AAED,uBAAqB,IAAI,IAAI,gBAAgB;AAE7C,SAAO,EAAC,WAAW,SAAA;AACrB;AASO,SAAS,KAAQ,EAAC,QAAQ,MAAM,OAAAC,QAAO,kBAAiC;AAC7E,qBAAmB;AACnB,SAAO,kBAAkB,CAAC;AAC1B,QAAM,UAA8B,CAAA;AAEpC;AAAA,IACE;AAAA,IACA,IAAI,SAAS;AAAA,MACX,eAAe;AAAA,MACf,OAAO,CAAC,OAAO,WAAW,aAAa;AACrC,YAAI;AACJ,YAAI;AACF,eAAK,MAAMA,OAAM,KAAK,OAAO,MAAM;AACjC,qBAAA;AACA;AAAA,UACF;AAAA,QACF,SAAS,KAAK;AACZ,mBAAS,YAAY,GAAG,CAAC;AACzB;AAAA,QACF;AAMA,cAAM,EAAC,OAAA,IAAU,KAAK,KAAK,GAAG;AAC9B,gBAAQ,KAAK,MAAM;AACnB,aAAK,OAAO,KAAK,MAAM,QAAQ,OAAO;AAEtC,YAAI,QAAQ,UAAU,gBAAgB;AAEpC,mBAAA;AAAA,QACF,OAAO;AAEL,kBAAQ,CAAC,EAAE;AAAA,YACT,MAAM,SAAA;AAAA,YACN,CAAA,QAAO,SAAS,YAAY,GAAG,CAAC;AAAA,UAAA;AAAA,QAEpC;AAAA,MACF;AAAA,MACA,SAAS,CAAC,KAAK,aAAa;AAC1B,YAAI,KAAK;AACP,eAAK,KAAK,YAAY,GAAG,CAAC;AAAA,QAC5B;AAEA,iBAAA;AAAA,MACF;AAAA,MACA,OAAO,CAAA,aAAY;AACjB,aAAK,OAAA;AACL,iBAAA;AAAA,MACF;AAAA,IAAA,CACD;AAAA,IACD,SAAQ,MAAM,KAAK,KAAK,GAAG,IAAI,KAAK,OAAA;AAAA,EAAO;AAE/C;AAEA,SAAS,YAAY,KAAc;AACjC,SAAO,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAC3D;AAEA,MAAM,YAAYC,OAAS,EAAC,KAAKC,OAAE,GAAS;AAY5C,eAAsB,UACpB,IACA,QACA,MACe;AACf,uBAAqB,IAAI,MAAM,gBAAgB;AAE/C,QAAM,SAAS,gBAAgB,UAAU,IAAI,MAAM,MAAM;AAEzD,QAAM,OAAO,IAAI,MAAA;AACjB,OAAK,iBAAiB,WAAW,CAAC,EAAC,WAAU;AAC3C,QAAI;AACF,UAAI,OAAO,SAAS,UAAU;AAC5B,cAAM,IAAI,MAAM,yBAAyB;AAAA,MAC3C;AACA,WAAK,QAAQH,MAAQ,KAAK,MAAM,IAAI,GAAG,SAAS,CAAC;AAAA,IACnD,SAAS,GAAG;AACV,SAAG,QAAQ,qBAAqB,CAAC;AACjC,aAAO,MAAM,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,MAAI;AACF,QAAI,SAAS;AACb,UAAM,EAAC,UAAAI,UAAAA,IAAY;AACnB,QAAIA,WAAU;AACZ,SAAG,QAAQ,mCAAmC;AAC9C,uBAAiB,EAAC,OAAO,KAAK,SAAA,KAAaA,WAAU;AACnD,cAAM,KAAK,EAAE;AACb,cAAM,OAAO,WAAW,UAAU,EAAC,KAAK,IAAyB;AAGjE,aAAK,KAAK,IAAI;AAEd,cAAM,YAAY;AAChB,gBAAM,EAAC,IAAA,IAAO,MAAM,KAAK,QAAA;AAEzB,cAAI,QAAQ,IAAI;AACd,kBAAM,IAAI,MAAM,sBAAsB,EAAE,KAAK,GAAG,EAAE;AAAA,UACpD;AACA,mBAAA;AAAA,QACF,GAAA;AAAA,MACF;AAAA,IACF,OAAO;AACL,SAAG,QAAQ,qCAAqC;AAChD,uBAAiB,OAAO,QAAQ;AAC9B,cAAM,KAAK,EAAE;AACb,cAAM,OAAO,WAAW,UAAU,EAAC,KAAK,IAAyB;AAGjE,aAAK,KAAK,IAAI;AAEd,cAAM,EAAC,IAAA,IAAO,MAAM,KAAK,QAAA;AACzB,YAAI,QAAQ,IAAI;AACd,gBAAM,IAAI,MAAM,sBAAsB,EAAE,KAAK,GAAG,EAAE;AAAA,QACpD;AAAA,MACF;AAAA,IACF;AACA,WAAO,MAAA;AAAA,EACT,SAAS,GAAG;AACV,WAAO,MAAM,CAAC;AAAA,EAChB;AACF;AAEA,eAAsB,SACpB,IACA,QACA,QACoB;AACpB,yBAAuB,IAAI,QAAQ,gBAAgB;AAEnD,QAAM,iBAAiBF,OAAS;AAAA,IAC9B,KAAK;AAAA,IACL,IAAIC,OAAE;AAAA,EAAO,CACd;AAED,QAAM,OAAqC,IAAI;AAAA,IAC7C;AAAA,MACE,UAAU,CAAC,EAAC,GAAA,MAAQ,OAAO,KAAK,KAAK,UAAU,EAAC,KAAK,GAAA,CAAiB,CAAC;AAAA,MACvE,SAAS,MAAM,OAAO,MAAA;AAAA,IAAM;AAAA,IAE9B,CAAC,EAAC,IAAA,MAAS;AAAA,EAAA;AAGb,QAAM,SAAS,gBAAgB,QAAQ,IAAI,QAAQ,MAAM,aAAa;AAEtE,WAAS,cAAc,OAAqB;AAC1C,UAAM,OAAO,MAAM,KAAK,SAAA;AACxB,QAAI,CAAC,KAAK,QAAQ;AAChB,SAAG,OAAO,4CAA4C,IAAI;AAC1D;AAAA,IACF;AACA,QAAI;AACF,YAAM,QAAQ,WAAW,MAAM,IAAI;AACnC,YAAM,MAAMH,MAAQ,OAAO,gBAAgB,aAAa;AAGxD,WAAK,KAAK,GAAG;AAAA,IACf,SAAS,GAAG;AACV,aAAO,MAAM,CAAC;AAAA,IAChB;AAAA,EACF;AAEA,QAAM,OAAO;AACb,SAAO;AACT;AAEA,MAAM,gBAAgB;AAAA,EACX;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAa,SAAA;AAAA,EAEtB,IAAI,YAA2B;AAC7B,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,OAAO,UAAa,IAAgB,IAAeK,SAAmB;AAGpE,WAAO,IAAI,gBAAgB,IAAI,IAAI,MAAMA,QAAO,QAAQ;AAAA,EAC1D;AAAA,EAEA,OAAO,QACL,IACA,IACAA,SACA,gBACA;AAGA,WAAO,IAAI,gBAAgB,IAAI,IAAI,MAAMA,QAAO,IAAA,GAAO,cAAc;AAAA,EACvE;AAAA,EAEQ,YACN,IACA,IACA,aACA,gBACA;AACA,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,eAAe;AACpB,SAAK,kBAAkB,kBAAkB;AAEzC,OAAG,iBAAiB,QAAQ,KAAK,WAAW;AAC5C,OAAG,iBAAiB,SAAS,KAAK,YAAY;AAC9C,OAAG,iBAAiB,SAAS,KAAK,YAAY;AAC9C,QAAI,KAAK,iBAAiB;AACxB,SAAG,iBAAiB,WAAW,KAAK,eAAe;AAAA,IACrD;AAEA,YAAQ,GAAG,YAAA;AAAA,MACT,KAAK,GAAG;AACN;AAAA;AAAA,MACF,KAAK,GAAG;AACN,aAAK,WAAW,QAAA;AAChB;AAAA,MACF;AACE,aAAK,WAAW;AAAA,UACd,IAAI,MAAM,8BAA8B,GAAG,UAAU,EAAE;AAAA,QAAA;AAEzD;AAAA,IAAA;AAAA,EAEN;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,gBAAgB,KAAK,IAAI,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK;AAAA,EAChE;AAAA,EAEA,cAAc,MAAM;AAClB,SAAK,IAAI,OAAO,GAAG,KAAK,KAAK,cAAc;AAC3C,SAAK,WAAW,QAAA;AAAA,EAClB;AAAA,EAEA,eAAe,CAAC,MAAkB;AAChC,UAAM,EAAC,MAAM,QAAQ,SAAA,IAAY;AACjC,SAAK,IAAI,OAAO,GAAG,KAAK,KAAK,WAAW;AAAA,MACtC;AAAA,MACA;AAAA,MACA;AAAA,IAAA,CACD;AACD,SAAK,MAAA;AACL,SAAK,WAAW,OAAO,GAAG,KAAK,KAAK,qBAAqB,IAAI,EAAE;AAAA,EACjE;AAAA,EAEA,eAAe,CAAC,EAAC,SAAS,YAAuB;AAC/C,QAAI,KAAK,IAAI,eAAe,KAAK,IAAI,MAAM;AACzC,WAAK,IAAI,QAAQ,YAAY,KAAK,KAAK,IAAI,SAAS,KAAK;AAAA,IAC3D;AACA,SAAK,WAAW,OAAO,KAAK;AAAA,EAC9B;AAAA,EAEA,MAAM,KAAe;AACnB,QAAI,KAAK;AACP,WAAK,IAAI,QAAQ,6BAA6B,GAAG;AAAA,IACnD;AACA,SAAK,aAAA;AACL,QAAI,CAAC,KAAK,UAAU;AAClB,WAAK,IAAI,MAAA;AAAA,IACX;AAAA,EACF;AAAA,EAEA,SAAS;AACP,WACE,KAAK,IAAI,eAAe,KAAK,IAAI,UACjC,KAAK,IAAI,eAAe,KAAK,IAAI;AAAA,EAErC;AACF;"}

package/out/zero-cache/src/types/subscription.d.ts

@@ -103,8 +103,14 @@ export declare class Subscription<T, M = T> implements Source<T>, Sink<M> {
     end(): void;
     /**
      * Cancels the subscription immediately, cleans up, and terminates any iteration.
+     * This is intended for the consumer to call when it is no longer interested
+     * in the subscription.
+     *
+     * @param err If an `err` is specified, an iteration over the Subscription /
+     *        Sink will throw the `err` (equivalent to the producer calling
+     *        {@link fail()}). If undefined, the iteration will exit gracefully.
      */
-    cancel(): void;
+    cancel(err?: Error): void;
     /** Fails the subscription, cleans up, and throws from any iteration. */
     fail(err: Error): void;
     get pipeline(): AsyncIterable<{

package/out/zero-cache/src/types/subscription.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"subscription.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/subscription.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,IAAI,EAAE,MAAM,EAAC,MAAM,cAAc,CAAC;AAE/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,qBAAa,YAAY,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAE,YAAW,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;;IAC/D;;;;OAIG;IACH,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,EAC9B,OAAO,GAAE,OAAO,CAAC,CAAC,CAAM,EACxB,OAAO,GAAE,CAAC,CAAC,EAAE,CAAC,KAAK,CAAU;IAmB/B;;;OAGG;gBACS,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,YAAK,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC;IAoC1D;;;;;;;;;;;OAWG;IACH,IAAI,CAAC,KAAK,EAAE,CAAC,GAAG,aAAa;IA4B7B,iEAAiE;IACjE,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,kDAAkD;IAClD,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED;;;;;;;;;;;OAWG;IACH,GAAG;IAUH;;OAEG;IACH,MAAM;IAIN,wEAAwE;IACxE,IAAI,CAAC,GAAG,EAAE,KAAK;IAyBf,IAAI,QAAQ,IAAI,aAAa,CAAC;QAAC,KAAK,EAAE,CAAC,CAAC;QAAC,QAAQ,EAAE,MAAM,IAAI,CAAA;KAAC,CAAC,GAAG,SAAS,CAI1E;IA8CD,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC;CA0B3C;AAED,MAAM,MAAM,OAAO,CAAC,CAAC,IAAI;IACvB;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC;IAEnC;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,CAAC;IAE7B;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC;IAEjD;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,yCAAyC;AACzC,MAAM,MAAM,MAAM,GAAG,UAAU,GAAG,WAAW,GAAG,YAAY,CAAC;AAE7D;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG;IAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;CAAC,CAAC"}
+{"version":3,"file":"subscription.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/subscription.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,IAAI,EAAE,MAAM,EAAC,MAAM,cAAc,CAAC;AAE/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,qBAAa,YAAY,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAE,YAAW,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;;IAC/D;;;;OAIG;IACH,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,EAC9B,OAAO,GAAE,OAAO,CAAC,CAAC,CAAM,EACxB,OAAO,GAAE,CAAC,CAAC,EAAE,CAAC,KAAK,CAAU;IAmB/B;;;OAGG;gBACS,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,YAAK,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC;IAoC1D;;;;;;;;;;;OAWG;IACH,IAAI,CAAC,KAAK,EAAE,CAAC,GAAG,aAAa;IA4B7B,iEAAiE;IACjE,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,kDAAkD;IAClD,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED;;;;;;;;;;;OAWG;IACH,GAAG;IAUH;;;;;;;;OAQG;IACH,MAAM,CAAC,GAAG,CAAC,EAAE,KAAK;IAIlB,wEAAwE;IACxE,IAAI,CAAC,GAAG,EAAE,KAAK;IAyBf,IAAI,QAAQ,IAAI,aAAa,CAAC;QAAC,KAAK,EAAE,CAAC,CAAC;QAAC,QAAQ,EAAE,MAAM,IAAI,CAAA;KAAC,CAAC,GAAG,SAAS,CAI1E;IA8CD,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC;CA0B3C;AAED,MAAM,MAAM,OAAO,CAAC,CAAC,IAAI;IACvB;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC;IAEnC;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,CAAC;IAE7B;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC;IAEjD;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,yCAAyC;AACzC,MAAM,MAAM,MAAM,GAAG,UAAU,GAAG,WAAW,GAAG,YAAY,CAAC;AAE7D;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG;IAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;CAAC,CAAC"}

package/out/zero-cache/src/types/subscription.js

@@ -119,9 +119,15 @@ class Subscription {
   }
   /**
    * Cancels the subscription immediately, cleans up, and terminates any iteration.
+   * This is intended for the consumer to call when it is no longer interested
+   * in the subscription.
+   *
+   * @param err If an `err` is specified, an iteration over the Subscription /
+   *        Sink will throw the `err` (equivalent to the producer calling
+   *        {@link fail()}). If undefined, the iteration will exit gracefully.
    */
-  cancel() {
-    this.#terminate("canceled");
+  cancel(err) {
+    this.#terminate(err ?? "canceled");
   }
   /** Fails the subscription, cleans up, and throws from any iteration. */
   fail(err) {

package/out/zero-cache/src/types/subscription.js.map

@@ -1 +1 @@
-{"version":3,"file":"subscription.js","sources":["../../../../../zero-cache/src/types/subscription.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport type {Sink, Source} from './streams.ts';\n\n/**\n * A Subscription abstracts a continuous, logically infinite stream of messages intended\n * for serial processing. Unlike the more general Node `Stream` API, a Subscription has\n * a limited API with specific semantics:\n *\n * * **Serial processing**: Messages must be consumed via the {@link AsyncIterable}\n *   interface, e.g.\n *   ```ts\n *   const subscription = server.subscribe(parameters);\n *\n *   for await (const message of subscription) {\n *     await process(message);  // fully process the message before consuming the next\n *   }\n *   ```\n *\n *   Moreover, the consumer is expected to completely process each message before\n *   requesting the next. This is important for cleanup semantics (explained later).\n *\n * * **cancel()**, not close(): The underlying data in a subscription is logically infinite\n *   and only terminated when the consumer is no longer interested in receiving the messages\n *   (or requires a Subscription with a different configuration). As such, there is no API\n *   for gracefully closing the subscription after pending messages are consumed; rather,\n *   cancellation is immediate, and upon cancellation, pending messages are dropped. A\n *   Subscription can also be terminated with exceptional (i.e. `Error`) circumstances,\n *   for which the behavior is equivalent.\n *\n * * **Coalescing** (optional): A producer can configure pending messages in the Subscription\n *   to be merged together with a {@link Options.coalesce coalesce} function. This is useful\n *   for semantics in which the consumer is not necessarily interested in every incremental\n *   change, but rather the cumulative change since the last processed message. A\n *   Subscription with coalescing is guaranteed to have at most one outstanding message,\n *   regardless of how quickly messages are produced and consumed. This effectively constrains\n *   the amount of outstanding work in the system.\n *\n * ### Resource Tracking and Cleanup\n *\n * Because message consumption is constrained to the async iteration API, standard\n * control flow mechanisms allow the producer to perform bookkeeping without any\n * explicit cleanup actions on the part of the consumer. This includes:\n *\n * * **Per-message cleanup**: Each request for the {@link AsyncIterator.next next}\n *   message, or the termination of the iteration, signals that the consumer has\n *   finished processing the previous message. The producer of a Subscription can\n *   supply a {@link Options.consumed consumed} callback to receive these processed\n *   messages, allowing it to clean up attached resources (e.g. TransactionPools, etc.).\n *\n * * **Per-subscription cleanup**: The producer of a Subscription can supply a\n *   {@link Options.cleanup cleanup} callback that is invoked when the Subscription\n *   is terminated, either explicitly via {@link Subscription.cancel cancel()} /\n *   {@link Subscription.fail fail()}, or implicitly when an iteration is exited via a\n *  `break`, `return`, or `throw` statement. All unconsumed messages are passed to the\n *   call back to facilitate bookkeeping.\n *\n * @param T The external message type, published to the AsyncIterable\n * @param M The internal message type used in the producer-side interfaces\n *          (e.g. {@link push}, {@link Options.consumed}, {@link Options.coalesce},\n *          and {@link Options.cleanup}). This is often the same as the external type\n *          T, but may be diverged to facilitate internal bookkeeping.\n */\nexport class Subscription<T, M = T> implements Source<T>, Sink<M> {\n  /**\n   * Convenience factory method for creating a {@link Subscription} with internal message type\n   * `M` as a subtype of `T`, defaulting to the same type. The default `publish` method publishes\n   * the message of type `M` directly to the AsyncIterable.\n   */\n  static create<T, M extends T = T>(\n    options: Options<M> = {},\n    publish: (m: M) => T = m => m,\n  ) {\n    return new Subscription(options, publish);\n  }\n\n  // Consumers waiting to consume messages (i.e. an async iteration awaiting the next message).\n  readonly #consumers: Resolver<Entry<M> | null>[] = [];\n  // Messages waiting to be consumed.\n  readonly #messages: (Entry<M> | 'terminus')[] = [];\n  readonly #pipelineEnabled: boolean;\n  // Sentinel value signaling that the subscription is \"done\" and no more\n  // messages can be added.\n  #sentinel: 'canceled' | Error | undefined = undefined;\n\n  #coalesce: ((curr: Entry<M>, prev: Entry<M>) => M) | undefined;\n  #consumed: (prev: Entry<M>) => void;\n  #cleanup: (unconsumed: Entry<M>[], err?: Error) => void;\n  #publish: (internal: M) => T;\n\n  /**\n   * @param publish function for converting the internally pushed / coalesced message\n   *        of type `M` to the external type `T` exposed via async iteration.\n   */\n  constructor(options: Options<M> = {}, publish: (m: M) => T) {\n    const {\n      coalesce,\n      consumed = () => {},\n      cleanup = () => {},\n      pipeline = coalesce === undefined,\n    } = options;\n\n    this.#coalesce = !coalesce\n      ? undefined\n      : (curr, prev) => {\n          try {\n            return coalesce(curr.value, prev.value);\n          } finally {\n            prev.resolve('coalesced');\n          }\n        };\n\n    this.#consumed = entry => {\n      consumed(entry.value);\n      entry.resolve('consumed');\n    };\n\n    this.#cleanup = (entries, err) => {\n      cleanup(\n        entries.map(e => e.value),\n        err,\n      );\n      entries.forEach(e => e.resolve('unconsumed'));\n    };\n\n    this.#publish = publish;\n\n    this.#pipelineEnabled = pipeline;\n  }\n\n  /**\n   * Pushes the next message to be consumed, and returns a `result` that resolves to the\n   * eventual {@link Result} of the `value`.\n   *\n   * If there is an existing unconsumed message and the Subscription has a\n   * {@link Options#coalesce coalesce} function, the specified `value` will be coalesced\n   * with the pending message. In this case, the result of the pending message\n   * is resolved to `coalesced`, regardless of the `coalesce` function implementation.\n   *\n   * If the subscription is in a terminal state, the message is dropped and the\n   * result resolves to `unconsumed`.\n   */\n  push(value: M): PendingResult {\n    const {promise: result, resolve} = resolver<Result>();\n    const entry = {value, resolve};\n\n    if (this.#sentinel) {\n      entry.resolve('unconsumed');\n      return {result};\n    }\n    const consumer = this.#consumers.shift();\n    if (consumer) {\n      consumer.resolve(entry);\n    } else if (\n      this.#coalesce &&\n      this.#messages.length &&\n      this.#messages[this.#messages.length - 1] !== 'terminus'\n    ) {\n      const prev = this.#messages[this.#messages.length - 1];\n      assert(prev !== 'terminus', 'prev should not be terminus after check');\n      this.#messages[this.#messages.length - 1] = {\n        value: this.#coalesce(entry, prev),\n        resolve,\n      };\n    } else {\n      this.#messages.push(entry);\n    }\n    return {result};\n  }\n\n  /** False if the subscription has been canceled or has failed. */\n  get active(): boolean {\n    return this.#sentinel === undefined;\n  }\n\n  /** The number messages waiting to be consumed. */\n  get queued(): number {\n    return this.#messages.length;\n  }\n\n  /**\n   * Cancels the subscription after any queued messages are consumed. This is\n   * meant for the producer-side code.\n   *\n   * Any messages pushed after calling `end()` will be unconsumed as if\n   * `cancel()` were called (once the first set of pending messages is\n   * consumed). In particular, if a coalesce function is defined, the new\n   * messages will not be coalesced with the messages enqueued before `end()`\n   * was called. However, to effect the intent of memory efficiency, multiple\n   * messages pushed after calling `end()` will be coalesced together.\n   *\n   */\n  end() {\n    if (this.#sentinel) {\n      // already terminated\n    } else if (this.#messages.length === 0) {\n      this.cancel();\n    } else {\n      this.#messages.push('terminus');\n    }\n  }\n\n  /**\n   * Cancels the subscription immediately, cleans up, and terminates any iteration.\n   */\n  cancel() {\n    this.#terminate('canceled');\n  }\n\n  /** Fails the subscription, cleans up, and throws from any iteration. */\n  fail(err: Error) {\n    this.#terminate(err);\n  }\n\n  #terminate(sentinel: 'canceled' | Error) {\n    if (!this.#sentinel) {\n      this.#sentinel = sentinel;\n      this.#cleanup(\n        this.#messages.filter(m => m !== 'terminus'),\n        sentinel instanceof Error ? sentinel : undefined,\n      );\n      this.#messages.splice(0);\n\n      for (\n        let consumer = this.#consumers.shift();\n        consumer;\n        consumer = this.#consumers.shift()\n      ) {\n        sentinel === 'canceled'\n          ? consumer.resolve(null)\n          : consumer.reject(sentinel);\n      }\n    }\n  }\n\n  get pipeline(): AsyncIterable<{value: T; consumed: () => void}> | undefined {\n    return this.#pipelineEnabled\n      ? {[Symbol.asyncIterator]: () => this.#pipeline()}\n      : undefined;\n  }\n\n  #pipeline(): AsyncIterator<{value: T; consumed: () => void}> {\n    return {\n      next: async () => {\n        const entry = this.#messages.shift();\n        if (entry === 'terminus') {\n          this.cancel();\n          return {value: undefined, done: true};\n        }\n        if (entry !== undefined) {\n          return {\n            value: {\n              value: this.#publish(entry.value),\n              consumed: () => this.#consumed(entry),\n            },\n          };\n        }\n        if (this.#sentinel === 'canceled') {\n          return {value: undefined, done: true};\n        }\n        if (this.#sentinel) {\n          return Promise.reject(this.#sentinel);\n        }\n        const consumer = resolver<Entry<M> | null>();\n        this.#consumers.push(consumer);\n\n        // Wait for push() (or termination) to resolve the consumer.\n        const result = await consumer.promise;\n        return result\n          ? {\n              value: {\n                value: this.#publish(result.value),\n                consumed: () => this.#consumed(result),\n              },\n            }\n          : {value: undefined, done: true};\n      },\n\n      return: value => {\n        this.cancel();\n        return Promise.resolve({value, done: true});\n      },\n    };\n  }\n\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    const delegate = this.#pipeline();\n\n    let prevConsumed = () => {};\n    return {\n      next: async () => {\n        prevConsumed();\n\n        const entry = await delegate.next();\n        if (entry.done) {\n          return entry;\n        }\n\n        const {value, consumed} = entry.value;\n        prevConsumed = consumed;\n        return {value};\n      },\n\n      return: value => {\n        prevConsumed();\n\n        this.cancel();\n        return Promise.resolve({value, done: true});\n      },\n    };\n  }\n}\n\nexport type Options<M> = {\n  /**\n   * Coalesces messages waiting to be consumed. This is useful for \"watermark\" type\n   * subscriptions in which the consumer is only interested in the cumulative state\n   * change since the last processed message. When a `coalesce` function is specified,\n   * there is guaranteed to be at most one message waiting to be consumed.\n   *\n   * Note that the `curr` argument comes before `prev`. This facilitates a common\n   * scenario in which coalescing just means using the newest value; in such a case,\n   * `coalesce` can simply be the identity function (e.g. `msg => msg`).\n   */\n  coalesce?: (curr: M, prev: M) => M;\n\n  /**\n   * Called on the previous message in an iteration (1) when the next message is requested,\n   * or (2) when the iteration is terminated. This allows the producer to perform\n   * per-message cleanup.\n   *\n   * Note that when a {@link Options.coalesce coalesce} function is defined,\n   * `consumed` is _not_ called on the `prev` message; it is the responsibility of\n   * producers requiring both coalescing and consumption notification to perform any\n   * necessary cleanup of `prev` messages when coalescing.\n   */\n  consumed?: (prev: M) => void;\n\n  /**\n   * `cleanup` is called exactly once when the subscription is terminated via a failure or\n   * cancelation (whichever happens first), which includes implicit cancelation when\n   * the consumer exits an iteration via a `break`, `return`, or `throw` statement.\n   *\n   * Note that the `err` argument will only reflect an explicit cancelation via a call\n   * to {@link Subscription.fail()}. On the other hand, if the iteration is canceled via\n   * a `throw` statement, the thrown reason is not reflected in the `err` parameter, as that\n   * information is not made available to the AsyncIterator implementation.\n   */\n  cleanup?: (unconsumed: M[], err?: Error) => void;\n\n  /**\n   * Enable or disable pipelining when streaming messages over a websocket.\n   *\n   * If unspecified, pipelining is enabled if there is no {@link Options.coalesce coalesce}\n   * method, as pipelining is counter to the semantics of coalescing. However, the\n   * application can explicitly enable pipelining even if there is a coalesce method\n   * by specifying `true` for this option. This assumes that coalescing is either\n   * not important for websocket semantics, or that the receiving end of the websocket\n   * transport performs the desired coalescing.\n   */\n  pipeline?: boolean;\n};\n\n/** Post-queueing results of messages. */\nexport type Result = 'consumed' | 'coalesced' | 'unconsumed';\n\n/**\n * {@link Subscription.subscribe()} wraps the `Promise<Result>` in a `PendingResult`\n * object to avoid forcing all callers to handle the Promise, as most logic does not\n * need to.\n */\nexport type PendingResult = {result: Promise<Result>};\n\ntype Entry<M> = {\n  readonly value: M;\n  readonly resolve: (r: Result) => void;\n};\n"],"names":[],"mappings":";;AA+DO,MAAM,aAAqD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMhE,OAAO,OACL,UAAsB,CAAA,GACtB,UAAuB,OAAK,GAC5B;AACA,WAAO,IAAI,aAAa,SAAS,OAAO;AAAA,EAC1C;AAAA;AAAA,EAGS,aAA0C,CAAA;AAAA;AAAA,EAE1C,YAAuC,CAAA;AAAA,EACvC;AAAA;AAAA;AAAA,EAGT,YAA4C;AAAA,EAE5C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,YAAY,UAAsB,CAAA,GAAI,SAAsB;AAC1D,UAAM;AAAA,MACJ;AAAA,MACA,WAAW,MAAM;AAAA,MAAC;AAAA,MAClB,UAAU,MAAM;AAAA,MAAC;AAAA,MACjB,WAAW,aAAa;AAAA,IAAA,IACtB;AAEJ,SAAK,YAAY,CAAC,WACd,SACA,CAAC,MAAM,SAAS;AACd,UAAI;AACF,eAAO,SAAS,KAAK,OAAO,KAAK,KAAK;AAAA,MACxC,UAAA;AACE,aAAK,QAAQ,WAAW;AAAA,MAC1B;AAAA,IACF;AAEJ,SAAK,YAAY,CAAA,UAAS;AACxB,eAAS,MAAM,KAAK;AACpB,YAAM,QAAQ,UAAU;AAAA,IAC1B;AAEA,SAAK,WAAW,CAAC,SAAS,QAAQ;AAChC;AAAA,QACE,QAAQ,IAAI,CAAA,MAAK,EAAE,KAAK;AAAA,QACxB;AAAA,MAAA;AAEF,cAAQ,QAAQ,CAAA,MAAK,EAAE,QAAQ,YAAY,CAAC;AAAA,IAC9C;AAEA,SAAK,WAAW;AAEhB,SAAK,mBAAmB;AAAA,EAC1B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAcA,KAAK,OAAyB;AAC5B,UAAM,EAAC,SAAS,QAAQ,QAAA,IAAW,SAAA;AACnC,UAAM,QAAQ,EAAC,OAAO,QAAA;AAEtB,QAAI,KAAK,WAAW;AAClB,YAAM,QAAQ,YAAY;AAC1B,aAAO,EAAC,OAAA;AAAA,IACV;AACA,UAAM,WAAW,KAAK,WAAW,MAAA;AACjC,QAAI,UAAU;AACZ,eAAS,QAAQ,KAAK;AAAA,IACxB,WACE,KAAK,aACL,KAAK,UAAU,UACf,KAAK,UAAU,KAAK,UAAU,SAAS,CAAC,MAAM,YAC9C;AACA,YAAM,OAAO,KAAK,UAAU,KAAK,UAAU,SAAS,CAAC;AACrD,aAAO,SAAS,YAAY,yCAAyC;AACrE,WAAK,UAAU,KAAK,UAAU,SAAS,CAAC,IAAI;AAAA,QAC1C,OAAO,KAAK,UAAU,OAAO,IAAI;AAAA,QACjC;AAAA,MAAA;AAAA,IAEJ,OAAO;AACL,WAAK,UAAU,KAAK,KAAK;AAAA,IAC3B;AACA,WAAO,EAAC,OAAA;AAAA,EACV;AAAA;AAAA,EAGA,IAAI,SAAkB;AACpB,WAAO,KAAK,cAAc;AAAA,EAC5B;AAAA;AAAA,EAGA,IAAI,SAAiB;AACnB,WAAO,KAAK,UAAU;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAcA,MAAM;AACJ,QAAI,KAAK,UAAW;AAAA,aAET,KAAK,UAAU,WAAW,GAAG;AACtC,WAAK,OAAA;AAAA,IACP,OAAO;AACL,WAAK,UAAU,KAAK,UAAU;AAAA,IAChC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,SAAS;AACP,SAAK,WAAW,UAAU;AAAA,EAC5B;AAAA;AAAA,EAGA,KAAK,KAAY;AACf,SAAK,WAAW,GAAG;AAAA,EACrB;AAAA,EAEA,WAAW,UAA8B;AACvC,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,YAAY;AACjB,WAAK;AAAA,QACH,KAAK,UAAU,OAAO,CAAA,MAAK,MAAM,UAAU;AAAA,QAC3C,oBAAoB,QAAQ,WAAW;AAAA,MAAA;AAEzC,WAAK,UAAU,OAAO,CAAC;AAEvB,eACM,WAAW,KAAK,WAAW,MAAA,GAC/B,UACA,WAAW,KAAK,WAAW,MAAA,GAC3B;AACA,qBAAa,aACT,SAAS,QAAQ,IAAI,IACrB,SAAS,OAAO,QAAQ;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,WAAwE;AAC1E,WAAO,KAAK,mBACR,EAAC,CAAC,OAAO,aAAa,GAAG,MAAM,KAAK,UAAA,EAAU,IAC9C;AAAA,EACN;AAAA,EAEA,YAA6D;AAC3D,WAAO;AAAA,MACL,MAAM,YAAY;AAChB,cAAM,QAAQ,KAAK,UAAU,MAAA;AAC7B,YAAI,UAAU,YAAY;AACxB,eAAK,OAAA;AACL,iBAAO,EAAC,OAAO,QAAW,MAAM,KAAA;AAAA,QAClC;AACA,YAAI,UAAU,QAAW;AACvB,iBAAO;AAAA,YACL,OAAO;AAAA,cACL,OAAO,KAAK,SAAS,MAAM,KAAK;AAAA,cAChC,UAAU,MAAM,KAAK,UAAU,KAAK;AAAA,YAAA;AAAA,UACtC;AAAA,QAEJ;AACA,YAAI,KAAK,cAAc,YAAY;AACjC,iBAAO,EAAC,OAAO,QAAW,MAAM,KAAA;AAAA,QAClC;AACA,YAAI,KAAK,WAAW;AAClB,iBAAO,QAAQ,OAAO,KAAK,SAAS;AAAA,QACtC;AACA,cAAM,WAAW,SAAA;AACjB,aAAK,WAAW,KAAK,QAAQ;AAG7B,cAAM,SAAS,MAAM,SAAS;AAC9B,eAAO,SACH;AAAA,UACE,OAAO;AAAA,YACL,OAAO,KAAK,SAAS,OAAO,KAAK;AAAA,YACjC,UAAU,MAAM,KAAK,UAAU,MAAM;AAAA,UAAA;AAAA,QACvC,IAEF,EAAC,OAAO,QAAW,MAAM,KAAA;AAAA,MAC/B;AAAA,MAEA,QAAQ,CAAA,UAAS;AACf,aAAK,OAAA;AACL,eAAO,QAAQ,QAAQ,EAAC,OAAO,MAAM,MAAK;AAAA,MAC5C;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEA,CAAC,OAAO,aAAa,IAAsB;AACzC,UAAM,WAAW,KAAK,UAAA;AAEtB,QAAI,eAAe,MAAM;AAAA,IAAC;AAC1B,WAAO;AAAA,MACL,MAAM,YAAY;AAChB,qBAAA;AAEA,cAAM,QAAQ,MAAM,SAAS,KAAA;AAC7B,YAAI,MAAM,MAAM;AACd,iBAAO;AAAA,QACT;AAEA,cAAM,EAAC,OAAO,SAAA,IAAY,MAAM;AAChC,uBAAe;AACf,eAAO,EAAC,MAAA;AAAA,MACV;AAAA,MAEA,QAAQ,CAAA,UAAS;AACf,qBAAA;AAEA,aAAK,OAAA;AACL,eAAO,QAAQ,QAAQ,EAAC,OAAO,MAAM,MAAK;AAAA,MAC5C;AAAA,IAAA;AAAA,EAEJ;AACF;"}
+{"version":3,"file":"subscription.js","sources":["../../../../../zero-cache/src/types/subscription.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport type {Sink, Source} from './streams.ts';\n\n/**\n * A Subscription abstracts a continuous, logically infinite stream of messages intended\n * for serial processing. Unlike the more general Node `Stream` API, a Subscription has\n * a limited API with specific semantics:\n *\n * * **Serial processing**: Messages must be consumed via the {@link AsyncIterable}\n *   interface, e.g.\n *   ```ts\n *   const subscription = server.subscribe(parameters);\n *\n *   for await (const message of subscription) {\n *     await process(message);  // fully process the message before consuming the next\n *   }\n *   ```\n *\n *   Moreover, the consumer is expected to completely process each message before\n *   requesting the next. This is important for cleanup semantics (explained later).\n *\n * * **cancel()**, not close(): The underlying data in a subscription is logically infinite\n *   and only terminated when the consumer is no longer interested in receiving the messages\n *   (or requires a Subscription with a different configuration). As such, there is no API\n *   for gracefully closing the subscription after pending messages are consumed; rather,\n *   cancellation is immediate, and upon cancellation, pending messages are dropped. A\n *   Subscription can also be terminated with exceptional (i.e. `Error`) circumstances,\n *   for which the behavior is equivalent.\n *\n * * **Coalescing** (optional): A producer can configure pending messages in the Subscription\n *   to be merged together with a {@link Options.coalesce coalesce} function. This is useful\n *   for semantics in which the consumer is not necessarily interested in every incremental\n *   change, but rather the cumulative change since the last processed message. A\n *   Subscription with coalescing is guaranteed to have at most one outstanding message,\n *   regardless of how quickly messages are produced and consumed. This effectively constrains\n *   the amount of outstanding work in the system.\n *\n * ### Resource Tracking and Cleanup\n *\n * Because message consumption is constrained to the async iteration API, standard\n * control flow mechanisms allow the producer to perform bookkeeping without any\n * explicit cleanup actions on the part of the consumer. This includes:\n *\n * * **Per-message cleanup**: Each request for the {@link AsyncIterator.next next}\n *   message, or the termination of the iteration, signals that the consumer has\n *   finished processing the previous message. The producer of a Subscription can\n *   supply a {@link Options.consumed consumed} callback to receive these processed\n *   messages, allowing it to clean up attached resources (e.g. TransactionPools, etc.).\n *\n * * **Per-subscription cleanup**: The producer of a Subscription can supply a\n *   {@link Options.cleanup cleanup} callback that is invoked when the Subscription\n *   is terminated, either explicitly via {@link Subscription.cancel cancel()} /\n *   {@link Subscription.fail fail()}, or implicitly when an iteration is exited via a\n *  `break`, `return`, or `throw` statement. All unconsumed messages are passed to the\n *   call back to facilitate bookkeeping.\n *\n * @param T The external message type, published to the AsyncIterable\n * @param M The internal message type used in the producer-side interfaces\n *          (e.g. {@link push}, {@link Options.consumed}, {@link Options.coalesce},\n *          and {@link Options.cleanup}). This is often the same as the external type\n *          T, but may be diverged to facilitate internal bookkeeping.\n */\nexport class Subscription<T, M = T> implements Source<T>, Sink<M> {\n  /**\n   * Convenience factory method for creating a {@link Subscription} with internal message type\n   * `M` as a subtype of `T`, defaulting to the same type. The default `publish` method publishes\n   * the message of type `M` directly to the AsyncIterable.\n   */\n  static create<T, M extends T = T>(\n    options: Options<M> = {},\n    publish: (m: M) => T = m => m,\n  ) {\n    return new Subscription(options, publish);\n  }\n\n  // Consumers waiting to consume messages (i.e. an async iteration awaiting the next message).\n  readonly #consumers: Resolver<Entry<M> | null>[] = [];\n  // Messages waiting to be consumed.\n  readonly #messages: (Entry<M> | 'terminus')[] = [];\n  readonly #pipelineEnabled: boolean;\n  // Sentinel value signaling that the subscription is \"done\" and no more\n  // messages can be added.\n  #sentinel: 'canceled' | Error | undefined = undefined;\n\n  #coalesce: ((curr: Entry<M>, prev: Entry<M>) => M) | undefined;\n  #consumed: (prev: Entry<M>) => void;\n  #cleanup: (unconsumed: Entry<M>[], err?: Error) => void;\n  #publish: (internal: M) => T;\n\n  /**\n   * @param publish function for converting the internally pushed / coalesced message\n   *        of type `M` to the external type `T` exposed via async iteration.\n   */\n  constructor(options: Options<M> = {}, publish: (m: M) => T) {\n    const {\n      coalesce,\n      consumed = () => {},\n      cleanup = () => {},\n      pipeline = coalesce === undefined,\n    } = options;\n\n    this.#coalesce = !coalesce\n      ? undefined\n      : (curr, prev) => {\n          try {\n            return coalesce(curr.value, prev.value);\n          } finally {\n            prev.resolve('coalesced');\n          }\n        };\n\n    this.#consumed = entry => {\n      consumed(entry.value);\n      entry.resolve('consumed');\n    };\n\n    this.#cleanup = (entries, err) => {\n      cleanup(\n        entries.map(e => e.value),\n        err,\n      );\n      entries.forEach(e => e.resolve('unconsumed'));\n    };\n\n    this.#publish = publish;\n\n    this.#pipelineEnabled = pipeline;\n  }\n\n  /**\n   * Pushes the next message to be consumed, and returns a `result` that resolves to the\n   * eventual {@link Result} of the `value`.\n   *\n   * If there is an existing unconsumed message and the Subscription has a\n   * {@link Options#coalesce coalesce} function, the specified `value` will be coalesced\n   * with the pending message. In this case, the result of the pending message\n   * is resolved to `coalesced`, regardless of the `coalesce` function implementation.\n   *\n   * If the subscription is in a terminal state, the message is dropped and the\n   * result resolves to `unconsumed`.\n   */\n  push(value: M): PendingResult {\n    const {promise: result, resolve} = resolver<Result>();\n    const entry = {value, resolve};\n\n    if (this.#sentinel) {\n      entry.resolve('unconsumed');\n      return {result};\n    }\n    const consumer = this.#consumers.shift();\n    if (consumer) {\n      consumer.resolve(entry);\n    } else if (\n      this.#coalesce &&\n      this.#messages.length &&\n      this.#messages[this.#messages.length - 1] !== 'terminus'\n    ) {\n      const prev = this.#messages[this.#messages.length - 1];\n      assert(prev !== 'terminus', 'prev should not be terminus after check');\n      this.#messages[this.#messages.length - 1] = {\n        value: this.#coalesce(entry, prev),\n        resolve,\n      };\n    } else {\n      this.#messages.push(entry);\n    }\n    return {result};\n  }\n\n  /** False if the subscription has been canceled or has failed. */\n  get active(): boolean {\n    return this.#sentinel === undefined;\n  }\n\n  /** The number messages waiting to be consumed. */\n  get queued(): number {\n    return this.#messages.length;\n  }\n\n  /**\n   * Cancels the subscription after any queued messages are consumed. This is\n   * meant for the producer-side code.\n   *\n   * Any messages pushed after calling `end()` will be unconsumed as if\n   * `cancel()` were called (once the first set of pending messages is\n   * consumed). In particular, if a coalesce function is defined, the new\n   * messages will not be coalesced with the messages enqueued before `end()`\n   * was called. However, to effect the intent of memory efficiency, multiple\n   * messages pushed after calling `end()` will be coalesced together.\n   *\n   */\n  end() {\n    if (this.#sentinel) {\n      // already terminated\n    } else if (this.#messages.length === 0) {\n      this.cancel();\n    } else {\n      this.#messages.push('terminus');\n    }\n  }\n\n  /**\n   * Cancels the subscription immediately, cleans up, and terminates any iteration.\n   * This is intended for the consumer to call when it is no longer interested\n   * in the subscription.\n   *\n   * @param err If an `err` is specified, an iteration over the Subscription /\n   *        Sink will throw the `err` (equivalent to the producer calling\n   *        {@link fail()}). If undefined, the iteration will exit gracefully.\n   */\n  cancel(err?: Error) {\n    this.#terminate(err ?? 'canceled');\n  }\n\n  /** Fails the subscription, cleans up, and throws from any iteration. */\n  fail(err: Error) {\n    this.#terminate(err);\n  }\n\n  #terminate(sentinel: 'canceled' | Error) {\n    if (!this.#sentinel) {\n      this.#sentinel = sentinel;\n      this.#cleanup(\n        this.#messages.filter(m => m !== 'terminus'),\n        sentinel instanceof Error ? sentinel : undefined,\n      );\n      this.#messages.splice(0);\n\n      for (\n        let consumer = this.#consumers.shift();\n        consumer;\n        consumer = this.#consumers.shift()\n      ) {\n        sentinel === 'canceled'\n          ? consumer.resolve(null)\n          : consumer.reject(sentinel);\n      }\n    }\n  }\n\n  get pipeline(): AsyncIterable<{value: T; consumed: () => void}> | undefined {\n    return this.#pipelineEnabled\n      ? {[Symbol.asyncIterator]: () => this.#pipeline()}\n      : undefined;\n  }\n\n  #pipeline(): AsyncIterator<{value: T; consumed: () => void}> {\n    return {\n      next: async () => {\n        const entry = this.#messages.shift();\n        if (entry === 'terminus') {\n          this.cancel();\n          return {value: undefined, done: true};\n        }\n        if (entry !== undefined) {\n          return {\n            value: {\n              value: this.#publish(entry.value),\n              consumed: () => this.#consumed(entry),\n            },\n          };\n        }\n        if (this.#sentinel === 'canceled') {\n          return {value: undefined, done: true};\n        }\n        if (this.#sentinel) {\n          return Promise.reject(this.#sentinel);\n        }\n        const consumer = resolver<Entry<M> | null>();\n        this.#consumers.push(consumer);\n\n        // Wait for push() (or termination) to resolve the consumer.\n        const result = await consumer.promise;\n        return result\n          ? {\n              value: {\n                value: this.#publish(result.value),\n                consumed: () => this.#consumed(result),\n              },\n            }\n          : {value: undefined, done: true};\n      },\n\n      return: value => {\n        this.cancel();\n        return Promise.resolve({value, done: true});\n      },\n    };\n  }\n\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    const delegate = this.#pipeline();\n\n    let prevConsumed = () => {};\n    return {\n      next: async () => {\n        prevConsumed();\n\n        const entry = await delegate.next();\n        if (entry.done) {\n          return entry;\n        }\n\n        const {value, consumed} = entry.value;\n        prevConsumed = consumed;\n        return {value};\n      },\n\n      return: value => {\n        prevConsumed();\n\n        this.cancel();\n        return Promise.resolve({value, done: true});\n      },\n    };\n  }\n}\n\nexport type Options<M> = {\n  /**\n   * Coalesces messages waiting to be consumed. This is useful for \"watermark\" type\n   * subscriptions in which the consumer is only interested in the cumulative state\n   * change since the last processed message. When a `coalesce` function is specified,\n   * there is guaranteed to be at most one message waiting to be consumed.\n   *\n   * Note that the `curr` argument comes before `prev`. This facilitates a common\n   * scenario in which coalescing just means using the newest value; in such a case,\n   * `coalesce` can simply be the identity function (e.g. `msg => msg`).\n   */\n  coalesce?: (curr: M, prev: M) => M;\n\n  /**\n   * Called on the previous message in an iteration (1) when the next message is requested,\n   * or (2) when the iteration is terminated. This allows the producer to perform\n   * per-message cleanup.\n   *\n   * Note that when a {@link Options.coalesce coalesce} function is defined,\n   * `consumed` is _not_ called on the `prev` message; it is the responsibility of\n   * producers requiring both coalescing and consumption notification to perform any\n   * necessary cleanup of `prev` messages when coalescing.\n   */\n  consumed?: (prev: M) => void;\n\n  /**\n   * `cleanup` is called exactly once when the subscription is terminated via a failure or\n   * cancelation (whichever happens first), which includes implicit cancelation when\n   * the consumer exits an iteration via a `break`, `return`, or `throw` statement.\n   *\n   * Note that the `err` argument will only reflect an explicit cancelation via a call\n   * to {@link Subscription.fail()}. On the other hand, if the iteration is canceled via\n   * a `throw` statement, the thrown reason is not reflected in the `err` parameter, as that\n   * information is not made available to the AsyncIterator implementation.\n   */\n  cleanup?: (unconsumed: M[], err?: Error) => void;\n\n  /**\n   * Enable or disable pipelining when streaming messages over a websocket.\n   *\n   * If unspecified, pipelining is enabled if there is no {@link Options.coalesce coalesce}\n   * method, as pipelining is counter to the semantics of coalescing. However, the\n   * application can explicitly enable pipelining even if there is a coalesce method\n   * by specifying `true` for this option. This assumes that coalescing is either\n   * not important for websocket semantics, or that the receiving end of the websocket\n   * transport performs the desired coalescing.\n   */\n  pipeline?: boolean;\n};\n\n/** Post-queueing results of messages. */\nexport type Result = 'consumed' | 'coalesced' | 'unconsumed';\n\n/**\n * {@link Subscription.subscribe()} wraps the `Promise<Result>` in a `PendingResult`\n * object to avoid forcing all callers to handle the Promise, as most logic does not\n * need to.\n */\nexport type PendingResult = {result: Promise<Result>};\n\ntype Entry<M> = {\n  readonly value: M;\n  readonly resolve: (r: Result) => void;\n};\n"],"names":[],"mappings":";;AA+DO,MAAM,aAAqD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMhE,OAAO,OACL,UAAsB,CAAA,GACtB,UAAuB,OAAK,GAC5B;AACA,WAAO,IAAI,aAAa,SAAS,OAAO;AAAA,EAC1C;AAAA;AAAA,EAGS,aAA0C,CAAA;AAAA;AAAA,EAE1C,YAAuC,CAAA;AAAA,EACvC;AAAA;AAAA;AAAA,EAGT,YAA4C;AAAA,EAE5C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,YAAY,UAAsB,CAAA,GAAI,SAAsB;AAC1D,UAAM;AAAA,MACJ;AAAA,MACA,WAAW,MAAM;AAAA,MAAC;AAAA,MAClB,UAAU,MAAM;AAAA,MAAC;AAAA,MACjB,WAAW,aAAa;AAAA,IAAA,IACtB;AAEJ,SAAK,YAAY,CAAC,WACd,SACA,CAAC,MAAM,SAAS;AACd,UAAI;AACF,eAAO,SAAS,KAAK,OAAO,KAAK,KAAK;AAAA,MACxC,UAAA;AACE,aAAK,QAAQ,WAAW;AAAA,MAC1B;AAAA,IACF;AAEJ,SAAK,YAAY,CAAA,UAAS;AACxB,eAAS,MAAM,KAAK;AACpB,YAAM,QAAQ,UAAU;AAAA,IAC1B;AAEA,SAAK,WAAW,CAAC,SAAS,QAAQ;AAChC;AAAA,QACE,QAAQ,IAAI,CAAA,MAAK,EAAE,KAAK;AAAA,QACxB;AAAA,MAAA;AAEF,cAAQ,QAAQ,CAAA,MAAK,EAAE,QAAQ,YAAY,CAAC;AAAA,IAC9C;AAEA,SAAK,WAAW;AAEhB,SAAK,mBAAmB;AAAA,EAC1B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAcA,KAAK,OAAyB;AAC5B,UAAM,EAAC,SAAS,QAAQ,QAAA,IAAW,SAAA;AACnC,UAAM,QAAQ,EAAC,OAAO,QAAA;AAEtB,QAAI,KAAK,WAAW;AAClB,YAAM,QAAQ,YAAY;AAC1B,aAAO,EAAC,OAAA;AAAA,IACV;AACA,UAAM,WAAW,KAAK,WAAW,MAAA;AACjC,QAAI,UAAU;AACZ,eAAS,QAAQ,KAAK;AAAA,IACxB,WACE,KAAK,aACL,KAAK,UAAU,UACf,KAAK,UAAU,KAAK,UAAU,SAAS,CAAC,MAAM,YAC9C;AACA,YAAM,OAAO,KAAK,UAAU,KAAK,UAAU,SAAS,CAAC;AACrD,aAAO,SAAS,YAAY,yCAAyC;AACrE,WAAK,UAAU,KAAK,UAAU,SAAS,CAAC,IAAI;AAAA,QAC1C,OAAO,KAAK,UAAU,OAAO,IAAI;AAAA,QACjC;AAAA,MAAA;AAAA,IAEJ,OAAO;AACL,WAAK,UAAU,KAAK,KAAK;AAAA,IAC3B;AACA,WAAO,EAAC,OAAA;AAAA,EACV;AAAA;AAAA,EAGA,IAAI,SAAkB;AACpB,WAAO,KAAK,cAAc;AAAA,EAC5B;AAAA;AAAA,EAGA,IAAI,SAAiB;AACnB,WAAO,KAAK,UAAU;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAcA,MAAM;AACJ,QAAI,KAAK,UAAW;AAAA,aAET,KAAK,UAAU,WAAW,GAAG;AACtC,WAAK,OAAA;AAAA,IACP,OAAO;AACL,WAAK,UAAU,KAAK,UAAU;AAAA,IAChC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,OAAO,KAAa;AAClB,SAAK,WAAW,OAAO,UAAU;AAAA,EACnC;AAAA;AAAA,EAGA,KAAK,KAAY;AACf,SAAK,WAAW,GAAG;AAAA,EACrB;AAAA,EAEA,WAAW,UAA8B;AACvC,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,YAAY;AACjB,WAAK;AAAA,QACH,KAAK,UAAU,OAAO,CAAA,MAAK,MAAM,UAAU;AAAA,QAC3C,oBAAoB,QAAQ,WAAW;AAAA,MAAA;AAEzC,WAAK,UAAU,OAAO,CAAC;AAEvB,eACM,WAAW,KAAK,WAAW,MAAA,GAC/B,UACA,WAAW,KAAK,WAAW,MAAA,GAC3B;AACA,qBAAa,aACT,SAAS,QAAQ,IAAI,IACrB,SAAS,OAAO,QAAQ;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,WAAwE;AAC1E,WAAO,KAAK,mBACR,EAAC,CAAC,OAAO,aAAa,GAAG,MAAM,KAAK,UAAA,EAAU,IAC9C;AAAA,EACN;AAAA,EAEA,YAA6D;AAC3D,WAAO;AAAA,MACL,MAAM,YAAY;AAChB,cAAM,QAAQ,KAAK,UAAU,MAAA;AAC7B,YAAI,UAAU,YAAY;AACxB,eAAK,OAAA;AACL,iBAAO,EAAC,OAAO,QAAW,MAAM,KAAA;AAAA,QAClC;AACA,YAAI,UAAU,QAAW;AACvB,iBAAO;AAAA,YACL,OAAO;AAAA,cACL,OAAO,KAAK,SAAS,MAAM,KAAK;AAAA,cAChC,UAAU,MAAM,KAAK,UAAU,KAAK;AAAA,YAAA;AAAA,UACtC;AAAA,QAEJ;AACA,YAAI,KAAK,cAAc,YAAY;AACjC,iBAAO,EAAC,OAAO,QAAW,MAAM,KAAA;AAAA,QAClC;AACA,YAAI,KAAK,WAAW;AAClB,iBAAO,QAAQ,OAAO,KAAK,SAAS;AAAA,QACtC;AACA,cAAM,WAAW,SAAA;AACjB,aAAK,WAAW,KAAK,QAAQ;AAG7B,cAAM,SAAS,MAAM,SAAS;AAC9B,eAAO,SACH;AAAA,UACE,OAAO;AAAA,YACL,OAAO,KAAK,SAAS,OAAO,KAAK;AAAA,YACjC,UAAU,MAAM,KAAK,UAAU,MAAM;AAAA,UAAA;AAAA,QACvC,IAEF,EAAC,OAAO,QAAW,MAAM,KAAA;AAAA,MAC/B;AAAA,MAEA,QAAQ,CAAA,UAAS;AACf,aAAK,OAAA;AACL,eAAO,QAAQ,QAAQ,EAAC,OAAO,MAAM,MAAK;AAAA,MAC5C;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEA,CAAC,OAAO,aAAa,IAAsB;AACzC,UAAM,WAAW,KAAK,UAAA;AAEtB,QAAI,eAAe,MAAM;AAAA,IAAC;AAC1B,WAAO;AAAA,MACL,MAAM,YAAY;AAChB,qBAAA;AAEA,cAAM,QAAQ,MAAM,SAAS,KAAA;AAC7B,YAAI,MAAM,MAAM;AACd,iBAAO;AAAA,QACT;AAEA,cAAM,EAAC,OAAO,SAAA,IAAY,MAAM;AAChC,uBAAe;AACf,eAAO,EAAC,MAAA;AAAA,MACV;AAAA,MAEA,QAAQ,CAAA,UAAS;AACf,qBAAA;AAEA,aAAK,OAAA;AACL,eAAO,QAAQ,QAAQ,EAAC,OAAO,MAAM,MAAK;AAAA,MAC5C;AAAA,IAAA;AAAA,EAEJ;AACF;"}

package/out/zero-client/src/client/version.js

@@ -1,4 +1,4 @@
-const version = "0.26.0-canary.14";
+const version = "0.26.0-canary.18";
 export {
   version
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rocicorp/zero",
-  "version": "0.26.0-canary.14",
+  "version": "0.26.0-canary.18",
   "description": "Zero is a web framework for serverless web development.",
   "author": "Rocicorp, Inc.",
   "repository": {