@rocicorp/zero 1.2.0-canary.1 → 1.2.0-canary.3

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#stream","#latestStatus","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getDefaultHighWaterMark} from 'node:stream';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n  type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Downstream,\n  type Status,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  replicationStatusPublisher: ReplicationStatusPublisher,\n  subscriptionState: SubscriptionState,\n  autoReset: boolean,\n  backPressureLimitHeapProportion: number,\n  flowControlConsensusPaddingSeconds: number,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n    setTimeoutFn,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    replicationStatusPublisher,\n    autoReset,\n    backPressureLimitHeapProportion,\n    flowControlConsensusPaddingSeconds,\n    setTimeoutFn,\n  );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n */\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n  readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n\n  #latestStatus: Status;\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    replicationStatusPublisher: ReplicationStatusPublisher,\n    autoReset: boolean,\n    backPressureLimitHeapProportion: number,\n    flowControlConsensusPaddingSeconds: number,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      backPressureLimitHeapProportion,\n    );\n    this.#forwarder = new Forwarder(lc, {\n      flowControlConsensusPaddingSeconds,\n    });\n    this.#replicationStatusPublisher = replicationStatusPublisher;\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n    this.#latestStatus = {tag: 'status'};\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    this.#forwarder.startProgressMonitor();\n\n    const lagReport = await this.#source.startLagReporter();\n    if (lagReport) {\n      this.#latestStatus.lagReport = lagReport;\n    }\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership();\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        if (\n          this.#state.resetBackoff() >\n          REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n        ) {\n          // After recovering from a backoff for which a replication status\n          // error was published, publish an OK status\n          this.#replicationStatusPublisher.publish(\n            this.#lc,\n            'Replicating',\n            `Replicating from ${lastWatermark}`,\n          );\n        }\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              if (msg.lagReport) {\n                // Lag reports are not stored in the cdc change log, but rather\n                // only forwarded on \"live\" connections. When a new subscriber\n                // is catching up, it is initialized with the #latestStatus\n                // from which it can measure lag while catching up.\n                this.#latestStatus.lagReport = msg.lagReport;\n                this.#forwarder.sendStatus(this.#latestStatus);\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          const entry: WatermarkedChange = [watermark, change];\n          unflushedBytes += this.#storer.store(entry);\n          if (unflushedBytes < flushBytesThreshold) {\n            // pipeline changes until flushBytesThreshold\n            this.#forwarder.forward(entry);\n          } else {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            await this.#forwarder.forwardWithFlowControl(entry);\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, ['rollback', {tag: 'rollback'}]]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n\n    this.#forwarder.stopProgressMonitor();\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n      () => this.#latestStatus,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n\n      if (initial) {\n        this.scheduleCleanup(watermark);\n      }\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAuDA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,iCACA,oCACA,eAAe,YACiB;AAEhC,OAAM,yBAAyB,IAAI,UAAU,MAAM;AACnD,OAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,aACD;CAED,MAAM,EAAC,mBAAkB;AACzB,QAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,iCACA,oCACA,aACD;;AAGH,IAAM,8CAA8C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwJpD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,KAAa;CAU/C,WAAoB,UAAU;CAE9B,aAAsB,mBACpB,eACA,gBACA,mCACD;CAED;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,WACA,iCACA,oCACA,eAAe,YACf;AACA,OAAK,KAAK;AACV,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AACzD,QAAA,QAAc;AACd,QAAA,WAAiB;AACjB,QAAA,iBAAuB;AACvB,QAAA,SAAe;AACf,QAAA,SAAe,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,MAAA,QAAc,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;GAAG,CAAC,GACzE,QAAO,KAAK,KAAK,IAAI,EACrB,gCACD;AACD,QAAA,YAAkB,IAAI,UAAU,IAAI,EAClC,oCACD,CAAC;AACF,QAAA,6BAAmC;AACnC,QAAA,YAAkB;AAClB,QAAA,QAAc,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,aAAa;AAChE,QAAA,eAAqB,EAAC,KAAK,UAAS;;CAGtC,MAAM,MAAM;AACV,QAAA,GAAS,OAAO,yBAAyB;AAEzC,QAAA,UAAgB,sBAAsB;EAEtC,MAAM,YAAY,MAAM,MAAA,OAAa,kBAAkB;AACvD,MAAI,UACF,OAAA,aAAmB,YAAY;AAKjC,QAAM,MAAA,OAAa,iBAAiB;EAIpC,MAAM,sBAAsB,wBAAwB,MAAM;AAE1D,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;AACrB,OAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,MAAA,OAAa,wCAAwC;IAC7D,MAAM,SAAS,MAAM,MAAA,OAAa,YAChC,eACA,iBACD;AACD,UAAA,OAAa,KAAK,CAAC,OAAM,MAAK,OAAO,QAAQ,OAAO,EAAE,CAAC;AAEvD,UAAA,SAAe;AACf,QACE,MAAA,MAAY,cAAc,GAC1B,4CAIA,OAAA,2BAAiC,QAC/B,MAAA,IACA,eACA,oBAAoB,gBACrB;AAEH,gBAAY;AAEZ,eAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;AACpB,aAAQ,MAAR;MACE,KAAK;AACH,WAAI,IAAI,IACN,OAAA,OAAa,OAAO,OAAO;AAE7B,WAAI,IAAI,WAAW;AAKjB,cAAA,aAAmB,YAAY,IAAI;AACnC,cAAA,UAAgB,WAAW,MAAA,aAAmB;;AAEhD;MACF,KAAK;AACH,aAAM,MAAA,qBAA2B,IAAI;AACrC;MACF,KAAK;AACH,mBAAY,OAAO,GAAG;AACtB;MACF,KAAK;AACH,WAAI,cAAc,OAAO,GAAG,UAC1B,OAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,YAC7E;AAEH,aAAA,UAAgB,IAAI,EAAE;AACtB;MACF;AACE,WAAI,cAAc,KAChB,OAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,mCAC5B;AAEH;;KAGJ,MAAM,QAA2B,CAAC,WAAW,OAAO;AACpD,uBAAkB,MAAA,OAAa,MAAM,MAAM;AAC3C,SAAI,iBAAiB,oBAEnB,OAAA,UAAgB,QAAQ,MAAM;UACzB;AAQL,YAAM,MAAA,UAAgB,uBAAuB,MAAM;AACnD,uBAAiB;;AAGnB,SAAI,SAAS,YAAY,SAAS,WAChC,aAAY;KAId,MAAM,eAAe,MAAA,OAAa,cAAc;AAChD,SAAI,aACF,OAAM;;YAGH,GAAG;AACV,UAAM;aACE;AACR,UAAA,QAAc,QAAQ,QAAQ;AAC9B,UAAA,SAAe,KAAA;;AAIjB,OAAI,WAAW;AACb,UAAA,GAAS,OAAO,oCAAoC,YAAY;AAChE,UAAA,OAAa,OAAO;AACpB,UAAA,UAAgB,QAAQ,CAAC,WAAW,CAAC,YAAY,EAAC,KAAK,YAAW,CAAC,CAAC,CAAC;;AAIvE,SAAM,QAAQ,IAAI;IAChB,MAAA,OAAa,MAAM;IACnB,MAAA,MAAY,QAAQ,MAAA,IAAU,IAAI;IAClC,MAAA,MAAY,aAAa,8CACrB,qBACE,MAAA,IACA,uBAAuB,MAAA,IAAU,eAAe,IAAI,CACrD,GACD;IACL,CAAC;;AAGJ,QAAA,UAAgB,qBAAqB;AACrC,QAAA,GAAS,OAAO,yBAAyB;;CAG3C,OAAA,qBAA4B,KAA6B;AACvD,QAAA,GAAS,OAAO,4BAA4B,IAAI;EAChD,MAAM,EAAC,QAAO;AAEd,UAAQ,KAAR;GACE,KAAK;AACH,UAAM,kBAAkB,MAAA,UAAgB,MAAA,MAAY;AACpD,UAAM,wBACJ,MAAA,IACA,eACA,IAAI,WAAW,mBACf,IAAI,aACL;AACD,QAAI,MAAA,WAAiB;AACnB,WAAA,GAAS,OAAO,+BAA+B;AAC/C,WAAM,KAAK,KAAK,IAAI,iBAAiB,CAAC;;AAExC;GACF,QACE,aAAY,IAAI;;;CAItB,UAAU,KAAqD;EAC7D,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,MAAI,SAAS,UACX,OAAA,QAAc,SAAS;EAEzB,MAAM,aAAa,aAAa,OAAmB,EACjD,eAAe,MAAA,UAAgB,OAAO,WAAW,EAClD,CAAC;EACF,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,MAAA,aACP;AACD,MAAI,mBAAmB,MAAA,gBAAsB;AAC3C,SAAA,GAAS,OACP,2CAA2C,iBAC5C;AACD,cAAW,MACT,GACA,8BACE,MAAA,eACD,cAAc,eAAe,GAC/B;SACI;AACL,SAAA,GAAS,QAAQ,qBAAqB,WAAW,KAAK;AAEtD,SAAA,UAAgB,IAAI,WAAW;AAC/B,SAAA,OAAa,QAAQ,YAAY,KAAK;AAEtC,OAAI,QACF,MAAK,gBAAgB,UAAU;;AAGnC,SAAO,QAAQ,QAAQ,WAAW;;CAGpC,gBAAgB,WAAmB;EACjC,MAAM,WAAW,MAAA,kBAAwB;AACzC,QAAA,kBAAwB,IAAI,UAAU;AAEtC,MAAI,aAAa,EACf,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;CAI3E,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,MAAA,OAAa,2BAA2B;AACnE,MAAI,CAAC,aACH,OAAA,GAAS,OACP,+EACD;AAEH,SAAO;GACL,gBAAgB,MAAA;GAChB,cAAc,gBAAgB,MAAA;GAC/B;;;;;;;CAQH,OAAA,kBAAwC;EACtC,MAAM,UAAU,CAAC,GAAG,MAAA,kBAAwB;AAC5C,MAAI,QAAQ,WAAW,GAAG;AACxB,SAAA,GAAS,OAAO,6CAA6C;AAC7D;;EAEF,MAAM,UAAU,CAAC,GAAG,MAAA,UAAgB,SAAS,CAAC;AAC9C,MAAI,QAAQ,WAAW,GAAG;AAGxB,SAAA,GAAS,OAAO,oCAAoC;AACpD;;AAEF,MAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,QAAoC;GACpE,MAAM,kBAAkB,IAAI,GAAI,QAAoC;AACpE,OAAI,kBAAkB,gBACpB,OAAA,GAAS,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,GAC/E;QACI;AACL,UAAA,GAAS,OAAO,0BAA0B,gBAAgB,MAAM;IAChE,MAAM,QAAQ,YAAY,KAAK;IAC/B,MAAM,UAAU,MAAM,MAAA,OAAa,mBAAmB,gBAAgB;IACtE,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,UAAA,GAAS,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,MACjE;AACD,UAAA,kBAAwB,OAAO,gBAAgB;;WAE1C,GAAG;AACV,SAAA,GAAS,OAAO,4BAA4B,EAAE;YACtC;AACR,OAAI,MAAA,kBAAwB,KAE1B,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;;CAK7E,MAAM,KAAK,KAAe;AACxB,QAAA,MAAY,KAAK,MAAA,IAAU,IAAI;AAC/B,QAAA,QAAc,QAAQ,QAAQ;AAC9B,QAAM,MAAA,OAAa,MAAM;;;AAmB7B,IAAM,mBAAmB,6BAA6B"}
+{"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#stream","#latestStatus","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getDefaultHighWaterMark} from 'node:stream';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n  type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Downstream,\n  type Status,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  replicationStatusPublisher: ReplicationStatusPublisher,\n  subscriptionState: SubscriptionState,\n  autoReset: boolean,\n  backPressureLimitHeapProportion: number,\n  flowControlConsensusPaddingSeconds: number,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n    setTimeoutFn,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    replicationStatusPublisher,\n    autoReset,\n    backPressureLimitHeapProportion,\n    flowControlConsensusPaddingSeconds,\n    setTimeoutFn,\n  );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n */\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n  readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n\n  #latestStatus: Status;\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    replicationStatusPublisher: ReplicationStatusPublisher,\n    autoReset: boolean,\n    backPressureLimitHeapProportion: number,\n    flowControlConsensusPaddingSeconds: number,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      backPressureLimitHeapProportion,\n    );\n    this.#forwarder = new Forwarder(lc, {\n      flowControlConsensusPaddingSeconds,\n    });\n    this.#replicationStatusPublisher = replicationStatusPublisher;\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n    this.#latestStatus = {tag: 'status'};\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    this.#forwarder.startProgressMonitor();\n\n    const lagReport = await this.#source.startLagReporter();\n    if (lagReport) {\n      this.#latestStatus.lagReport = lagReport;\n    }\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership();\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        if (\n          this.#state.resetBackoff() >\n          REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n        ) {\n          // After recovering from a backoff for which a replication status\n          // error was published, publish an OK status\n          this.#replicationStatusPublisher.publish(\n            this.#lc,\n            'Replicating',\n            `Replicating from ${lastWatermark}`,\n          );\n        }\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              if (msg.lagReport) {\n                // Lag reports are not stored in the cdc change log, but rather\n                // only forwarded on \"live\" connections. When a new subscriber\n                // is catching up, it is initialized with the #latestStatus\n                // from which it can measure lag while catching up.\n                this.#latestStatus.lagReport = msg.lagReport;\n                this.#forwarder.sendStatus(this.#latestStatus);\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          const entry: WatermarkedChange = [watermark, change];\n          unflushedBytes += this.#storer.store(entry);\n          if (unflushedBytes < flushBytesThreshold) {\n            // pipeline changes until flushBytesThreshold\n            this.#forwarder.forward(entry);\n          } else {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            await this.#forwarder.forwardWithFlowControl(entry);\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, ['rollback', {tag: 'rollback'}]]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n\n    this.#forwarder.stopProgressMonitor();\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n      () => this.#latestStatus,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAuDA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,iCACA,oCACA,eAAe,YACiB;AAEhC,OAAM,yBAAyB,IAAI,UAAU,MAAM;AACnD,OAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,aACD;CAED,MAAM,EAAC,mBAAkB;AACzB,QAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,iCACA,oCACA,aACD;;AAGH,IAAM,8CAA8C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwJpD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,KAAa;CAU/C,WAAoB,UAAU;CAE9B,aAAsB,mBACpB,eACA,gBACA,mCACD;CAED;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,WACA,iCACA,oCACA,eAAe,YACf;AACA,OAAK,KAAK;AACV,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AACzD,QAAA,QAAc;AACd,QAAA,WAAiB;AACjB,QAAA,iBAAuB;AACvB,QAAA,SAAe;AACf,QAAA,SAAe,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,MAAA,QAAc,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;GAAG,CAAC,GACzE,QAAO,KAAK,KAAK,IAAI,EACrB,gCACD;AACD,QAAA,YAAkB,IAAI,UAAU,IAAI,EAClC,oCACD,CAAC;AACF,QAAA,6BAAmC;AACnC,QAAA,YAAkB;AAClB,QAAA,QAAc,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,aAAa;AAChE,QAAA,eAAqB,EAAC,KAAK,UAAS;;CAGtC,MAAM,MAAM;AACV,QAAA,GAAS,OAAO,yBAAyB;AAEzC,QAAA,UAAgB,sBAAsB;EAEtC,MAAM,YAAY,MAAM,MAAA,OAAa,kBAAkB;AACvD,MAAI,UACF,OAAA,aAAmB,YAAY;AAKjC,QAAM,MAAA,OAAa,iBAAiB;EAIpC,MAAM,sBAAsB,wBAAwB,MAAM;AAE1D,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;AACrB,OAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,MAAA,OAAa,wCAAwC;IAC7D,MAAM,SAAS,MAAM,MAAA,OAAa,YAChC,eACA,iBACD;AACD,UAAA,OAAa,KAAK,CAAC,OAAM,MAAK,OAAO,QAAQ,OAAO,EAAE,CAAC;AAEvD,UAAA,SAAe;AACf,QACE,MAAA,MAAY,cAAc,GAC1B,4CAIA,OAAA,2BAAiC,QAC/B,MAAA,IACA,eACA,oBAAoB,gBACrB;AAEH,gBAAY;AAEZ,eAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;AACpB,aAAQ,MAAR;MACE,KAAK;AACH,WAAI,IAAI,IACN,OAAA,OAAa,OAAO,OAAO;AAE7B,WAAI,IAAI,WAAW;AAKjB,cAAA,aAAmB,YAAY,IAAI;AACnC,cAAA,UAAgB,WAAW,MAAA,aAAmB;;AAEhD;MACF,KAAK;AACH,aAAM,MAAA,qBAA2B,IAAI;AACrC;MACF,KAAK;AACH,mBAAY,OAAO,GAAG;AACtB;MACF,KAAK;AACH,WAAI,cAAc,OAAO,GAAG,UAC1B,OAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,YAC7E;AAEH,aAAA,UAAgB,IAAI,EAAE;AACtB;MACF;AACE,WAAI,cAAc,KAChB,OAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,mCAC5B;AAEH;;KAGJ,MAAM,QAA2B,CAAC,WAAW,OAAO;AACpD,uBAAkB,MAAA,OAAa,MAAM,MAAM;AAC3C,SAAI,iBAAiB,oBAEnB,OAAA,UAAgB,QAAQ,MAAM;UACzB;AAQL,YAAM,MAAA,UAAgB,uBAAuB,MAAM;AACnD,uBAAiB;;AAGnB,SAAI,SAAS,YAAY,SAAS,WAChC,aAAY;KAId,MAAM,eAAe,MAAA,OAAa,cAAc;AAChD,SAAI,aACF,OAAM;;YAGH,GAAG;AACV,UAAM;aACE;AACR,UAAA,QAAc,QAAQ,QAAQ;AAC9B,UAAA,SAAe,KAAA;;AAIjB,OAAI,WAAW;AACb,UAAA,GAAS,OAAO,oCAAoC,YAAY;AAChE,UAAA,OAAa,OAAO;AACpB,UAAA,UAAgB,QAAQ,CAAC,WAAW,CAAC,YAAY,EAAC,KAAK,YAAW,CAAC,CAAC,CAAC;;AAIvE,SAAM,QAAQ,IAAI;IAChB,MAAA,OAAa,MAAM;IACnB,MAAA,MAAY,QAAQ,MAAA,IAAU,IAAI;IAClC,MAAA,MAAY,aAAa,8CACrB,qBACE,MAAA,IACA,uBAAuB,MAAA,IAAU,eAAe,IAAI,CACrD,GACD;IACL,CAAC;;AAGJ,QAAA,UAAgB,qBAAqB;AACrC,QAAA,GAAS,OAAO,yBAAyB;;CAG3C,OAAA,qBAA4B,KAA6B;AACvD,QAAA,GAAS,OAAO,4BAA4B,IAAI;EAChD,MAAM,EAAC,QAAO;AAEd,UAAQ,KAAR;GACE,KAAK;AACH,UAAM,kBAAkB,MAAA,UAAgB,MAAA,MAAY;AACpD,UAAM,wBACJ,MAAA,IACA,eACA,IAAI,WAAW,mBACf,IAAI,aACL;AACD,QAAI,MAAA,WAAiB;AACnB,WAAA,GAAS,OAAO,+BAA+B;AAC/C,WAAM,KAAK,KAAK,IAAI,iBAAiB,CAAC;;AAExC;GACF,QACE,aAAY,IAAI;;;CAItB,UAAU,KAAqD;EAC7D,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;AAC/D,MAAI,SAAS,UACX,OAAA,QAAc,SAAS;EAEzB,MAAM,aAAa,aAAa,OAAmB,EACjD,eAAe,MAAA,UAAgB,OAAO,WAAW,EAClD,CAAC;EACF,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,MAAA,aACP;AACD,MAAI,mBAAmB,MAAA,gBAAsB;AAC3C,SAAA,GAAS,OACP,2CAA2C,iBAC5C;AACD,cAAW,MACT,GACA,8BACE,MAAA,eACD,cAAc,eAAe,GAC/B;SACI;AACL,SAAA,GAAS,QAAQ,qBAAqB,WAAW,KAAK;AAEtD,SAAA,UAAgB,IAAI,WAAW;AAC/B,SAAA,OAAa,QAAQ,YAAY,KAAK;;AAExC,SAAO,QAAQ,QAAQ,WAAW;;CAGpC,gBAAgB,WAAmB;EACjC,MAAM,WAAW,MAAA,kBAAwB;AACzC,QAAA,kBAAwB,IAAI,UAAU;AAEtC,MAAI,aAAa,EACf,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;CAI3E,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,MAAA,OAAa,2BAA2B;AACnE,MAAI,CAAC,aACH,OAAA,GAAS,OACP,+EACD;AAEH,SAAO;GACL,gBAAgB,MAAA;GAChB,cAAc,gBAAgB,MAAA;GAC/B;;;;;;;CAQH,OAAA,kBAAwC;EACtC,MAAM,UAAU,CAAC,GAAG,MAAA,kBAAwB;AAC5C,MAAI,QAAQ,WAAW,GAAG;AACxB,SAAA,GAAS,OAAO,6CAA6C;AAC7D;;EAEF,MAAM,UAAU,CAAC,GAAG,MAAA,UAAgB,SAAS,CAAC;AAC9C,MAAI,QAAQ,WAAW,GAAG;AAGxB,SAAA,GAAS,OAAO,oCAAoC;AACpD;;AAEF,MAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,QAAoC;GACpE,MAAM,kBAAkB,IAAI,GAAI,QAAoC;AACpE,OAAI,kBAAkB,gBACpB,OAAA,GAAS,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,GAC/E;QACI;AACL,UAAA,GAAS,OAAO,0BAA0B,gBAAgB,MAAM;IAChE,MAAM,QAAQ,YAAY,KAAK;IAC/B,MAAM,UAAU,MAAM,MAAA,OAAa,mBAAmB,gBAAgB;IACtE,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,UAAA,GAAS,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,MACjE;AACD,UAAA,kBAAwB,OAAO,gBAAgB;;WAE1C,GAAG;AACV,SAAA,GAAS,OAAO,4BAA4B,EAAE;YACtC;AACR,OAAI,MAAA,kBAAwB,KAE1B,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;;CAK7E,MAAM,KAAK,KAAe;AACxB,QAAA,MAAY,KAAK,MAAA,IAAU,IAAI;AAC/B,QAAA,QAAc,QAAQ,QAAQ;AAC9B,QAAM,MAAA,OAAa,MAAM;;;AAmB7B,IAAM,mBAAmB,6BAA6B"}

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts

@@ -1,8 +1,11 @@
+import type { Enum } from '../../../../shared/src/enum.ts';
 import * as v from '../../../../shared/src/valita.ts';
 import type { Source } from '../../types/streams.ts';
 import { type Change } from '../change-source/protocol/current/data.ts';
 import type { ReplicatorMode } from '../replicator/replicator.ts';
 import type { Service } from '../service.ts';
+import * as ErrorType from './error-type-enum.ts';
+type ErrorType = Enum<typeof ErrorType>;
 /**
  * The ChangeStreamer is the component between replicators ("subscribers")
  * and a canonical upstream source of changes (e.g. a Postgres logical
@@ -377,6 +380,7 @@ export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status"
     message: v.Optional<string>;
 }, undefined>]>]>;
 export type Error = v.Infer<typeof errorSchema>;
+export declare function errorTypeToReadableName(val: ErrorType): "WrongReplicaVersion" | "WatermarkTooOld" | "Unknown";
 /**
  * A stream of transactions, each starting with a {@link Begin} message,
  * containing one or more {@link Data} messages, and ending with a

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,2CAA2C,CAAC;AAEtE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;CAChE;AA0BD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;;;;;;;aASvB,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAElD,eAAO,MAAM,mBAAmB;;;;;;;;;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,MAAM,WAAW,qBAAsB,SAAQ,cAAc,EAAE,OAAO;IACpE;;;;OAIG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzC,iBAAiB,IAAI,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACJ"}
+{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AACzD,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,2CAA2C,CAAC;AAEtE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAElD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;CAChE;AA0BD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;;;;;;;aASvB,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAElD,eAAO,MAAM,mBAAmB;;;;;;;;;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,SAAS,yDAWrD;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,MAAM,WAAW,qBAAsB,SAAQ,cAAc,EAAE,OAAO;IACpE;;;;OAIG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzC,iBAAiB,IAAI,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACJ"}

package/out/zero-cache/src/services/change-streamer/change-streamer.js

@@ -21,7 +21,15 @@ var subscriptionErrorSchema = valita_exports.object({
 });
 var errorSchema = valita_exports.tuple([valita_exports.literal("error"), subscriptionErrorSchema]);
 var downstreamSchema = valita_exports.union(statusMessageSchema, changeStreamDataSchema, errorSchema);
+function errorTypeToReadableName(val) {
+	switch (val) {
+		case 1: return "WrongReplicaVersion";
+		case 2: return "WatermarkTooOld";
+		case 0: return "Unknown";
+		default: return "Unknown";
+	}
+}
 //#endregion
-export { downstreamSchema };
+export { downstreamSchema, errorTypeToReadableName };
 
 //# sourceMappingURL=change-streamer.js.map

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {type Change} from '../change-source/protocol/current/data.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the BackupMonitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string | null; // TODO: Make required when v3 is min.\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n};\n\nexport type ChangeEntry = {\n  change: Change;\n\n  /**\n   * Note that it is technically possible for multiple changes to have\n   * the same watermark, but that of a commit is guaranteed to be final,\n   * so subscribers should only store the watermark of commit changes.\n   */\n  watermark: string;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService extends ChangeStreamer, Service {\n  /**\n   * Notifies the change streamer of a watermark that has been backed up,\n   * indicating that changes before the watermark can be purged if active\n   * subscribers have progressed beyond the watermark.\n   */\n  scheduleCleanup(watermark: string): void;\n\n  getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }>;\n}\n"],"mappings":";;;;;;;;;AA0IA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,SAAS;CAExB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,UAAU;EACjD,gBAAgB,eAAE,QAAQ;EAC3B,CAAC,CACD,UAAU;CACd,CAAC;AAIF,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,SAAS,EAAE,aAAa,CAAC;AAU/E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,QAAQ;CAChB,SAAS,eAAE,QAAQ,CAAC,UAAU;CAC/B,CAAC;AAIF,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,EAAE,wBAAwB,CAAC;AAE1E,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,YACD"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {type Change} from '../change-source/protocol/current/data.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the BackupMonitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string | null; // TODO: Make required when v3 is min.\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n};\n\nexport type ChangeEntry = {\n  change: Change;\n\n  /**\n   * Note that it is technically possible for multiple changes to have\n   * the same watermark, but that of a commit is guaranteed to be final,\n   * so subscribers should only store the watermark of commit changes.\n   */\n  watermark: string;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService extends ChangeStreamer, Service {\n  /**\n   * Notifies the change streamer of a watermark that has been backed up,\n   * indicating that changes before the watermark can be purged if active\n   * subscribers have progressed beyond the watermark.\n   */\n  scheduleCleanup(watermark: string): void;\n\n  getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }>;\n}\n"],"mappings":";;;;;;;;;AA8IA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,SAAS;CAExB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,UAAU;EACjD,gBAAgB,eAAE,QAAQ;EAC3B,CAAC,CACD,UAAU;CACd,CAAC;AAIF,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,SAAS,EAAE,aAAa,CAAC;AAU/E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,QAAQ;CAChB,SAAS,eAAE,QAAQ,CAAC,UAAU;CAC/B,CAAC;AAIF,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,EAAE,wBAAwB,CAAC;AAE1E,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,YACD;AAID,SAAgB,wBAAwB,KAAgB;AACtD,SAAQ,KAAR;EACE,KAAK,EACH,QAAO;EACT,KAAK,EACH,QAAO;EACT,KAAK,EACH,QAAO;EACT,QACE,QAAO"}

package/out/zero-cache/src/services/mutagen/pusher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pusher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAStE,OAAO,EAGL,KAAK,UAAU,EACf,KAAK,QAAQ,EAEd,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,aAAa,EAAE,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC7E,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAE9D,MAAM,WAAW,MAAO,SAAQ,iBAAiB;IAC/C,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAErC,cAAc,CACZ,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,EACnD,aAAa,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GACvC,MAAM,CAAC,UAAU,CAAC,CAAC;IACtB,WAAW,CACT,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,QAAQ,EACd,IAAI,EAAE,MAAM,GAAG,SAAS,EACxB,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,MAAM,EAAE,MAAM,GAAG,SAAS,GACzB,aAAa,CAAC;IACjB,oBAAoB,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,qBAAqB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3D;AAED,KAAK,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;AAEhD;;;;;;;;;;;;GAYG;AACH,qBAAa,aAAc,YAAW,OAAO,EAAE,MAAM;;IACnD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAYlB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,UAAU,CAAC,MAAM,CAAC,GAAG;QAAC,GAAG,EAAE,MAAM,EAAE,CAAA;KAAC,EAChD,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM;IAkBvB,IAAI,OAAO,IAAI,MAAM,GAAG,SAAS,CAEhC;IAED,cAAc,CACZ,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,EACnD,aAAa,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAW1C,WAAW,CACT,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,QAAQ,EACd,IAAI,EAAE,MAAM,GAAG,SAAS,EACxB,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,MAAM,EAAE,MAAM,GAAG,SAAS,GACzB,OAAO,CAAC,aAAa,EAAE,YAAY,CAAC;IAWjC,oBAAoB,CAAC,MAAM,EAAE,UAAU;IAiDvC,qBAAqB,CAAC,SAAS,EAAE,MAAM,EAAE;IAmD/C,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAKpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAQtB;AAED,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AACF,KAAK,iBAAiB,GAAG,WAAW,GAAG,MAAM,CAAC;AAmV9C;;;;;GAKG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,CAAC,iBAAiB,GAAG,SAAS,CAAC,EAAE,GAClD,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,CAqC1B"}
+{"version":3,"file":"pusher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAStE,OAAO,EAGL,KAAK,UAAU,EACf,KAAK,QAAQ,EAEd,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,aAAa,EAAE,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC7E,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAE9D,MAAM,WAAW,MAAO,SAAQ,iBAAiB;IAC/C,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAErC,cAAc,CACZ,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,EACnD,aAAa,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GACvC,MAAM,CAAC,UAAU,CAAC,CAAC;IACtB,WAAW,CACT,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,QAAQ,EACd,IAAI,EAAE,MAAM,GAAG,SAAS,EACxB,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,MAAM,EAAE,MAAM,GAAG,SAAS,GACzB,aAAa,CAAC;IACjB,oBAAoB,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,qBAAqB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3D;AAED,KAAK,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;AAEhD;;;;;;;;;;;;GAYG;AACH,qBAAa,aAAc,YAAW,OAAO,EAAE,MAAM;;IACnD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAYlB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,UAAU,CAAC,MAAM,CAAC,GAAG;QAAC,GAAG,EAAE,MAAM,EAAE,CAAA;KAAC,EAChD,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM;IAkBvB,IAAI,OAAO,IAAI,MAAM,GAAG,SAAS,CAEhC;IAED,cAAc,CACZ,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,EACnD,aAAa,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAW1C,WAAW,CACT,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,QAAQ,EACd,IAAI,EAAE,MAAM,GAAG,SAAS,EACxB,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,MAAM,EAAE,MAAM,GAAG,SAAS,GACzB,OAAO,CAAC,aAAa,EAAE,YAAY,CAAC;IAWjC,oBAAoB,CAAC,MAAM,EAAE,UAAU;IAiDvC,qBAAqB,CAAC,SAAS,EAAE,MAAM,EAAE;IAmD/C,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAKpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAQtB;AAED,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AACF,KAAK,iBAAiB,GAAG,WAAW,GAAG,MAAM,CAAC;AAuV9C;;;;;GAKG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,CAAC,iBAAiB,GAAG,SAAS,CAAC,EAAE,GAClD,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,CAqC1B"}

package/out/zero-cache/src/services/mutagen/pusher.js

@@ -50,7 +50,7 @@ var PusherService = class {
 		this.#pushConfig = pushConfig;
 	}
 	get pushURL() {
-		return this.#pusher.pushURL[0];
+		return this.#pusher.pushURLs[0];
 	}
 	initConnection(clientID, wsID, userPushURL, userPushHeaders, onAuthFailure) {
 		return this.#pusher.initConnection(clientID, wsID, userPushURL, userPushHeaders, onAuthFailure);
@@ -67,7 +67,7 @@ var PusherService = class {
 		return { type: "ok" };
 	}
 	async ackMutationResponses(upToID) {
-		const url = this.#pushConfig.url[0];
+		const url = this.#pusher.effectivePushURL;
 		if (!url) return;
 		const cleanupBody = {
 			clientGroupID: this.id,
@@ -99,7 +99,7 @@ var PusherService = class {
 	}
 	async deleteClientMutations(clientIDs) {
 		if (clientIDs.length === 0) return;
-		const url = this.#pushConfig.url[0];
+		const url = this.#pusher.effectivePushURL;
 		if (!url) return;
 		const cleanupBody = {
 			clientGroupID: this.id,
@@ -178,9 +178,12 @@ var PushWorker = class {
 		this.#config = config;
 		this.#clients = /* @__PURE__ */ new Map();
 	}
-	get pushURL() {
+	get pushURLs() {
 		return this.#pushURLs;
 	}
+	get effectivePushURL() {
+		return this.#userPushURL ?? this.#pushURLs[0];
+	}
 	/**
 	* Returns a new downstream stream if the clientID,wsID pair has not been seen before.
 	* If a clientID already exists with a different wsID, that client's downstream is cancelled.

package/out/zero-cache/src/services/mutagen/pusher.js.map

@@ -1 +1 @@
-{"version":3,"file":"pusher.js","names":["#pusher","#queue","#pushConfig","#config","#lc","#pushURLPatterns","#isStopped","#refCount","#stopped","#pushURLs","#apiKey","#allowedClientHeaders","#clients","#customMutations","#pushes","#userPushURL","#userPushHeaders","#processPush","#fanOutResponses","#failDownstream"],"sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {groupBy} from '../../../../shared/src/arrays.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {getErrorMessage} from '../../../../shared/src/error.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport type {Downstream} from '../../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../../../zero-protocol/src/error-reason.ts';\nimport {\n  isProtocolError,\n  type PushFailedBody,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  CLEANUP_RESULTS_MUTATION_NAME,\n  pushResponseSchema,\n  type MutationID,\n  type PushBody,\n  type PushResponse,\n} from '../../../../zero-protocol/src/push.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport {compileUrlPattern, fetchFromAPIServer} from '../../custom/fetch.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport {ProtocolErrorWithLevel} from '../../types/error-with-level.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {HandlerResult, StreamResult} from '../../workers/connection.ts';\nimport type {RefCountedService, Service} from '../service.ts';\n\nexport interface Pusher extends RefCountedService {\n  readonly pushURL: string | undefined;\n\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n    userPushHeaders: Record<string, string> | undefined,\n    onAuthFailure?: (() => void) | undefined,\n  ): Source<Downstream>;\n  enqueuePush(\n    clientID: string,\n    push: PushBody,\n    auth: string | undefined,\n    httpCookie: string | undefined,\n    origin: string | undefined,\n  ): HandlerResult;\n  ackMutationResponses(upToID: MutationID): Promise<void>;\n  deleteClientMutations(clientIDs: string[]): Promise<void>;\n}\n\ntype Config = Pick<ZeroConfig, 'app' | 'shard'>;\n\n/**\n * Receives push messages from zero-client and forwards\n * them the the user's API server.\n *\n * If the user's API server is taking too long to process\n * the push, the PusherService will add the push to a queue\n * and send pushes in bulk the next time the user's API server\n * is available.\n *\n * - One PusherService exists per client group.\n * - Mutations for a given client are always sent in-order\n * - Mutations for different clients in the same group may be interleaved\n */\nexport class PusherService implements Service, Pusher {\n  readonly id: string;\n  readonly #pusher: PushWorker;\n  readonly #queue: Queue<PusherEntryOrStop>;\n  readonly #pushConfig: ZeroConfig['push'] & {url: string[]};\n  readonly #config: Config;\n  readonly #lc: LogContext;\n  readonly #pushURLPatterns: URLPattern[];\n  #stopped: Promise<void> | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  constructor(\n    appConfig: Config,\n    pushConfig: ZeroConfig['push'] & {url: string[]},\n    lc: LogContext,\n    clientGroupID: string,\n  ) {\n    this.#config = appConfig;\n    this.#lc = lc.withContext('component', 'pusherService');\n    this.#pushURLPatterns = pushConfig.url.map(compileUrlPattern);\n    this.#queue = new Queue();\n    this.#pusher = new PushWorker(\n      appConfig,\n      lc,\n      pushConfig.url,\n      pushConfig.apiKey,\n      pushConfig.allowedClientHeaders,\n      this.#queue,\n    );\n    this.id = clientGroupID;\n    this.#pushConfig = pushConfig;\n  }\n\n  get pushURL(): string | undefined {\n    return this.#pusher.pushURL[0];\n  }\n\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n    userPushHeaders: Record<string, string> | undefined,\n    onAuthFailure?: (() => void) | undefined,\n  ) {\n    return this.#pusher.initConnection(\n      clientID,\n      wsID,\n      userPushURL,\n      userPushHeaders,\n      onAuthFailure,\n    );\n  }\n\n  enqueuePush(\n    clientID: string,\n    push: PushBody,\n    auth: string | undefined,\n    httpCookie: string | undefined,\n    origin: string | undefined,\n  ): Exclude<HandlerResult, StreamResult> {\n    if (!this.#pushConfig.forwardCookies) {\n      httpCookie = undefined; // remove cookies if not forwarded\n    }\n    this.#queue.enqueue({push, auth, clientID, httpCookie, origin});\n\n    return {\n      type: 'ok',\n    };\n  }\n\n  async ackMutationResponses(upToID: MutationID) {\n    const url = this.#pushConfig.url[0];\n    if (!url) {\n      // No push URL configured, skip cleanup\n      return;\n    }\n\n    const cleanupBody: PushBody = {\n      clientGroupID: this.id,\n      mutations: [\n        {\n          type: MutationType.Custom,\n          id: 0, // Not tracked - this is fire-and-forget\n          clientID: upToID.clientID,\n          name: CLEANUP_RESULTS_MUTATION_NAME,\n          args: [\n            {\n              type: 'single',\n              clientGroupID: this.id,\n              clientID: upToID.clientID,\n              upToMutationID: upToID.id,\n            },\n          ],\n          timestamp: Date.now(),\n        },\n      ],\n      pushVersion: 1,\n      timestamp: Date.now(),\n      requestID: `cleanup-${this.id}-${upToID.clientID}-${upToID.id}`,\n    };\n\n    try {\n      await fetchFromAPIServer(\n        pushResponseSchema,\n        'push',\n        this.#lc,\n        url,\n        this.#pushURLPatterns,\n        {appID: this.#config.app.id, shardNum: this.#config.shard.num},\n        {apiKey: this.#pushConfig.apiKey},\n        cleanupBody,\n      );\n    } catch (e) {\n      this.#lc.warn?.('Failed to send cleanup mutation', {\n        error: getErrorMessage(e),\n      });\n    }\n  }\n\n  async deleteClientMutations(clientIDs: string[]) {\n    if (clientIDs.length === 0) {\n      return;\n    }\n    const url = this.#pushConfig.url[0];\n    if (!url) {\n      // No push URL configured, skip cleanup\n      return;\n    }\n\n    const cleanupBody: PushBody = {\n      clientGroupID: this.id,\n      mutations: [\n        {\n          type: MutationType.Custom,\n          id: 0, // Not tracked - this is fire-and-forget\n          clientID: clientIDs[0], // Use first client as sender\n          name: CLEANUP_RESULTS_MUTATION_NAME,\n          args: [\n            {\n              type: 'bulk',\n              clientGroupID: this.id,\n              clientIDs,\n            },\n          ],\n          timestamp: Date.now(),\n        },\n      ],\n      pushVersion: 1,\n      timestamp: Date.now(),\n      requestID: `cleanup-bulk-${this.id}-${Date.now()}`,\n    };\n\n    try {\n      await fetchFromAPIServer(\n        pushResponseSchema,\n        'push',\n        this.#lc,\n        url,\n        this.#pushURLPatterns,\n        {appID: this.#config.app.id, shardNum: this.#config.shard.num},\n        {apiKey: this.#pushConfig.apiKey},\n        cleanupBody,\n      );\n    } catch (e) {\n      this.#lc.warn?.('Failed to send bulk cleanup mutation', {\n        error: getErrorMessage(e),\n      });\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'PusherService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'PusherService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  run(): Promise<void> {\n    this.#stopped = this.#pusher.run();\n    return this.#stopped;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return must(this.#stopped, 'Stop was called before `run`');\n    }\n    this.#isStopped = true;\n    this.#queue.enqueue('stop');\n    return must(this.#stopped, 'Stop was called before `run`');\n  }\n}\n\ntype PusherEntry = {\n  push: PushBody;\n  auth: string | undefined;\n  httpCookie: string | undefined;\n  origin: string | undefined;\n  clientID: string;\n};\ntype PusherEntryOrStop = PusherEntry | 'stop';\n\n/**\n * Awaits items in the queue then drains and sends them all\n * to the user's API server.\n */\nclass PushWorker {\n  readonly #pushURLs: string[];\n  readonly #pushURLPatterns: URLPattern[];\n  readonly #apiKey: string | undefined;\n  readonly #allowedClientHeaders: readonly string[] | undefined;\n  readonly #queue: Queue<PusherEntryOrStop>;\n  readonly #lc: LogContext;\n  readonly #config: Config;\n  readonly #clients: Map<\n    string,\n    {\n      wsID: string;\n      downstream: Subscription<Downstream>;\n      onAuthFailure: (() => void) | undefined;\n    }\n  >;\n  #userPushURL?: string | undefined;\n  #userPushHeaders?: Record<string, string> | undefined;\n\n  readonly #customMutations = getOrCreateCounter(\n    'mutation',\n    'custom',\n    'Number of custom mutations processed',\n  );\n  readonly #pushes = getOrCreateCounter(\n    'mutation',\n    'pushes',\n    'Number of pushes processed by the pusher',\n  );\n\n  constructor(\n    config: Config,\n    lc: LogContext,\n    pushURL: string[],\n    apiKey: string | undefined,\n    allowedClientHeaders: readonly string[] | undefined,\n    queue: Queue<PusherEntryOrStop>,\n  ) {\n    this.#pushURLs = pushURL;\n    this.#lc = lc.withContext('component', 'pusher');\n    this.#pushURLPatterns = pushURL.map(compileUrlPattern);\n    this.#apiKey = apiKey;\n    this.#allowedClientHeaders = allowedClientHeaders;\n    this.#queue = queue;\n    this.#config = config;\n    this.#clients = new Map();\n  }\n\n  get pushURL() {\n    return this.#pushURLs;\n  }\n\n  /**\n   * Returns a new downstream stream if the clientID,wsID pair has not been seen before.\n   * If a clientID already exists with a different wsID, that client's downstream is cancelled.\n   */\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n    userPushHeaders: Record<string, string> | undefined,\n    onAuthFailure?: (() => void) | undefined,\n  ) {\n    const existing = this.#clients.get(clientID);\n    if (existing && existing.wsID === wsID) {\n      // already initialized for this socket\n      throw new Error('Connection was already initialized');\n    }\n\n    // client is back on a new connection\n    if (existing) {\n      existing.downstream.cancel();\n    }\n\n    // Handle client group level URL parameters\n    if (this.#userPushURL === undefined) {\n      // First client in the group - store its URL and headers\n      this.#userPushURL = userPushURL;\n      this.#userPushHeaders = userPushHeaders;\n    } else {\n      // Validate that subsequent clients have compatible parameters\n      if (this.#userPushURL !== userPushURL) {\n        this.#lc.warn?.(\n          'Client provided different mutate parameters than client group',\n          {\n            clientID,\n            clientURL: userPushURL,\n            clientGroupURL: this.#userPushURL,\n          },\n        );\n      }\n    }\n\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => {\n        this.#clients.delete(clientID);\n      },\n    });\n    this.#clients.set(clientID, {wsID, downstream, onAuthFailure});\n    return downstream;\n  }\n\n  async run() {\n    for (;;) {\n      const task = await this.#queue.dequeue();\n      const rest = this.#queue.drain();\n      const [pushes, terminate] = combinePushes([task, ...rest]);\n      for (const push of pushes) {\n        const response = await this.#processPush(push);\n        await this.#fanOutResponses(response);\n      }\n\n      if (terminate) {\n        break;\n      }\n    }\n  }\n\n  /**\n   * 1. If the entire `push` fails, we send the error to relevant clients.\n   * 2. If the push succeeds, we look for any mutation failure that should cause the connection to terminate\n   *  and terminate the connection for those clients.\n   */\n  #fanOutResponses(response: PushResponse) {\n    const connectionTerminations: (() => void)[] = [];\n\n    // if the entire push failed, send that to the client.\n    if ('kind' in response || 'error' in response) {\n      this.#lc.warn?.(\n        'The server behind ZERO_MUTATE_URL returned a push error.',\n        response,\n      );\n      const groupedMutationIDs = groupBy(\n        response.mutationIDs ?? [],\n        m => m.clientID,\n      );\n      for (const [clientID, mutationIDs] of groupedMutationIDs) {\n        const client = this.#clients.get(clientID);\n        if (!client) {\n          continue;\n        }\n\n        // We do not resolve mutations on the client if the push fails\n        // as those mutations will be retried.\n        if ('error' in response) {\n          // This error code path will eventually be removed when we\n          // no longer support the legacy push error format.\n          const pushFailedBody: PushFailedBody =\n            response.error === 'http'\n              ? {\n                  kind: ErrorKind.PushFailed,\n                  origin: ErrorOrigin.ZeroCache,\n                  reason: ErrorReason.HTTP,\n                  status: response.status,\n                  bodyPreview: response.details,\n                  mutationIDs,\n                  message: `Fetch from API server returned non-OK status ${response.status}`,\n                }\n              : response.error === 'unsupportedPushVersion'\n                ? {\n                    kind: ErrorKind.PushFailed,\n                    origin: ErrorOrigin.Server,\n                    reason: ErrorReason.UnsupportedPushVersion,\n                    mutationIDs,\n                    message: `Unsupported push version`,\n                  }\n                : {\n                    kind: ErrorKind.PushFailed,\n                    origin: ErrorOrigin.Server,\n                    reason: ErrorReason.Internal,\n                    mutationIDs,\n                    message:\n                      response.error === 'zeroPusher'\n                        ? response.details\n                        : response.error === 'unsupportedSchemaVersion'\n                          ? 'Unsupported schema version'\n                          : 'An unknown error occurred while pushing to the API server',\n                  };\n\n          this.#failDownstream(client.downstream, pushFailedBody);\n          if (isPushAuthFailure(pushFailedBody)) {\n            this.#lc.debug?.('Auth failure detected in push response');\n            client.onAuthFailure?.();\n          }\n        } else if ('kind' in response) {\n          this.#failDownstream(client.downstream, response);\n          if (isPushAuthFailure(response)) {\n            this.#lc.debug?.('Auth failure detected in push response');\n            client.onAuthFailure?.();\n          }\n        } else {\n          unreachable(response);\n        }\n      }\n    } else {\n      // Look for mutations results that should cause us to terminate the connection\n      const groupedMutations = groupBy(response.mutations, m => m.id.clientID);\n      for (const [clientID, mutations] of groupedMutations) {\n        const client = this.#clients.get(clientID);\n        if (!client) {\n          continue;\n        }\n\n        let failure: PushFailedBody | undefined;\n        let i = 0;\n        for (; i < mutations.length; i++) {\n          const m = mutations[i];\n          if ('error' in m.result) {\n            this.#lc.warn?.(\n              'The server behind ZERO_MUTATE_URL returned a mutation error.',\n              m.result,\n            );\n          }\n          // This error code path will eventually be removed,\n          // keeping this for backwards compatibility, but the server\n          // should now return a PushFailedBody with the mutationIDs\n          if ('error' in m.result && m.result.error === 'oooMutation') {\n            failure = {\n              kind: ErrorKind.PushFailed,\n              origin: ErrorOrigin.Server,\n              reason: ErrorReason.OutOfOrderMutation,\n              message: 'mutation was out of order',\n              details: m.result.details,\n              mutationIDs: mutations.map(m => ({\n                clientID: m.id.clientID,\n                id: m.id.id,\n              })),\n            };\n            break;\n          }\n        }\n\n        if (failure && i < mutations.length - 1) {\n          this.#lc.warn?.(\n            'push-response contains mutations after a mutation which should fatal the connection',\n          );\n        }\n\n        if (failure) {\n          connectionTerminations.push(() =>\n            this.#failDownstream(client.downstream, failure),\n          );\n        }\n      }\n    }\n\n    connectionTerminations.forEach(cb => cb());\n  }\n\n  async #processPush(entry: PusherEntry): Promise<PushResponse> {\n    this.#customMutations.add(entry.push.mutations.length, {\n      clientGroupID: entry.push.clientGroupID,\n    });\n    this.#pushes.add(1, {\n      clientGroupID: entry.push.clientGroupID,\n    });\n\n    // Record custom mutations for telemetry\n    recordMutation('custom', entry.push.mutations.length);\n\n    const url =\n      this.#userPushURL ??\n      must(this.#pushURLs[0], 'ZERO_MUTATE_URL is not set');\n\n    this.#lc.debug?.(\n      'pushing to',\n      url,\n      'with',\n      entry.push.mutations.length,\n      'mutations',\n    );\n\n    let mutationIDs: MutationID[] = [];\n\n    try {\n      mutationIDs = entry.push.mutations.map(m => ({\n        id: m.id,\n        clientID: m.clientID,\n      }));\n\n      return await fetchFromAPIServer(\n        pushResponseSchema,\n        'push',\n        this.#lc,\n        url,\n        this.#pushURLPatterns,\n        {\n          appID: this.#config.app.id,\n          shardNum: this.#config.shard.num,\n        },\n        {\n          apiKey: this.#apiKey,\n          customHeaders: this.#userPushHeaders,\n          allowedClientHeaders: this.#allowedClientHeaders,\n          token: entry.auth,\n          cookie: entry.httpCookie,\n          origin: entry.origin,\n        },\n        entry.push,\n      );\n    } catch (e) {\n      if (isProtocolError(e) && e.errorBody.kind === ErrorKind.PushFailed) {\n        return {\n          ...e.errorBody,\n          mutationIDs,\n        } as const satisfies PushFailedBody;\n      }\n\n      return {\n        kind: ErrorKind.PushFailed,\n        origin: ErrorOrigin.ZeroCache,\n        reason: ErrorReason.Internal,\n        message: `Failed to push: ${getErrorMessage(e)}`,\n        mutationIDs,\n      } as const satisfies PushFailedBody;\n    }\n  }\n\n  #failDownstream(\n    downstream: Subscription<Downstream>,\n    errorBody: PushFailedBody,\n  ): void {\n    downstream.fail(new ProtocolErrorWithLevel(errorBody, 'warn'));\n  }\n}\n\nfunction isPushAuthFailure(errorBody: PushFailedBody): boolean {\n  return (\n    errorBody.reason === ErrorReason.HTTP &&\n    (errorBody.status === 401 || errorBody.status === 403)\n  );\n}\n\n/**\n * Pushes for different clientIDs could theoretically be interleaved.\n *\n * In order to do efficient batching to the user's API server,\n * we collect all pushes for the same clientID into a single push.\n */\nexport function combinePushes(\n  entries: readonly (PusherEntryOrStop | undefined)[],\n): [PusherEntry[], boolean] {\n  const pushesByClientID = new Map<string, PusherEntry[]>();\n\n  function collect() {\n    const ret: PusherEntry[] = [];\n    for (const entries of pushesByClientID.values()) {\n      const composite: PusherEntry = {\n        ...entries[0],\n        push: {\n          ...entries[0].push,\n          mutations: [],\n        },\n      };\n      ret.push(composite);\n      for (const entry of entries) {\n        assertAreCompatiblePushes(composite, entry);\n        composite.push.mutations.push(...entry.push.mutations);\n      }\n    }\n    return ret;\n  }\n\n  for (const entry of entries) {\n    if (entry === 'stop' || entry === undefined) {\n      return [collect(), true];\n    }\n\n    const {clientID} = entry;\n    const existing = pushesByClientID.get(clientID);\n    if (existing) {\n      existing.push(entry);\n    } else {\n      pushesByClientID.set(clientID, [entry]);\n    }\n  }\n\n  return [collect(), false] as const;\n}\n\n// These invariants should always be true for a given clientID.\n// If they are not, we have a bug in the code somewhere.\nfunction assertAreCompatiblePushes(left: PusherEntry, right: PusherEntry) {\n  assert(\n    left.clientID === right.clientID,\n    'clientID must be the same for all pushes',\n  );\n  assert(\n    left.auth === right.auth,\n    'auth must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.push.schemaVersion === right.push.schemaVersion,\n    'schemaVersion must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.push.pushVersion === right.push.pushVersion,\n    'pushVersion must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.httpCookie === right.httpCookie,\n    'httpCookie must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.origin === right.origin,\n    'origin must be the same for all pushes with the same clientID',\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoEA,IAAa,gBAAb,MAAsD;CACpD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,YACE,WACA,YACA,IACA,eACA;AACA,QAAA,SAAe;AACf,QAAA,KAAW,GAAG,YAAY,aAAa,gBAAgB;AACvD,QAAA,kBAAwB,WAAW,IAAI,IAAI,kBAAkB;AAC7D,QAAA,QAAc,IAAI,OAAO;AACzB,QAAA,SAAe,IAAI,WACjB,WACA,IACA,WAAW,KACX,WAAW,QACX,WAAW,sBACX,MAAA,MACD;AACD,OAAK,KAAK;AACV,QAAA,aAAmB;;CAGrB,IAAI,UAA8B;AAChC,SAAO,MAAA,OAAa,QAAQ;;CAG9B,eACE,UACA,MACA,aACA,iBACA,eACA;AACA,SAAO,MAAA,OAAa,eAClB,UACA,MACA,aACA,iBACA,cACD;;CAGH,YACE,UACA,MACA,MACA,YACA,QACsC;AACtC,MAAI,CAAC,MAAA,WAAiB,eACpB,cAAa,KAAA;AAEf,QAAA,MAAY,QAAQ;GAAC;GAAM;GAAM;GAAU;GAAY;GAAO,CAAC;AAE/D,SAAO,EACL,MAAM,MACP;;CAGH,MAAM,qBAAqB,QAAoB;EAC7C,MAAM,MAAM,MAAA,WAAiB,IAAI;AACjC,MAAI,CAAC,IAEH;EAGF,MAAM,cAAwB;GAC5B,eAAe,KAAK;GACpB,WAAW,CACT;IACE,MAAM;IACN,IAAI;IACJ,UAAU,OAAO;IACjB,MAAM;IACN,MAAM,CACJ;KACE,MAAM;KACN,eAAe,KAAK;KACpB,UAAU,OAAO;KACjB,gBAAgB,OAAO;KACxB,CACF;IACD,WAAW,KAAK,KAAK;IACtB,CACF;GACD,aAAa;GACb,WAAW,KAAK,KAAK;GACrB,WAAW,WAAW,KAAK,GAAG,GAAG,OAAO,SAAS,GAAG,OAAO;GAC5D;AAED,MAAI;AACF,SAAM,mBACJ,oBACA,QACA,MAAA,IACA,KACA,MAAA,iBACA;IAAC,OAAO,MAAA,OAAa,IAAI;IAAI,UAAU,MAAA,OAAa,MAAM;IAAI,EAC9D,EAAC,QAAQ,MAAA,WAAiB,QAAO,EACjC,YACD;WACM,GAAG;AACV,SAAA,GAAS,OAAO,mCAAmC,EACjD,OAAO,gBAAgB,EAAE,EAC1B,CAAC;;;CAIN,MAAM,sBAAsB,WAAqB;AAC/C,MAAI,UAAU,WAAW,EACvB;EAEF,MAAM,MAAM,MAAA,WAAiB,IAAI;AACjC,MAAI,CAAC,IAEH;EAGF,MAAM,cAAwB;GAC5B,eAAe,KAAK;GACpB,WAAW,CACT;IACE,MAAM;IACN,IAAI;IACJ,UAAU,UAAU;IACpB,MAAM;IACN,MAAM,CACJ;KACE,MAAM;KACN,eAAe,KAAK;KACpB;KACD,CACF;IACD,WAAW,KAAK,KAAK;IACtB,CACF;GACD,aAAa;GACb,WAAW,KAAK,KAAK;GACrB,WAAW,gBAAgB,KAAK,GAAG,GAAG,KAAK,KAAK;GACjD;AAED,MAAI;AACF,SAAM,mBACJ,oBACA,QACA,MAAA,IACA,KACA,MAAA,iBACA;IAAC,OAAO,MAAA,OAAa,IAAI;IAAI,UAAU,MAAA,OAAa,MAAM;IAAI,EAC9D,EAAC,QAAQ,MAAA,WAAiB,QAAO,EACjC,YACD;WACM,GAAG;AACV,SAAA,GAAS,OAAO,wCAAwC,EACtD,OAAO,gBAAgB,EAAE,EAC1B,CAAC;;;CAIN,MAAM;AACJ,SAAO,CAAC,MAAA,WAAiB,mCAAmC;AAC5D,IAAE,MAAA;;CAGJ,QAAQ;AACN,SAAO,CAAC,MAAA,WAAiB,mCAAmC;AAC5D,IAAE,MAAA;AACF,MAAI,MAAA,YAAkB,EACf,MAAK,MAAM;;CAIpB,UAAmB;AACjB,SAAO,MAAA,WAAiB;;CAG1B,MAAqB;AACnB,QAAA,UAAgB,MAAA,OAAa,KAAK;AAClC,SAAO,MAAA;;CAGT,OAAsB;AACpB,MAAI,MAAA,UACF,QAAO,KAAK,MAAA,SAAe,+BAA+B;AAE5D,QAAA,YAAkB;AAClB,QAAA,MAAY,QAAQ,OAAO;AAC3B,SAAO,KAAK,MAAA,SAAe,+BAA+B;;;;;;;AAiB9D,IAAM,aAAN,MAAiB;CACf;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAQA;CACA;CAEA,mBAA4B,mBAC1B,YACA,UACA,uCACD;CACD,UAAmB,mBACjB,YACA,UACA,2CACD;CAED,YACE,QACA,IACA,SACA,QACA,sBACA,OACA;AACA,QAAA,WAAiB;AACjB,QAAA,KAAW,GAAG,YAAY,aAAa,SAAS;AAChD,QAAA,kBAAwB,QAAQ,IAAI,kBAAkB;AACtD,QAAA,SAAe;AACf,QAAA,uBAA6B;AAC7B,QAAA,QAAc;AACd,QAAA,SAAe;AACf,QAAA,0BAAgB,IAAI,KAAK;;CAG3B,IAAI,UAAU;AACZ,SAAO,MAAA;;;;;;CAOT,eACE,UACA,MACA,aACA,iBACA,eACA;EACA,MAAM,WAAW,MAAA,QAAc,IAAI,SAAS;AAC5C,MAAI,YAAY,SAAS,SAAS,KAEhC,OAAM,IAAI,MAAM,qCAAqC;AAIvD,MAAI,SACF,UAAS,WAAW,QAAQ;AAI9B,MAAI,MAAA,gBAAsB,KAAA,GAAW;AAEnC,SAAA,cAAoB;AACpB,SAAA,kBAAwB;aAGpB,MAAA,gBAAsB,YACxB,OAAA,GAAS,OACP,iEACA;GACE;GACA,WAAW;GACX,gBAAgB,MAAA;GACjB,CACF;EAIL,MAAM,aAAa,aAAa,OAAmB,EACjD,eAAe;AACb,SAAA,QAAc,OAAO,SAAS;KAEjC,CAAC;AACF,QAAA,QAAc,IAAI,UAAU;GAAC;GAAM;GAAY;GAAc,CAAC;AAC9D,SAAO;;CAGT,MAAM,MAAM;AACV,WAAS;GAGP,MAAM,CAAC,QAAQ,aAAa,cAAc,CAF7B,MAAM,MAAA,MAAY,SAAS,EAES,GADpC,MAAA,MAAY,OAAO,CACyB,CAAC;AAC1D,QAAK,MAAM,QAAQ,QAAQ;IACzB,MAAM,WAAW,MAAM,MAAA,YAAkB,KAAK;AAC9C,UAAM,MAAA,gBAAsB,SAAS;;AAGvC,OAAI,UACF;;;;;;;;CAUN,iBAAiB,UAAwB;EACvC,MAAM,yBAAyC,EAAE;AAGjD,MAAI,UAAU,YAAY,WAAW,UAAU;AAC7C,SAAA,GAAS,OACP,4DACA,SACD;GACD,MAAM,qBAAqB,QACzB,SAAS,eAAe,EAAE,GAC1B,MAAK,EAAE,SACR;AACD,QAAK,MAAM,CAAC,UAAU,gBAAgB,oBAAoB;IACxD,MAAM,SAAS,MAAA,QAAc,IAAI,SAAS;AAC1C,QAAI,CAAC,OACH;AAKF,QAAI,WAAW,UAAU;KAGvB,MAAM,iBACJ,SAAS,UAAU,SACf;MACE,MAAM;MACN,QAAQ;MACR,QAAQ;MACR,QAAQ,SAAS;MACjB,aAAa,SAAS;MACtB;MACA,SAAS,gDAAgD,SAAS;MACnE,GACD,SAAS,UAAU,2BACjB;MACE,MAAM;MACN,QAAQ;MACR,QAAQ;MACR;MACA,SAAS;MACV,GACD;MACE,MAAM;MACN,QAAQ;MACR,QAAQ;MACR;MACA,SACE,SAAS,UAAU,eACf,SAAS,UACT,SAAS,UAAU,6BACjB,+BACA;MACT;AAET,WAAA,eAAqB,OAAO,YAAY,eAAe;AACvD,SAAI,kBAAkB,eAAe,EAAE;AACrC,YAAA,GAAS,QAAQ,yCAAyC;AAC1D,aAAO,iBAAiB;;eAEjB,UAAU,UAAU;AAC7B,WAAA,eAAqB,OAAO,YAAY,SAAS;AACjD,SAAI,kBAAkB,SAAS,EAAE;AAC/B,YAAA,GAAS,QAAQ,yCAAyC;AAC1D,aAAO,iBAAiB;;UAG1B,aAAY,SAAS;;SAGpB;GAEL,MAAM,mBAAmB,QAAQ,SAAS,YAAW,MAAK,EAAE,GAAG,SAAS;AACxE,QAAK,MAAM,CAAC,UAAU,cAAc,kBAAkB;IACpD,MAAM,SAAS,MAAA,QAAc,IAAI,SAAS;AAC1C,QAAI,CAAC,OACH;IAGF,IAAI;IACJ,IAAI,IAAI;AACR,WAAO,IAAI,UAAU,QAAQ,KAAK;KAChC,MAAM,IAAI,UAAU;AACpB,SAAI,WAAW,EAAE,OACf,OAAA,GAAS,OACP,gEACA,EAAE,OACH;AAKH,SAAI,WAAW,EAAE,UAAU,EAAE,OAAO,UAAU,eAAe;AAC3D,gBAAU;OACR,MAAM;OACN,QAAQ;OACR,QAAQ;OACR,SAAS;OACT,SAAS,EAAE,OAAO;OAClB,aAAa,UAAU,KAAI,OAAM;QAC/B,UAAU,EAAE,GAAG;QACf,IAAI,EAAE,GAAG;QACV,EAAE;OACJ;AACD;;;AAIJ,QAAI,WAAW,IAAI,UAAU,SAAS,EACpC,OAAA,GAAS,OACP,sFACD;AAGH,QAAI,QACF,wBAAuB,WACrB,MAAA,eAAqB,OAAO,YAAY,QAAQ,CACjD;;;AAKP,yBAAuB,SAAQ,OAAM,IAAI,CAAC;;CAG5C,OAAA,YAAmB,OAA2C;AAC5D,QAAA,gBAAsB,IAAI,MAAM,KAAK,UAAU,QAAQ,EACrD,eAAe,MAAM,KAAK,eAC3B,CAAC;AACF,QAAA,OAAa,IAAI,GAAG,EAClB,eAAe,MAAM,KAAK,eAC3B,CAAC;AAGF,iBAAe,UAAU,MAAM,KAAK,UAAU,OAAO;EAErD,MAAM,MACJ,MAAA,eACA,KAAK,MAAA,SAAe,IAAI,6BAA6B;AAEvD,QAAA,GAAS,QACP,cACA,KACA,QACA,MAAM,KAAK,UAAU,QACrB,YACD;EAED,IAAI,cAA4B,EAAE;AAElC,MAAI;AACF,iBAAc,MAAM,KAAK,UAAU,KAAI,OAAM;IAC3C,IAAI,EAAE;IACN,UAAU,EAAE;IACb,EAAE;AAEH,UAAO,MAAM,mBACX,oBACA,QACA,MAAA,IACA,KACA,MAAA,iBACA;IACE,OAAO,MAAA,OAAa,IAAI;IACxB,UAAU,MAAA,OAAa,MAAM;IAC9B,EACD;IACE,QAAQ,MAAA;IACR,eAAe,MAAA;IACf,sBAAsB,MAAA;IACtB,OAAO,MAAM;IACb,QAAQ,MAAM;IACd,QAAQ,MAAM;IACf,EACD,MAAM,KACP;WACM,GAAG;AACV,OAAI,gBAAgB,EAAE,IAAI,EAAE,UAAU,SAAS,aAC7C,QAAO;IACL,GAAG,EAAE;IACL;IACD;AAGH,UAAO;IACL,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,SAAS,mBAAmB,gBAAgB,EAAE;IAC9C;IACD;;;CAIL,gBACE,YACA,WACM;AACN,aAAW,KAAK,IAAI,uBAAuB,WAAW,OAAO,CAAC;;;AAIlE,SAAS,kBAAkB,WAAoC;AAC7D,QACE,UAAU,WAAW,WACpB,UAAU,WAAW,OAAO,UAAU,WAAW;;;;;;;;AAUtD,SAAgB,cACd,SAC0B;CAC1B,MAAM,mCAAmB,IAAI,KAA4B;CAEzD,SAAS,UAAU;EACjB,MAAM,MAAqB,EAAE;AAC7B,OAAK,MAAM,WAAW,iBAAiB,QAAQ,EAAE;GAC/C,MAAM,YAAyB;IAC7B,GAAG,QAAQ;IACX,MAAM;KACJ,GAAG,QAAQ,GAAG;KACd,WAAW,EAAE;KACd;IACF;AACD,OAAI,KAAK,UAAU;AACnB,QAAK,MAAM,SAAS,SAAS;AAC3B,8BAA0B,WAAW,MAAM;AAC3C,cAAU,KAAK,UAAU,KAAK,GAAG,MAAM,KAAK,UAAU;;;AAG1D,SAAO;;AAGT,MAAK,MAAM,SAAS,SAAS;AAC3B,MAAI,UAAU,UAAU,UAAU,KAAA,EAChC,QAAO,CAAC,SAAS,EAAE,KAAK;EAG1B,MAAM,EAAC,aAAY;EACnB,MAAM,WAAW,iBAAiB,IAAI,SAAS;AAC/C,MAAI,SACF,UAAS,KAAK,MAAM;MAEpB,kBAAiB,IAAI,UAAU,CAAC,MAAM,CAAC;;AAI3C,QAAO,CAAC,SAAS,EAAE,MAAM;;AAK3B,SAAS,0BAA0B,MAAmB,OAAoB;AACxE,QACE,KAAK,aAAa,MAAM,UACxB,2CACD;AACD,QACE,KAAK,SAAS,MAAM,MACpB,8DACD;AACD,QACE,KAAK,KAAK,kBAAkB,MAAM,KAAK,eACvC,uEACD;AACD,QACE,KAAK,KAAK,gBAAgB,MAAM,KAAK,aACrC,qEACD;AACD,QACE,KAAK,eAAe,MAAM,YAC1B,oEACD;AACD,QACE,KAAK,WAAW,MAAM,QACtB,gEACD"}
+{"version":3,"file":"pusher.js","names":["#pusher","#queue","#pushConfig","#config","#lc","#pushURLPatterns","#isStopped","#refCount","#stopped","#pushURLs","#apiKey","#allowedClientHeaders","#clients","#customMutations","#pushes","#userPushURL","#userPushHeaders","#processPush","#fanOutResponses","#failDownstream"],"sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {groupBy} from '../../../../shared/src/arrays.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {getErrorMessage} from '../../../../shared/src/error.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport type {Downstream} from '../../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../../../zero-protocol/src/error-reason.ts';\nimport {\n  isProtocolError,\n  type PushFailedBody,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  CLEANUP_RESULTS_MUTATION_NAME,\n  pushResponseSchema,\n  type MutationID,\n  type PushBody,\n  type PushResponse,\n} from '../../../../zero-protocol/src/push.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport {compileUrlPattern, fetchFromAPIServer} from '../../custom/fetch.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport {ProtocolErrorWithLevel} from '../../types/error-with-level.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {HandlerResult, StreamResult} from '../../workers/connection.ts';\nimport type {RefCountedService, Service} from '../service.ts';\n\nexport interface Pusher extends RefCountedService {\n  readonly pushURL: string | undefined;\n\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n    userPushHeaders: Record<string, string> | undefined,\n    onAuthFailure?: (() => void) | undefined,\n  ): Source<Downstream>;\n  enqueuePush(\n    clientID: string,\n    push: PushBody,\n    auth: string | undefined,\n    httpCookie: string | undefined,\n    origin: string | undefined,\n  ): HandlerResult;\n  ackMutationResponses(upToID: MutationID): Promise<void>;\n  deleteClientMutations(clientIDs: string[]): Promise<void>;\n}\n\ntype Config = Pick<ZeroConfig, 'app' | 'shard'>;\n\n/**\n * Receives push messages from zero-client and forwards\n * them the the user's API server.\n *\n * If the user's API server is taking too long to process\n * the push, the PusherService will add the push to a queue\n * and send pushes in bulk the next time the user's API server\n * is available.\n *\n * - One PusherService exists per client group.\n * - Mutations for a given client are always sent in-order\n * - Mutations for different clients in the same group may be interleaved\n */\nexport class PusherService implements Service, Pusher {\n  readonly id: string;\n  readonly #pusher: PushWorker;\n  readonly #queue: Queue<PusherEntryOrStop>;\n  readonly #pushConfig: ZeroConfig['push'] & {url: string[]};\n  readonly #config: Config;\n  readonly #lc: LogContext;\n  readonly #pushURLPatterns: URLPattern[];\n  #stopped: Promise<void> | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  constructor(\n    appConfig: Config,\n    pushConfig: ZeroConfig['push'] & {url: string[]},\n    lc: LogContext,\n    clientGroupID: string,\n  ) {\n    this.#config = appConfig;\n    this.#lc = lc.withContext('component', 'pusherService');\n    this.#pushURLPatterns = pushConfig.url.map(compileUrlPattern);\n    this.#queue = new Queue();\n    this.#pusher = new PushWorker(\n      appConfig,\n      lc,\n      pushConfig.url,\n      pushConfig.apiKey,\n      pushConfig.allowedClientHeaders,\n      this.#queue,\n    );\n    this.id = clientGroupID;\n    this.#pushConfig = pushConfig;\n  }\n\n  get pushURL(): string | undefined {\n    return this.#pusher.pushURLs[0];\n  }\n\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n    userPushHeaders: Record<string, string> | undefined,\n    onAuthFailure?: (() => void) | undefined,\n  ) {\n    return this.#pusher.initConnection(\n      clientID,\n      wsID,\n      userPushURL,\n      userPushHeaders,\n      onAuthFailure,\n    );\n  }\n\n  enqueuePush(\n    clientID: string,\n    push: PushBody,\n    auth: string | undefined,\n    httpCookie: string | undefined,\n    origin: string | undefined,\n  ): Exclude<HandlerResult, StreamResult> {\n    if (!this.#pushConfig.forwardCookies) {\n      httpCookie = undefined; // remove cookies if not forwarded\n    }\n    this.#queue.enqueue({push, auth, clientID, httpCookie, origin});\n\n    return {\n      type: 'ok',\n    };\n  }\n\n  async ackMutationResponses(upToID: MutationID) {\n    const url = this.#pusher.effectivePushURL;\n    if (!url) {\n      // No push URL configured, skip cleanup\n      return;\n    }\n\n    const cleanupBody: PushBody = {\n      clientGroupID: this.id,\n      mutations: [\n        {\n          type: MutationType.Custom,\n          id: 0, // Not tracked - this is fire-and-forget\n          clientID: upToID.clientID,\n          name: CLEANUP_RESULTS_MUTATION_NAME,\n          args: [\n            {\n              type: 'single',\n              clientGroupID: this.id,\n              clientID: upToID.clientID,\n              upToMutationID: upToID.id,\n            },\n          ],\n          timestamp: Date.now(),\n        },\n      ],\n      pushVersion: 1,\n      timestamp: Date.now(),\n      requestID: `cleanup-${this.id}-${upToID.clientID}-${upToID.id}`,\n    };\n\n    try {\n      await fetchFromAPIServer(\n        pushResponseSchema,\n        'push',\n        this.#lc,\n        url,\n        this.#pushURLPatterns,\n        {appID: this.#config.app.id, shardNum: this.#config.shard.num},\n        {apiKey: this.#pushConfig.apiKey},\n        cleanupBody,\n      );\n    } catch (e) {\n      this.#lc.warn?.('Failed to send cleanup mutation', {\n        error: getErrorMessage(e),\n      });\n    }\n  }\n\n  async deleteClientMutations(clientIDs: string[]) {\n    if (clientIDs.length === 0) {\n      return;\n    }\n    const url = this.#pusher.effectivePushURL;\n    if (!url) {\n      // No push URL configured, skip cleanup\n      return;\n    }\n\n    const cleanupBody: PushBody = {\n      clientGroupID: this.id,\n      mutations: [\n        {\n          type: MutationType.Custom,\n          id: 0, // Not tracked - this is fire-and-forget\n          clientID: clientIDs[0], // Use first client as sender\n          name: CLEANUP_RESULTS_MUTATION_NAME,\n          args: [\n            {\n              type: 'bulk',\n              clientGroupID: this.id,\n              clientIDs,\n            },\n          ],\n          timestamp: Date.now(),\n        },\n      ],\n      pushVersion: 1,\n      timestamp: Date.now(),\n      requestID: `cleanup-bulk-${this.id}-${Date.now()}`,\n    };\n\n    try {\n      await fetchFromAPIServer(\n        pushResponseSchema,\n        'push',\n        this.#lc,\n        url,\n        this.#pushURLPatterns,\n        {appID: this.#config.app.id, shardNum: this.#config.shard.num},\n        {apiKey: this.#pushConfig.apiKey},\n        cleanupBody,\n      );\n    } catch (e) {\n      this.#lc.warn?.('Failed to send bulk cleanup mutation', {\n        error: getErrorMessage(e),\n      });\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'PusherService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'PusherService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  run(): Promise<void> {\n    this.#stopped = this.#pusher.run();\n    return this.#stopped;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return must(this.#stopped, 'Stop was called before `run`');\n    }\n    this.#isStopped = true;\n    this.#queue.enqueue('stop');\n    return must(this.#stopped, 'Stop was called before `run`');\n  }\n}\n\ntype PusherEntry = {\n  push: PushBody;\n  auth: string | undefined;\n  httpCookie: string | undefined;\n  origin: string | undefined;\n  clientID: string;\n};\ntype PusherEntryOrStop = PusherEntry | 'stop';\n\n/**\n * Awaits items in the queue then drains and sends them all\n * to the user's API server.\n */\nclass PushWorker {\n  readonly #pushURLs: string[];\n  readonly #pushURLPatterns: URLPattern[];\n  readonly #apiKey: string | undefined;\n  readonly #allowedClientHeaders: readonly string[] | undefined;\n  readonly #queue: Queue<PusherEntryOrStop>;\n  readonly #lc: LogContext;\n  readonly #config: Config;\n  readonly #clients: Map<\n    string,\n    {\n      wsID: string;\n      downstream: Subscription<Downstream>;\n      onAuthFailure: (() => void) | undefined;\n    }\n  >;\n  #userPushURL?: string | undefined;\n  #userPushHeaders?: Record<string, string> | undefined;\n\n  readonly #customMutations = getOrCreateCounter(\n    'mutation',\n    'custom',\n    'Number of custom mutations processed',\n  );\n  readonly #pushes = getOrCreateCounter(\n    'mutation',\n    'pushes',\n    'Number of pushes processed by the pusher',\n  );\n\n  constructor(\n    config: Config,\n    lc: LogContext,\n    pushURL: string[],\n    apiKey: string | undefined,\n    allowedClientHeaders: readonly string[] | undefined,\n    queue: Queue<PusherEntryOrStop>,\n  ) {\n    this.#pushURLs = pushURL;\n    this.#lc = lc.withContext('component', 'pusher');\n    this.#pushURLPatterns = pushURL.map(compileUrlPattern);\n    this.#apiKey = apiKey;\n    this.#allowedClientHeaders = allowedClientHeaders;\n    this.#queue = queue;\n    this.#config = config;\n    this.#clients = new Map();\n  }\n\n  get pushURLs() {\n    return this.#pushURLs;\n  }\n\n  get effectivePushURL(): string | undefined {\n    return this.#userPushURL ?? this.#pushURLs[0];\n  }\n\n  /**\n   * Returns a new downstream stream if the clientID,wsID pair has not been seen before.\n   * If a clientID already exists with a different wsID, that client's downstream is cancelled.\n   */\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n    userPushHeaders: Record<string, string> | undefined,\n    onAuthFailure?: (() => void) | undefined,\n  ) {\n    const existing = this.#clients.get(clientID);\n    if (existing && existing.wsID === wsID) {\n      // already initialized for this socket\n      throw new Error('Connection was already initialized');\n    }\n\n    // client is back on a new connection\n    if (existing) {\n      existing.downstream.cancel();\n    }\n\n    // Handle client group level URL parameters\n    if (this.#userPushURL === undefined) {\n      // First client in the group - store its URL and headers\n      this.#userPushURL = userPushURL;\n      this.#userPushHeaders = userPushHeaders;\n    } else {\n      // Validate that subsequent clients have compatible parameters\n      if (this.#userPushURL !== userPushURL) {\n        this.#lc.warn?.(\n          'Client provided different mutate parameters than client group',\n          {\n            clientID,\n            clientURL: userPushURL,\n            clientGroupURL: this.#userPushURL,\n          },\n        );\n      }\n    }\n\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => {\n        this.#clients.delete(clientID);\n      },\n    });\n    this.#clients.set(clientID, {wsID, downstream, onAuthFailure});\n    return downstream;\n  }\n\n  async run() {\n    for (;;) {\n      const task = await this.#queue.dequeue();\n      const rest = this.#queue.drain();\n      const [pushes, terminate] = combinePushes([task, ...rest]);\n      for (const push of pushes) {\n        const response = await this.#processPush(push);\n        await this.#fanOutResponses(response);\n      }\n\n      if (terminate) {\n        break;\n      }\n    }\n  }\n\n  /**\n   * 1. If the entire `push` fails, we send the error to relevant clients.\n   * 2. If the push succeeds, we look for any mutation failure that should cause the connection to terminate\n   *  and terminate the connection for those clients.\n   */\n  #fanOutResponses(response: PushResponse) {\n    const connectionTerminations: (() => void)[] = [];\n\n    // if the entire push failed, send that to the client.\n    if ('kind' in response || 'error' in response) {\n      this.#lc.warn?.(\n        'The server behind ZERO_MUTATE_URL returned a push error.',\n        response,\n      );\n      const groupedMutationIDs = groupBy(\n        response.mutationIDs ?? [],\n        m => m.clientID,\n      );\n      for (const [clientID, mutationIDs] of groupedMutationIDs) {\n        const client = this.#clients.get(clientID);\n        if (!client) {\n          continue;\n        }\n\n        // We do not resolve mutations on the client if the push fails\n        // as those mutations will be retried.\n        if ('error' in response) {\n          // This error code path will eventually be removed when we\n          // no longer support the legacy push error format.\n          const pushFailedBody: PushFailedBody =\n            response.error === 'http'\n              ? {\n                  kind: ErrorKind.PushFailed,\n                  origin: ErrorOrigin.ZeroCache,\n                  reason: ErrorReason.HTTP,\n                  status: response.status,\n                  bodyPreview: response.details,\n                  mutationIDs,\n                  message: `Fetch from API server returned non-OK status ${response.status}`,\n                }\n              : response.error === 'unsupportedPushVersion'\n                ? {\n                    kind: ErrorKind.PushFailed,\n                    origin: ErrorOrigin.Server,\n                    reason: ErrorReason.UnsupportedPushVersion,\n                    mutationIDs,\n                    message: `Unsupported push version`,\n                  }\n                : {\n                    kind: ErrorKind.PushFailed,\n                    origin: ErrorOrigin.Server,\n                    reason: ErrorReason.Internal,\n                    mutationIDs,\n                    message:\n                      response.error === 'zeroPusher'\n                        ? response.details\n                        : response.error === 'unsupportedSchemaVersion'\n                          ? 'Unsupported schema version'\n                          : 'An unknown error occurred while pushing to the API server',\n                  };\n\n          this.#failDownstream(client.downstream, pushFailedBody);\n          if (isPushAuthFailure(pushFailedBody)) {\n            this.#lc.debug?.('Auth failure detected in push response');\n            client.onAuthFailure?.();\n          }\n        } else if ('kind' in response) {\n          this.#failDownstream(client.downstream, response);\n          if (isPushAuthFailure(response)) {\n            this.#lc.debug?.('Auth failure detected in push response');\n            client.onAuthFailure?.();\n          }\n        } else {\n          unreachable(response);\n        }\n      }\n    } else {\n      // Look for mutations results that should cause us to terminate the connection\n      const groupedMutations = groupBy(response.mutations, m => m.id.clientID);\n      for (const [clientID, mutations] of groupedMutations) {\n        const client = this.#clients.get(clientID);\n        if (!client) {\n          continue;\n        }\n\n        let failure: PushFailedBody | undefined;\n        let i = 0;\n        for (; i < mutations.length; i++) {\n          const m = mutations[i];\n          if ('error' in m.result) {\n            this.#lc.warn?.(\n              'The server behind ZERO_MUTATE_URL returned a mutation error.',\n              m.result,\n            );\n          }\n          // This error code path will eventually be removed,\n          // keeping this for backwards compatibility, but the server\n          // should now return a PushFailedBody with the mutationIDs\n          if ('error' in m.result && m.result.error === 'oooMutation') {\n            failure = {\n              kind: ErrorKind.PushFailed,\n              origin: ErrorOrigin.Server,\n              reason: ErrorReason.OutOfOrderMutation,\n              message: 'mutation was out of order',\n              details: m.result.details,\n              mutationIDs: mutations.map(m => ({\n                clientID: m.id.clientID,\n                id: m.id.id,\n              })),\n            };\n            break;\n          }\n        }\n\n        if (failure && i < mutations.length - 1) {\n          this.#lc.warn?.(\n            'push-response contains mutations after a mutation which should fatal the connection',\n          );\n        }\n\n        if (failure) {\n          connectionTerminations.push(() =>\n            this.#failDownstream(client.downstream, failure),\n          );\n        }\n      }\n    }\n\n    connectionTerminations.forEach(cb => cb());\n  }\n\n  async #processPush(entry: PusherEntry): Promise<PushResponse> {\n    this.#customMutations.add(entry.push.mutations.length, {\n      clientGroupID: entry.push.clientGroupID,\n    });\n    this.#pushes.add(1, {\n      clientGroupID: entry.push.clientGroupID,\n    });\n\n    // Record custom mutations for telemetry\n    recordMutation('custom', entry.push.mutations.length);\n\n    const url =\n      this.#userPushURL ??\n      must(this.#pushURLs[0], 'ZERO_MUTATE_URL is not set');\n\n    this.#lc.debug?.(\n      'pushing to',\n      url,\n      'with',\n      entry.push.mutations.length,\n      'mutations',\n    );\n\n    let mutationIDs: MutationID[] = [];\n\n    try {\n      mutationIDs = entry.push.mutations.map(m => ({\n        id: m.id,\n        clientID: m.clientID,\n      }));\n\n      return await fetchFromAPIServer(\n        pushResponseSchema,\n        'push',\n        this.#lc,\n        url,\n        this.#pushURLPatterns,\n        {\n          appID: this.#config.app.id,\n          shardNum: this.#config.shard.num,\n        },\n        {\n          apiKey: this.#apiKey,\n          customHeaders: this.#userPushHeaders,\n          allowedClientHeaders: this.#allowedClientHeaders,\n          token: entry.auth,\n          cookie: entry.httpCookie,\n          origin: entry.origin,\n        },\n        entry.push,\n      );\n    } catch (e) {\n      if (isProtocolError(e) && e.errorBody.kind === ErrorKind.PushFailed) {\n        return {\n          ...e.errorBody,\n          mutationIDs,\n        } as const satisfies PushFailedBody;\n      }\n\n      return {\n        kind: ErrorKind.PushFailed,\n        origin: ErrorOrigin.ZeroCache,\n        reason: ErrorReason.Internal,\n        message: `Failed to push: ${getErrorMessage(e)}`,\n        mutationIDs,\n      } as const satisfies PushFailedBody;\n    }\n  }\n\n  #failDownstream(\n    downstream: Subscription<Downstream>,\n    errorBody: PushFailedBody,\n  ): void {\n    downstream.fail(new ProtocolErrorWithLevel(errorBody, 'warn'));\n  }\n}\n\nfunction isPushAuthFailure(errorBody: PushFailedBody): boolean {\n  return (\n    errorBody.reason === ErrorReason.HTTP &&\n    (errorBody.status === 401 || errorBody.status === 403)\n  );\n}\n\n/**\n * Pushes for different clientIDs could theoretically be interleaved.\n *\n * In order to do efficient batching to the user's API server,\n * we collect all pushes for the same clientID into a single push.\n */\nexport function combinePushes(\n  entries: readonly (PusherEntryOrStop | undefined)[],\n): [PusherEntry[], boolean] {\n  const pushesByClientID = new Map<string, PusherEntry[]>();\n\n  function collect() {\n    const ret: PusherEntry[] = [];\n    for (const entries of pushesByClientID.values()) {\n      const composite: PusherEntry = {\n        ...entries[0],\n        push: {\n          ...entries[0].push,\n          mutations: [],\n        },\n      };\n      ret.push(composite);\n      for (const entry of entries) {\n        assertAreCompatiblePushes(composite, entry);\n        composite.push.mutations.push(...entry.push.mutations);\n      }\n    }\n    return ret;\n  }\n\n  for (const entry of entries) {\n    if (entry === 'stop' || entry === undefined) {\n      return [collect(), true];\n    }\n\n    const {clientID} = entry;\n    const existing = pushesByClientID.get(clientID);\n    if (existing) {\n      existing.push(entry);\n    } else {\n      pushesByClientID.set(clientID, [entry]);\n    }\n  }\n\n  return [collect(), false] as const;\n}\n\n// These invariants should always be true for a given clientID.\n// If they are not, we have a bug in the code somewhere.\nfunction assertAreCompatiblePushes(left: PusherEntry, right: PusherEntry) {\n  assert(\n    left.clientID === right.clientID,\n    'clientID must be the same for all pushes',\n  );\n  assert(\n    left.auth === right.auth,\n    'auth must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.push.schemaVersion === right.push.schemaVersion,\n    'schemaVersion must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.push.pushVersion === right.push.pushVersion,\n    'pushVersion must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.httpCookie === right.httpCookie,\n    'httpCookie must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.origin === right.origin,\n    'origin must be the same for all pushes with the same clientID',\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoEA,IAAa,gBAAb,MAAsD;CACpD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,YACE,WACA,YACA,IACA,eACA;AACA,QAAA,SAAe;AACf,QAAA,KAAW,GAAG,YAAY,aAAa,gBAAgB;AACvD,QAAA,kBAAwB,WAAW,IAAI,IAAI,kBAAkB;AAC7D,QAAA,QAAc,IAAI,OAAO;AACzB,QAAA,SAAe,IAAI,WACjB,WACA,IACA,WAAW,KACX,WAAW,QACX,WAAW,sBACX,MAAA,MACD;AACD,OAAK,KAAK;AACV,QAAA,aAAmB;;CAGrB,IAAI,UAA8B;AAChC,SAAO,MAAA,OAAa,SAAS;;CAG/B,eACE,UACA,MACA,aACA,iBACA,eACA;AACA,SAAO,MAAA,OAAa,eAClB,UACA,MACA,aACA,iBACA,cACD;;CAGH,YACE,UACA,MACA,MACA,YACA,QACsC;AACtC,MAAI,CAAC,MAAA,WAAiB,eACpB,cAAa,KAAA;AAEf,QAAA,MAAY,QAAQ;GAAC;GAAM;GAAM;GAAU;GAAY;GAAO,CAAC;AAE/D,SAAO,EACL,MAAM,MACP;;CAGH,MAAM,qBAAqB,QAAoB;EAC7C,MAAM,MAAM,MAAA,OAAa;AACzB,MAAI,CAAC,IAEH;EAGF,MAAM,cAAwB;GAC5B,eAAe,KAAK;GACpB,WAAW,CACT;IACE,MAAM;IACN,IAAI;IACJ,UAAU,OAAO;IACjB,MAAM;IACN,MAAM,CACJ;KACE,MAAM;KACN,eAAe,KAAK;KACpB,UAAU,OAAO;KACjB,gBAAgB,OAAO;KACxB,CACF;IACD,WAAW,KAAK,KAAK;IACtB,CACF;GACD,aAAa;GACb,WAAW,KAAK,KAAK;GACrB,WAAW,WAAW,KAAK,GAAG,GAAG,OAAO,SAAS,GAAG,OAAO;GAC5D;AAED,MAAI;AACF,SAAM,mBACJ,oBACA,QACA,MAAA,IACA,KACA,MAAA,iBACA;IAAC,OAAO,MAAA,OAAa,IAAI;IAAI,UAAU,MAAA,OAAa,MAAM;IAAI,EAC9D,EAAC,QAAQ,MAAA,WAAiB,QAAO,EACjC,YACD;WACM,GAAG;AACV,SAAA,GAAS,OAAO,mCAAmC,EACjD,OAAO,gBAAgB,EAAE,EAC1B,CAAC;;;CAIN,MAAM,sBAAsB,WAAqB;AAC/C,MAAI,UAAU,WAAW,EACvB;EAEF,MAAM,MAAM,MAAA,OAAa;AACzB,MAAI,CAAC,IAEH;EAGF,MAAM,cAAwB;GAC5B,eAAe,KAAK;GACpB,WAAW,CACT;IACE,MAAM;IACN,IAAI;IACJ,UAAU,UAAU;IACpB,MAAM;IACN,MAAM,CACJ;KACE,MAAM;KACN,eAAe,KAAK;KACpB;KACD,CACF;IACD,WAAW,KAAK,KAAK;IACtB,CACF;GACD,aAAa;GACb,WAAW,KAAK,KAAK;GACrB,WAAW,gBAAgB,KAAK,GAAG,GAAG,KAAK,KAAK;GACjD;AAED,MAAI;AACF,SAAM,mBACJ,oBACA,QACA,MAAA,IACA,KACA,MAAA,iBACA;IAAC,OAAO,MAAA,OAAa,IAAI;IAAI,UAAU,MAAA,OAAa,MAAM;IAAI,EAC9D,EAAC,QAAQ,MAAA,WAAiB,QAAO,EACjC,YACD;WACM,GAAG;AACV,SAAA,GAAS,OAAO,wCAAwC,EACtD,OAAO,gBAAgB,EAAE,EAC1B,CAAC;;;CAIN,MAAM;AACJ,SAAO,CAAC,MAAA,WAAiB,mCAAmC;AAC5D,IAAE,MAAA;;CAGJ,QAAQ;AACN,SAAO,CAAC,MAAA,WAAiB,mCAAmC;AAC5D,IAAE,MAAA;AACF,MAAI,MAAA,YAAkB,EACf,MAAK,MAAM;;CAIpB,UAAmB;AACjB,SAAO,MAAA,WAAiB;;CAG1B,MAAqB;AACnB,QAAA,UAAgB,MAAA,OAAa,KAAK;AAClC,SAAO,MAAA;;CAGT,OAAsB;AACpB,MAAI,MAAA,UACF,QAAO,KAAK,MAAA,SAAe,+BAA+B;AAE5D,QAAA,YAAkB;AAClB,QAAA,MAAY,QAAQ,OAAO;AAC3B,SAAO,KAAK,MAAA,SAAe,+BAA+B;;;;;;;AAiB9D,IAAM,aAAN,MAAiB;CACf;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAQA;CACA;CAEA,mBAA4B,mBAC1B,YACA,UACA,uCACD;CACD,UAAmB,mBACjB,YACA,UACA,2CACD;CAED,YACE,QACA,IACA,SACA,QACA,sBACA,OACA;AACA,QAAA,WAAiB;AACjB,QAAA,KAAW,GAAG,YAAY,aAAa,SAAS;AAChD,QAAA,kBAAwB,QAAQ,IAAI,kBAAkB;AACtD,QAAA,SAAe;AACf,QAAA,uBAA6B;AAC7B,QAAA,QAAc;AACd,QAAA,SAAe;AACf,QAAA,0BAAgB,IAAI,KAAK;;CAG3B,IAAI,WAAW;AACb,SAAO,MAAA;;CAGT,IAAI,mBAAuC;AACzC,SAAO,MAAA,eAAqB,MAAA,SAAe;;;;;;CAO7C,eACE,UACA,MACA,aACA,iBACA,eACA;EACA,MAAM,WAAW,MAAA,QAAc,IAAI,SAAS;AAC5C,MAAI,YAAY,SAAS,SAAS,KAEhC,OAAM,IAAI,MAAM,qCAAqC;AAIvD,MAAI,SACF,UAAS,WAAW,QAAQ;AAI9B,MAAI,MAAA,gBAAsB,KAAA,GAAW;AAEnC,SAAA,cAAoB;AACpB,SAAA,kBAAwB;aAGpB,MAAA,gBAAsB,YACxB,OAAA,GAAS,OACP,iEACA;GACE;GACA,WAAW;GACX,gBAAgB,MAAA;GACjB,CACF;EAIL,MAAM,aAAa,aAAa,OAAmB,EACjD,eAAe;AACb,SAAA,QAAc,OAAO,SAAS;KAEjC,CAAC;AACF,QAAA,QAAc,IAAI,UAAU;GAAC;GAAM;GAAY;GAAc,CAAC;AAC9D,SAAO;;CAGT,MAAM,MAAM;AACV,WAAS;GAGP,MAAM,CAAC,QAAQ,aAAa,cAAc,CAF7B,MAAM,MAAA,MAAY,SAAS,EAES,GADpC,MAAA,MAAY,OAAO,CACyB,CAAC;AAC1D,QAAK,MAAM,QAAQ,QAAQ;IACzB,MAAM,WAAW,MAAM,MAAA,YAAkB,KAAK;AAC9C,UAAM,MAAA,gBAAsB,SAAS;;AAGvC,OAAI,UACF;;;;;;;;CAUN,iBAAiB,UAAwB;EACvC,MAAM,yBAAyC,EAAE;AAGjD,MAAI,UAAU,YAAY,WAAW,UAAU;AAC7C,SAAA,GAAS,OACP,4DACA,SACD;GACD,MAAM,qBAAqB,QACzB,SAAS,eAAe,EAAE,GAC1B,MAAK,EAAE,SACR;AACD,QAAK,MAAM,CAAC,UAAU,gBAAgB,oBAAoB;IACxD,MAAM,SAAS,MAAA,QAAc,IAAI,SAAS;AAC1C,QAAI,CAAC,OACH;AAKF,QAAI,WAAW,UAAU;KAGvB,MAAM,iBACJ,SAAS,UAAU,SACf;MACE,MAAM;MACN,QAAQ;MACR,QAAQ;MACR,QAAQ,SAAS;MACjB,aAAa,SAAS;MACtB;MACA,SAAS,gDAAgD,SAAS;MACnE,GACD,SAAS,UAAU,2BACjB;MACE,MAAM;MACN,QAAQ;MACR,QAAQ;MACR;MACA,SAAS;MACV,GACD;MACE,MAAM;MACN,QAAQ;MACR,QAAQ;MACR;MACA,SACE,SAAS,UAAU,eACf,SAAS,UACT,SAAS,UAAU,6BACjB,+BACA;MACT;AAET,WAAA,eAAqB,OAAO,YAAY,eAAe;AACvD,SAAI,kBAAkB,eAAe,EAAE;AACrC,YAAA,GAAS,QAAQ,yCAAyC;AAC1D,aAAO,iBAAiB;;eAEjB,UAAU,UAAU;AAC7B,WAAA,eAAqB,OAAO,YAAY,SAAS;AACjD,SAAI,kBAAkB,SAAS,EAAE;AAC/B,YAAA,GAAS,QAAQ,yCAAyC;AAC1D,aAAO,iBAAiB;;UAG1B,aAAY,SAAS;;SAGpB;GAEL,MAAM,mBAAmB,QAAQ,SAAS,YAAW,MAAK,EAAE,GAAG,SAAS;AACxE,QAAK,MAAM,CAAC,UAAU,cAAc,kBAAkB;IACpD,MAAM,SAAS,MAAA,QAAc,IAAI,SAAS;AAC1C,QAAI,CAAC,OACH;IAGF,IAAI;IACJ,IAAI,IAAI;AACR,WAAO,IAAI,UAAU,QAAQ,KAAK;KAChC,MAAM,IAAI,UAAU;AACpB,SAAI,WAAW,EAAE,OACf,OAAA,GAAS,OACP,gEACA,EAAE,OACH;AAKH,SAAI,WAAW,EAAE,UAAU,EAAE,OAAO,UAAU,eAAe;AAC3D,gBAAU;OACR,MAAM;OACN,QAAQ;OACR,QAAQ;OACR,SAAS;OACT,SAAS,EAAE,OAAO;OAClB,aAAa,UAAU,KAAI,OAAM;QAC/B,UAAU,EAAE,GAAG;QACf,IAAI,EAAE,GAAG;QACV,EAAE;OACJ;AACD;;;AAIJ,QAAI,WAAW,IAAI,UAAU,SAAS,EACpC,OAAA,GAAS,OACP,sFACD;AAGH,QAAI,QACF,wBAAuB,WACrB,MAAA,eAAqB,OAAO,YAAY,QAAQ,CACjD;;;AAKP,yBAAuB,SAAQ,OAAM,IAAI,CAAC;;CAG5C,OAAA,YAAmB,OAA2C;AAC5D,QAAA,gBAAsB,IAAI,MAAM,KAAK,UAAU,QAAQ,EACrD,eAAe,MAAM,KAAK,eAC3B,CAAC;AACF,QAAA,OAAa,IAAI,GAAG,EAClB,eAAe,MAAM,KAAK,eAC3B,CAAC;AAGF,iBAAe,UAAU,MAAM,KAAK,UAAU,OAAO;EAErD,MAAM,MACJ,MAAA,eACA,KAAK,MAAA,SAAe,IAAI,6BAA6B;AAEvD,QAAA,GAAS,QACP,cACA,KACA,QACA,MAAM,KAAK,UAAU,QACrB,YACD;EAED,IAAI,cAA4B,EAAE;AAElC,MAAI;AACF,iBAAc,MAAM,KAAK,UAAU,KAAI,OAAM;IAC3C,IAAI,EAAE;IACN,UAAU,EAAE;IACb,EAAE;AAEH,UAAO,MAAM,mBACX,oBACA,QACA,MAAA,IACA,KACA,MAAA,iBACA;IACE,OAAO,MAAA,OAAa,IAAI;IACxB,UAAU,MAAA,OAAa,MAAM;IAC9B,EACD;IACE,QAAQ,MAAA;IACR,eAAe,MAAA;IACf,sBAAsB,MAAA;IACtB,OAAO,MAAM;IACb,QAAQ,MAAM;IACd,QAAQ,MAAM;IACf,EACD,MAAM,KACP;WACM,GAAG;AACV,OAAI,gBAAgB,EAAE,IAAI,EAAE,UAAU,SAAS,aAC7C,QAAO;IACL,GAAG,EAAE;IACL;IACD;AAGH,UAAO;IACL,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,SAAS,mBAAmB,gBAAgB,EAAE;IAC9C;IACD;;;CAIL,gBACE,YACA,WACM;AACN,aAAW,KAAK,IAAI,uBAAuB,WAAW,OAAO,CAAC;;;AAIlE,SAAS,kBAAkB,WAAoC;AAC7D,QACE,UAAU,WAAW,WACpB,UAAU,WAAW,OAAO,UAAU,WAAW;;;;;;;;AAUtD,SAAgB,cACd,SAC0B;CAC1B,MAAM,mCAAmB,IAAI,KAA4B;CAEzD,SAAS,UAAU;EACjB,MAAM,MAAqB,EAAE;AAC7B,OAAK,MAAM,WAAW,iBAAiB,QAAQ,EAAE;GAC/C,MAAM,YAAyB;IAC7B,GAAG,QAAQ;IACX,MAAM;KACJ,GAAG,QAAQ,GAAG;KACd,WAAW,EAAE;KACd;IACF;AACD,OAAI,KAAK,UAAU;AACnB,QAAK,MAAM,SAAS,SAAS;AAC3B,8BAA0B,WAAW,MAAM;AAC3C,cAAU,KAAK,UAAU,KAAK,GAAG,MAAM,KAAK,UAAU;;;AAG1D,SAAO;;AAGT,MAAK,MAAM,SAAS,SAAS;AAC3B,MAAI,UAAU,UAAU,UAAU,KAAA,EAChC,QAAO,CAAC,SAAS,EAAE,KAAK;EAG1B,MAAM,EAAC,aAAY;EACnB,MAAM,WAAW,iBAAiB,IAAI,SAAS;AAC/C,MAAI,SACF,UAAS,KAAK,MAAM;MAEpB,kBAAiB,IAAI,UAAU,CAAC,MAAM,CAAC;;AAI3C,QAAO,CAAC,SAAS,EAAE,MAAM;;AAK3B,SAAS,0BAA0B,MAAmB,OAAoB;AACxE,QACE,KAAK,aAAa,MAAM,UACxB,2CACD;AACD,QACE,KAAK,SAAS,MAAM,MACpB,8DACD;AACD,QACE,KAAK,KAAK,kBAAkB,MAAM,KAAK,eACvC,uEACD;AACD,QACE,KAAK,KAAK,gBAAgB,MAAM,KAAK,aACrC,qEACD;AACD,QACE,KAAK,eAAe,MAAM,YAC1B,oEACD;AACD,QACE,KAAK,WAAW,MAAM,QACtB,gEACD"}

package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,EAEL,KAAK,cAAc,EAEpB,MAAM,uCAAuC,CAAC;AAI/C,OAAO,KAAK,EAAC,0BAA0B,EAAC,MAAM,yBAAyB,CAAC;AACxE,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEhE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;gBAoB1B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,eAAe,EAAE,0BAA0B,GAAG,IAAI;IAa9C,GAAG;IA6IT,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC;IAIjC,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO;CAGnC"}
+{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,EAGL,KAAK,cAAc,EAEpB,MAAM,uCAAuC,CAAC;AAK/C,OAAO,KAAK,EAAC,0BAA0B,EAAC,MAAM,yBAAyB,CAAC;AACxE,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAIhE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;gBAoB1B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,eAAe,EAAE,0BAA0B,GAAG,IAAI;IAa9C,GAAG;IAsJT,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC;IAIjC,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO;CAGnC"}

package/out/zero-cache/src/services/replicator/incremental-sync.js

@@ -1,6 +1,7 @@
+import { AbortError } from "../../../../shared/src/abort-error.js";
 import { RunningState } from "../running-state.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
-import "../change-streamer/change-streamer.js";
+import { errorTypeToReadableName } from "../change-streamer/change-streamer.js";
 import { Notifier } from "./notifier.js";
 import { ReplicationReportRecorder } from "./reporter/recorder.js";
 //#region ../zero-cache/src/services/replicator/incremental-sync.ts
@@ -74,9 +75,11 @@ var IncrementalSyncer = class {
 							}
 							break;
 						}
-						case "error":
-							this.stop(lc, message[1]);
+						case "error": {
+							const { type, message: msg } = message[1];
+							this.stop(lc, new AbortError(`${errorTypeToReadableName(type)}: ${msg}`));
 							break;
+						}
 						default: {
 							const msg = message[1];
 							if (msg.tag === "backfill" && msg.status) {

package/out/zero-cache/src/services/replicator/incremental-sync.js.map

@@ -1 +1 @@
-{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type Downstream,\n} from '../change-streamer/change-streamer.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport type {ReplicationStatusPublisher} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n  readonly #lc: LogContext;\n  readonly #taskID: string;\n  readonly #id: string;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #worker: WriteWorkerClient;\n  readonly #mode: ReplicatorMode;\n  readonly #statusPublisher: ReplicationStatusPublisher | null;\n  readonly #notifier: Notifier;\n  readonly #reporter: ReplicationReportRecorder;\n\n  readonly #state = new RunningState('IncrementalSyncer');\n\n  readonly #replicationEvents = getOrCreateCounter(\n    'replication',\n    'events',\n    'Number of replication events processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    mode: ReplicatorMode,\n    statusPublisher: ReplicationStatusPublisher | null,\n  ) {\n    this.#lc = lc;\n    this.#taskID = taskID;\n    this.#id = id;\n    this.#changeStreamer = changeStreamer;\n    this.#worker = worker;\n    this.#mode = mode;\n    this.#statusPublisher = statusPublisher;\n    this.#notifier = new Notifier();\n    this.#reporter = new ReplicationReportRecorder(lc);\n  }\n\n  async run() {\n    const lc = this.#lc;\n    this.#worker.onError(err => this.#state.stop(lc, err));\n    lc.info?.(`Starting IncrementalSyncer`);\n    const {watermark: initialWatermark} =\n      await this.#worker.getSubscriptionState();\n\n    // Notify any waiting subscribers that the replica is ready to be read.\n    void this.#notifier.notifySubscribers();\n\n    while (this.#state.shouldRun()) {\n      const {replicaVersion, watermark} =\n        await this.#worker.getSubscriptionState();\n\n      let downstream: Source<Downstream> | undefined;\n      let unregister = () => {};\n      let err: unknown | undefined;\n\n      try {\n        downstream = await this.#changeStreamer.subscribe({\n          protocolVersion: PROTOCOL_VERSION,\n          taskID: this.#taskID,\n          id: this.#id,\n          mode: this.#mode,\n          watermark,\n          replicaVersion,\n          initial: watermark === initialWatermark,\n        });\n        this.#state.resetBackoff();\n        unregister = this.#state.cancelOnStop(downstream);\n        this.#statusPublisher?.publish(\n          lc,\n          'Replicating',\n          `Replicating from ${watermark}`,\n        );\n\n        let backfillStatus: DownloadStatus | undefined;\n\n        for await (const message of downstream) {\n          this.#replicationEvents.add(1);\n          switch (message[0]) {\n            case 'status': {\n              const {lagReport} = message[1];\n              if (lagReport) {\n                const report: ReplicationReport = {\n                  nextSendTimeMs: lagReport.nextSendTimeMs,\n                };\n                if (lagReport.lastTimings) {\n                  report.lastTimings = {\n                    ...lagReport.lastTimings,\n                    replicateTimeMs: Date.now(),\n                  };\n                }\n                this.#reporter.record(report);\n              }\n              break;\n            }\n            case 'error':\n              // Unrecoverable error. Stop the service.\n              this.stop(lc, message[1]);\n              break;\n            default: {\n              const msg = message[1];\n              if (msg.tag === 'backfill' && msg.status) {\n                const {status} = msg;\n                if (!backfillStatus) {\n                  // Start publishing the status every 3 seconds.\n                  backfillStatus = status;\n                  this.#statusPublisher?.publish(\n                    lc,\n                    'Replicating',\n                    `Backfilling ${msg.relation.name} table`,\n                    3000,\n                    () =>\n                      backfillStatus\n                        ? {\n                            downloadStatus: [\n                              {\n                                ...backfillStatus,\n                                table: msg.relation.name,\n                                columns: [\n                                  ...msg.relation.rowKey.columns,\n                                  ...msg.columns,\n                                ],\n                              },\n                            ],\n                          }\n                        : {},\n                  );\n                }\n                backfillStatus = status; // Update the current status\n              }\n\n              const result = await this.#worker.processMessage(\n                message as ChangeStreamData,\n              );\n\n              this.#handleResult(lc, result);\n              if (result?.completedBackfill) {\n                backfillStatus = undefined;\n              }\n              break;\n            }\n          }\n        }\n        this.#worker.abort();\n      } catch (e) {\n        err = e;\n        this.#worker.abort();\n      } finally {\n        downstream?.cancel();\n        unregister();\n        this.#statusPublisher?.stop();\n      }\n      await this.#state.backoff(lc, err);\n    }\n    lc.info?.('IncrementalSyncer stopped');\n  }\n\n  #handleResult(lc: LogContext, result: CommitResult | null) {\n    if (!result) {\n      return;\n    }\n    if (result.completedBackfill) {\n      // Publish the final status\n      const status = result.completedBackfill;\n      this.#statusPublisher?.publish(\n        lc,\n        'Replicating',\n        `Backfilled ${status.table} table`,\n        0,\n        () => ({downloadStatus: [status]}),\n      );\n    } else if (result.schemaUpdated) {\n      this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n    }\n    if (result.watermark && result.changeLogUpdated) {\n      void this.#notifier.notifySubscribers({state: 'version-ready'});\n    }\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#notifier.subscribe();\n  }\n\n  stop(lc: LogContext, err?: unknown) {\n    this.#state.stop(lc, err);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AA0BA,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAkB,IAAI,aAAa,oBAAoB;CAEvD,qBAA8B,mBAC5B,eACA,UACA,yCACD;CAED,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA;AACA,QAAA,KAAW;AACX,QAAA,SAAe;AACf,QAAA,KAAW;AACX,QAAA,iBAAuB;AACvB,QAAA,SAAe;AACf,QAAA,OAAa;AACb,QAAA,kBAAwB;AACxB,QAAA,WAAiB,IAAI,UAAU;AAC/B,QAAA,WAAiB,IAAI,0BAA0B,GAAG;;CAGpD,MAAM,MAAM;EACV,MAAM,KAAK,MAAA;AACX,QAAA,OAAa,SAAQ,QAAO,MAAA,MAAY,KAAK,IAAI,IAAI,CAAC;AACtD,KAAG,OAAO,6BAA6B;EACvC,MAAM,EAAC,WAAW,qBAChB,MAAM,MAAA,OAAa,sBAAsB;AAGtC,QAAA,SAAe,mBAAmB;AAEvC,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,MAAA,OAAa,sBAAsB;GAE3C,IAAI;GACJ,IAAI,mBAAmB;GACvB,IAAI;AAEJ,OAAI;AACF,iBAAa,MAAM,MAAA,eAAqB,UAAU;KAChD,iBAAA;KACA,QAAQ,MAAA;KACR,IAAI,MAAA;KACJ,MAAM,MAAA;KACN;KACA;KACA,SAAS,cAAc;KACxB,CAAC;AACF,UAAA,MAAY,cAAc;AAC1B,iBAAa,MAAA,MAAY,aAAa,WAAW;AACjD,UAAA,iBAAuB,QACrB,IACA,eACA,oBAAoB,YACrB;IAED,IAAI;AAEJ,eAAW,MAAM,WAAW,YAAY;AACtC,WAAA,kBAAwB,IAAI,EAAE;AAC9B,aAAQ,QAAQ,IAAhB;MACE,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;AAC5B,WAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,gBAC3B;AACD,YAAI,UAAU,YACZ,QAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,KAAK;SAC5B;AAEH,cAAA,SAAe,OAAO,OAAO;;AAE/B;;MAEF,KAAK;AAEH,YAAK,KAAK,IAAI,QAAQ,GAAG;AACzB;MACF,SAAS;OACP,MAAM,MAAM,QAAQ;AACpB,WAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,WAAU;AACjB,YAAI,CAAC,gBAAgB;AAEnB,0BAAiB;AACjB,eAAA,iBAAuB,QACrB,IACA,eACA,eAAe,IAAI,SAAS,KAAK,SACjC,WAEE,iBACI,EACE,gBAAgB,CACd;UACE,GAAG;UACH,OAAO,IAAI,SAAS;UACpB,SAAS,CACP,GAAG,IAAI,SAAS,OAAO,SACvB,GAAG,IAAI,QACR;UACF,CACF,EACF,GACD,EAAE,CACT;;AAEH,yBAAiB;;OAGnB,MAAM,SAAS,MAAM,MAAA,OAAa,eAChC,QACD;AAED,aAAA,aAAmB,IAAI,OAAO;AAC9B,WAAI,QAAQ,kBACV,kBAAiB,KAAA;AAEnB;;;;AAIN,UAAA,OAAa,OAAO;YACb,GAAG;AACV,UAAM;AACN,UAAA,OAAa,OAAO;aACZ;AACR,gBAAY,QAAQ;AACpB,gBAAY;AACZ,UAAA,iBAAuB,MAAM;;AAE/B,SAAM,MAAA,MAAY,QAAQ,IAAI,IAAI;;AAEpC,KAAG,OAAO,4BAA4B;;CAGxC,cAAc,IAAgB,QAA6B;AACzD,MAAI,CAAC,OACH;AAEF,MAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;AACtB,SAAA,iBAAuB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,OAAO,EAAC,EAClC;aACQ,OAAO,cAChB,OAAA,iBAAuB,QAAQ,IAAI,eAAe,iBAAiB;AAErE,MAAI,OAAO,aAAa,OAAO,iBACxB,OAAA,SAAe,kBAAkB,EAAC,OAAO,iBAAgB,CAAC;;CAInE,YAAkC;AAChC,SAAO,MAAA,SAAe,WAAW;;CAGnC,KAAK,IAAgB,KAAe;AAClC,QAAA,MAAY,KAAK,IAAI,IAAI"}
+{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n  errorTypeToReadableName,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type Downstream,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport type {ReplicationStatusPublisher} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n  readonly #lc: LogContext;\n  readonly #taskID: string;\n  readonly #id: string;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #worker: WriteWorkerClient;\n  readonly #mode: ReplicatorMode;\n  readonly #statusPublisher: ReplicationStatusPublisher | null;\n  readonly #notifier: Notifier;\n  readonly #reporter: ReplicationReportRecorder;\n\n  readonly #state = new RunningState('IncrementalSyncer');\n\n  readonly #replicationEvents = getOrCreateCounter(\n    'replication',\n    'events',\n    'Number of replication events processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    mode: ReplicatorMode,\n    statusPublisher: ReplicationStatusPublisher | null,\n  ) {\n    this.#lc = lc;\n    this.#taskID = taskID;\n    this.#id = id;\n    this.#changeStreamer = changeStreamer;\n    this.#worker = worker;\n    this.#mode = mode;\n    this.#statusPublisher = statusPublisher;\n    this.#notifier = new Notifier();\n    this.#reporter = new ReplicationReportRecorder(lc);\n  }\n\n  async run() {\n    const lc = this.#lc;\n    this.#worker.onError(err => this.#state.stop(lc, err));\n    lc.info?.(`Starting IncrementalSyncer`);\n    const {watermark: initialWatermark} =\n      await this.#worker.getSubscriptionState();\n\n    // Notify any waiting subscribers that the replica is ready to be read.\n    void this.#notifier.notifySubscribers();\n\n    while (this.#state.shouldRun()) {\n      const {replicaVersion, watermark} =\n        await this.#worker.getSubscriptionState();\n\n      let downstream: Source<Downstream> | undefined;\n      let unregister = () => {};\n      let err: unknown | undefined;\n\n      try {\n        downstream = await this.#changeStreamer.subscribe({\n          protocolVersion: PROTOCOL_VERSION,\n          taskID: this.#taskID,\n          id: this.#id,\n          mode: this.#mode,\n          watermark,\n          replicaVersion,\n          initial: watermark === initialWatermark,\n        });\n        this.#state.resetBackoff();\n        unregister = this.#state.cancelOnStop(downstream);\n        this.#statusPublisher?.publish(\n          lc,\n          'Replicating',\n          `Replicating from ${watermark}`,\n        );\n\n        let backfillStatus: DownloadStatus | undefined;\n\n        for await (const message of downstream) {\n          this.#replicationEvents.add(1);\n          switch (message[0]) {\n            case 'status': {\n              const {lagReport} = message[1];\n              if (lagReport) {\n                const report: ReplicationReport = {\n                  nextSendTimeMs: lagReport.nextSendTimeMs,\n                };\n                if (lagReport.lastTimings) {\n                  report.lastTimings = {\n                    ...lagReport.lastTimings,\n                    replicateTimeMs: Date.now(),\n                  };\n                }\n                this.#reporter.record(report);\n              }\n              break;\n            }\n            case 'error': {\n              // Signal from the replication-manager that the view-syncer must\n              // shut down and restore a new backup from litestream.\n              const {type, message: msg} = message[1];\n              this.stop(\n                lc,\n                // Note: The AbortError indicates a clean / intentional shutdown.\n                new AbortError(\n                  `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n                ),\n              );\n              break;\n            }\n            default: {\n              const msg = message[1];\n              if (msg.tag === 'backfill' && msg.status) {\n                const {status} = msg;\n                if (!backfillStatus) {\n                  // Start publishing the status every 3 seconds.\n                  backfillStatus = status;\n                  this.#statusPublisher?.publish(\n                    lc,\n                    'Replicating',\n                    `Backfilling ${msg.relation.name} table`,\n                    3000,\n                    () =>\n                      backfillStatus\n                        ? {\n                            downloadStatus: [\n                              {\n                                ...backfillStatus,\n                                table: msg.relation.name,\n                                columns: [\n                                  ...msg.relation.rowKey.columns,\n                                  ...msg.columns,\n                                ],\n                              },\n                            ],\n                          }\n                        : {},\n                  );\n                }\n                backfillStatus = status; // Update the current status\n              }\n\n              const result = await this.#worker.processMessage(\n                message as ChangeStreamData,\n              );\n\n              this.#handleResult(lc, result);\n              if (result?.completedBackfill) {\n                backfillStatus = undefined;\n              }\n              break;\n            }\n          }\n        }\n        this.#worker.abort();\n      } catch (e) {\n        err = e;\n        this.#worker.abort();\n      } finally {\n        downstream?.cancel();\n        unregister();\n        this.#statusPublisher?.stop();\n      }\n      await this.#state.backoff(lc, err);\n    }\n    lc.info?.('IncrementalSyncer stopped');\n  }\n\n  #handleResult(lc: LogContext, result: CommitResult | null) {\n    if (!result) {\n      return;\n    }\n    if (result.completedBackfill) {\n      // Publish the final status\n      const status = result.completedBackfill;\n      this.#statusPublisher?.publish(\n        lc,\n        'Replicating',\n        `Backfilled ${status.table} table`,\n        0,\n        () => ({downloadStatus: [status]}),\n      );\n    } else if (result.schemaUpdated) {\n      this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n    }\n    if (result.watermark && result.changeLogUpdated) {\n      void this.#notifier.notifySubscribers({state: 'version-ready'});\n    }\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#notifier.subscribe();\n  }\n\n  stop(lc: LogContext, err?: unknown) {\n    this.#state.stop(lc, err);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAgCA,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAkB,IAAI,aAAa,oBAAoB;CAEvD,qBAA8B,mBAC5B,eACA,UACA,yCACD;CAED,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA;AACA,QAAA,KAAW;AACX,QAAA,SAAe;AACf,QAAA,KAAW;AACX,QAAA,iBAAuB;AACvB,QAAA,SAAe;AACf,QAAA,OAAa;AACb,QAAA,kBAAwB;AACxB,QAAA,WAAiB,IAAI,UAAU;AAC/B,QAAA,WAAiB,IAAI,0BAA0B,GAAG;;CAGpD,MAAM,MAAM;EACV,MAAM,KAAK,MAAA;AACX,QAAA,OAAa,SAAQ,QAAO,MAAA,MAAY,KAAK,IAAI,IAAI,CAAC;AACtD,KAAG,OAAO,6BAA6B;EACvC,MAAM,EAAC,WAAW,qBAChB,MAAM,MAAA,OAAa,sBAAsB;AAGtC,QAAA,SAAe,mBAAmB;AAEvC,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,MAAA,OAAa,sBAAsB;GAE3C,IAAI;GACJ,IAAI,mBAAmB;GACvB,IAAI;AAEJ,OAAI;AACF,iBAAa,MAAM,MAAA,eAAqB,UAAU;KAChD,iBAAA;KACA,QAAQ,MAAA;KACR,IAAI,MAAA;KACJ,MAAM,MAAA;KACN;KACA;KACA,SAAS,cAAc;KACxB,CAAC;AACF,UAAA,MAAY,cAAc;AAC1B,iBAAa,MAAA,MAAY,aAAa,WAAW;AACjD,UAAA,iBAAuB,QACrB,IACA,eACA,oBAAoB,YACrB;IAED,IAAI;AAEJ,eAAW,MAAM,WAAW,YAAY;AACtC,WAAA,kBAAwB,IAAI,EAAE;AAC9B,aAAQ,QAAQ,IAAhB;MACE,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;AAC5B,WAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,gBAC3B;AACD,YAAI,UAAU,YACZ,QAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,KAAK;SAC5B;AAEH,cAAA,SAAe,OAAO,OAAO;;AAE/B;;MAEF,KAAK,SAAS;OAGZ,MAAM,EAAC,MAAM,SAAS,QAAO,QAAQ;AACrC,YAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,KAAkB,CAAC,IAAI,MACnD,CACF;AACD;;MAEF,SAAS;OACP,MAAM,MAAM,QAAQ;AACpB,WAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,WAAU;AACjB,YAAI,CAAC,gBAAgB;AAEnB,0BAAiB;AACjB,eAAA,iBAAuB,QACrB,IACA,eACA,eAAe,IAAI,SAAS,KAAK,SACjC,WAEE,iBACI,EACE,gBAAgB,CACd;UACE,GAAG;UACH,OAAO,IAAI,SAAS;UACpB,SAAS,CACP,GAAG,IAAI,SAAS,OAAO,SACvB,GAAG,IAAI,QACR;UACF,CACF,EACF,GACD,EAAE,CACT;;AAEH,yBAAiB;;OAGnB,MAAM,SAAS,MAAM,MAAA,OAAa,eAChC,QACD;AAED,aAAA,aAAmB,IAAI,OAAO;AAC9B,WAAI,QAAQ,kBACV,kBAAiB,KAAA;AAEnB;;;;AAIN,UAAA,OAAa,OAAO;YACb,GAAG;AACV,UAAM;AACN,UAAA,OAAa,OAAO;aACZ;AACR,gBAAY,QAAQ;AACpB,gBAAY;AACZ,UAAA,iBAAuB,MAAM;;AAE/B,SAAM,MAAA,MAAY,QAAQ,IAAI,IAAI;;AAEpC,KAAG,OAAO,4BAA4B;;CAGxC,cAAc,IAAgB,QAA6B;AACzD,MAAI,CAAC,OACH;AAEF,MAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;AACtB,SAAA,iBAAuB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,OAAO,EAAC,EAClC;aACQ,OAAO,cAChB,OAAA,iBAAuB,QAAQ,IAAI,eAAe,iBAAiB;AAErE,MAAI,OAAO,aAAa,OAAO,iBACxB,OAAA,SAAe,kBAAkB,EAAC,OAAO,iBAAgB,CAAC;;CAInE,YAAkC;AAChC,SAAO,MAAA,SAAe,WAAW;;CAGnC,KAAK,IAAgB,KAAe;AAClC,QAAA,MAAY,KAAK,IAAI,IAAI"}

package/out/zero-cache/src/services/view-syncer/cvr.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cvr.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAEtE,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,gCAAgC,CAAC;AAQxC,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,sCAAsC,CAAC;AAC9D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,gDAAgD,CAAC;AASjF,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,6BAA6B,CAAC;AAE7D,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnE,OAAO,KAAK,EAAQ,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAC,KAAK,aAAa,EAAE,KAAK,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AACjE,OAAO,EAKL,KAAK,YAAY,EAEjB,KAAK,UAAU,EACf,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAChB,KAAK,KAAK,EAEX,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAmB,KAAK,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AAE/D,MAAM,MAAM,SAAS,GAAG;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,SAAS,EAAE;QAAC,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;CACrC,CAAC;AAEF,wCAAwC;AACxC,MAAM,MAAM,GAAG,GAAG;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,QAAQ,CAAC;IACnB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACtC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACrC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAClC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,CAAC;AAEF,mCAAmC;AAEnC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;IACzD,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;IACxD,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC,CAAC;AAcF,wBAAgB,uBAAuB,CACrC,cAAc,EAAE,MAAM,EACtB,aAAa,EAAE,MAAM,GACpB,mBAAmB,CA+BrB;AAED;;;;;;;;GAQG;AACH,qBAAa,UAAU;IACrB,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IACtC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC;IAE7B,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC;IAEvC;;;OAGG;gBAED,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,WAAW,EAChB,cAAc,EAAE,MAAM,GAAG,IAAI;IAQ/B,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU;;;;IASzC;;;;OAIG;IACH,SAAS,CAAC,iBAAiB,IAAI,UAAU;IAOnC,KAAK,CACT,EAAE,EAAE,UAAU,EACd,eAAe,EAAE,MAAM,EACvB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,QAAQ,GACjB,OAAO,CAAC;QACT,GAAG,EAAE,WAAW,CAAC;QACjB,OAAO,EAAE,aAAa,GAAG,KAAK,CAAC;KAChC,CAAC;CAcH;AAED;;;;GAIG;AACH,qBAAa,sBAAuB,SAAQ,UAAU;;gBAGxC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO;IAKhE,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,YAAY;IAqDtC,eAAe,CAAC,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY;IA0B1D,YAAY,CAAC,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM;IAkB9C,iBAAiB,CACf,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,QAAQ,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,CAAC,EAAE,GAAG,GAAG,SAAS,CAAC;QACtB,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QAC1B,IAAI,CAAC,EAAE,SAAS,iBAAiB,EAAE,GAAG,SAAS,CAAC;QAChD,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;KAC1B,CAAC,EAAE,GACH,cAAc,EAAE;IAwFnB,4BAA4B,CAC1B,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EAAE,EACrB,QAAQ,EAAE,QAAQ,GACjB,cAAc,EAAE;IAInB,oBAAoB,CAClB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EAAE,GACpB,cAAc,EAAE;IAuEnB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,EAAE;IAKvD,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,cAAc,EAAE;CAyBrE;AAED,KAAK,IAAI,GAAG,MAAM,CAAC;AACnB,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC;AAC5B,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AAO7C;;;;;;;;;;;;;GAaG;AACH,qBAAa,qBAAsB,SAAQ,UAAU;;IASnD;;OAEG;gBAED,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,WAAW,EAChB,YAAY,EAAE,WAAW,EACzB,cAAc,EAAE,MAAM;IAqBxB;;;;;;;;;;;;OAYG;IACH,YAAY,CACV,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,kBAAkB,EAAE,MAAM,CAAA;KAAC,EAAE,EACpD,OAAO,EAAE;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,EAAE,GACtB;QAAC,UAAU,EAAE,UAAU,CAAC;QAAC,YAAY,EAAE,cAAc,EAAE,CAAA;KAAC;IAqJ3D,cAAc,IAAI,UAAU;IAI5B;;;;;OAKG;IACH,QAAQ,CACN,GAAG,EAAE,UAAU,EACf,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,GAC1B,OAAO,CAAC,cAAc,EAAE,CAAC;IAuG5B;;;;;;;;;;;OAWG;IACH,sBAAsB,CAAC,EAAE,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;CAyEnE;AAgCD;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,GAAG,GAAG;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,QAAQ,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;CACb,EAAE,CA+DF;AAED,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,GAAG,GAAG,QAAQ,GAAG,SAAS,CAS/D"}
+{"version":3,"file":"cvr.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAEtE,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,gCAAgC,CAAC;AAQxC,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,sCAAsC,CAAC;AAC9D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,gDAAgD,CAAC;AASjF,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,6BAA6B,CAAC;AAE7D,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnE,OAAO,KAAK,EAAQ,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAC,KAAK,aAAa,EAAE,KAAK,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AACjE,OAAO,EAKL,KAAK,YAAY,EAEjB,KAAK,UAAU,EACf,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAChB,KAAK,KAAK,EAGX,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAmB,KAAK,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AAE/D,MAAM,MAAM,SAAS,GAAG;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,SAAS,EAAE;QAAC,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;CACrC,CAAC;AAEF,wCAAwC;AACxC,MAAM,MAAM,GAAG,GAAG;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,QAAQ,CAAC;IACnB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACtC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACrC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAClC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,CAAC;AAEF,mCAAmC;AAEnC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;IACzD,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;IACxD,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC,CAAC;AAcF,wBAAgB,uBAAuB,CACrC,cAAc,EAAE,MAAM,EACtB,aAAa,EAAE,MAAM,GACpB,mBAAmB,CA+BrB;AAED;;;;;;;;GAQG;AACH,qBAAa,UAAU;IACrB,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IACtC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC;IAE7B,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC;IAEvC;;;OAGG;gBAED,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,WAAW,EAChB,cAAc,EAAE,MAAM,GAAG,IAAI;IAQ/B,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU;;;;IASzC;;;;OAIG;IACH,SAAS,CAAC,iBAAiB,IAAI,UAAU;IAOnC,KAAK,CACT,EAAE,EAAE,UAAU,EACd,eAAe,EAAE,MAAM,EACvB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,QAAQ,GACjB,OAAO,CAAC;QACT,GAAG,EAAE,WAAW,CAAC;QACjB,OAAO,EAAE,aAAa,GAAG,KAAK,CAAC;KAChC,CAAC;CAcH;AAED;;;;GAIG;AACH,qBAAa,sBAAuB,SAAQ,UAAU;;gBAGxC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO;IAKhE,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,YAAY;IAqDtC,eAAe,CAAC,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY;IA0B1D,YAAY,CAAC,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM;IAkB9C,iBAAiB,CACf,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,QAAQ,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,CAAC,EAAE,GAAG,GAAG,SAAS,CAAC;QACtB,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QAC1B,IAAI,CAAC,EAAE,SAAS,iBAAiB,EAAE,GAAG,SAAS,CAAC;QAChD,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;KAC1B,CAAC,EAAE,GACH,cAAc,EAAE;IAwFnB,4BAA4B,CAC1B,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EAAE,EACrB,QAAQ,EAAE,QAAQ,GACjB,cAAc,EAAE;IAInB,oBAAoB,CAClB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EAAE,GACpB,cAAc,EAAE;IAuEnB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,EAAE;IAKvD,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,cAAc,EAAE;CAyBrE;AAED,KAAK,IAAI,GAAG,MAAM,CAAC;AACnB,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC;AAC5B,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AAO7C;;;;;;;;;;;;;GAaG;AACH,qBAAa,qBAAsB,SAAQ,UAAU;;IASnD;;OAEG;gBAED,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,WAAW,EAChB,YAAY,EAAE,WAAW,EACzB,cAAc,EAAE,MAAM;IAqBxB;;;;;;;;;;;;OAYG;IACH,YAAY,CACV,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,kBAAkB,EAAE,MAAM,CAAA;KAAC,EAAE,EACpD,OAAO,EAAE;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,EAAE,GACtB;QAAC,UAAU,EAAE,UAAU,CAAC;QAAC,YAAY,EAAE,cAAc,EAAE,CAAA;KAAC;IAwK3D,cAAc,IAAI,UAAU;IAI5B;;;;;OAKG;IACH,QAAQ,CACN,GAAG,EAAE,UAAU,EACf,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,GAC1B,OAAO,CAAC,cAAc,EAAE,CAAC;IA4G5B;;;;;;;;;;;OAWG;IACH,sBAAsB,CAAC,EAAE,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;CA8EnE;AAgCD;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,GAAG,GAAG;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,QAAQ,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;CACb,EAAE,CA+DF;AAED,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,GAAG,GAAG,QAAQ,GAAG,SAAS,CAS/D"}