@rocicorp/zero 1.9.1-canary.0 → 1.9.1-canary.2

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#changeCounter","#stream","#purgeLock","#latestStatus","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import {getDefaultHighWaterMark} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n  type Rollback,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n  type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Status,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {\n  Storer,\n  type PurgeLock,\n  type TuningOptions as StorerOptions,\n} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\nexport type TuningOptions = StorerOptions & {\n  flowControlConsensusPaddingSeconds: number;\n};\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  replicationStatusPublisher: ReplicationStatusPublisher,\n  subscriptionState: SubscriptionState,\n  purgeLock: PurgeLock | null,\n  autoReset: boolean,\n  opts: TuningOptions,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n    setTimeoutFn,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    replicationStatusPublisher,\n    purgeLock,\n    autoReset,\n    opts,\n    setTimeoutFn,\n  );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n *\n * Additionally, the ChangeStreamData is eagerly stringified once, after which\n * the string is passed to the changeLog and all subscribers, eliminating\n * redundant stringification and reducing GC churn.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n  readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n  readonly #changeCounter = getOrCreateCounter(\n    'replication',\n    'changes',\n    'Count of replicated changes (DML or DDL statements)',\n  );\n\n  #latestStatus: Status;\n  #purgeLock: PurgeLock | null;\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    replicationStatusPublisher: ReplicationStatusPublisher,\n    initialPurgeLock: PurgeLock | null,\n    autoReset: boolean,\n    opts: TuningOptions,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      opts,\n    );\n    this.#forwarder = new Forwarder(lc, {\n      flowControlConsensusPaddingSeconds:\n        opts.flowControlConsensusPaddingSeconds,\n    });\n    this.#replicationStatusPublisher = replicationStatusPublisher;\n    this.#purgeLock = initialPurgeLock;\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n    this.#latestStatus = {tag: 'status'};\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    this.#forwarder.startProgressMonitor();\n\n    const lagReport = await this.#source.startLagReporter();\n    if (lagReport) {\n      this.#latestStatus.lagReport = lagReport;\n    }\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership(this.#purgeLock);\n    this.#purgeLock = null;\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        if (\n          this.#state.resetBackoff() >\n          REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n        ) {\n          // After recovering from a backoff for which a replication status\n          // error was published, publish an OK status\n          this.#replicationStatusPublisher.publish(\n            this.#lc,\n            'Replicating',\n            `Replicating from ${lastWatermark}`,\n          );\n        }\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              if (msg.lagReport) {\n                // Lag reports are not stored in the cdc change log, but rather\n                // only forwarded on \"live\" connections. When a new subscriber\n                // is catching up, it is initialized with the #latestStatus\n                // from which it can measure lag while catching up.\n                this.#latestStatus.lagReport = msg.lagReport;\n                this.#forwarder.sendStatus(this.#latestStatus);\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (type === 'data') {\n                this.#changeCounter.add(1);\n              }\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          const json = this.#storer.store(watermark, change);\n          const entry: WatermarkedChange = [watermark, change[1].tag, json];\n          unflushedBytes += json.length;\n          if (unflushedBytes < flushBytesThreshold) {\n            // pipeline changes until flushBytesThreshold\n            this.#forwarder.forward(entry);\n          } else {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            const forwarded = this.#forwarder.forwardWithFlowControl(entry);\n            await stream.changes.doneOr(forwarded);\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await stream.changes.doneOr(readyForMore);\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, 'rollback', ROLLBACK_JSON]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n\n    this.#forwarder.stopProgressMonitor();\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<string>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<string>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n      () => this.#latestStatus,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await Promise.allSettled([this.#storer.stop(), this.#source.stop()]);\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n\nconst ROLLBACK_JSON = JSON.stringify([\n  'rollback',\n  {tag: 'rollback'},\n] satisfies Rollback);\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+DA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,WACA,MACA,eAAe,YACiB;CAEhC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,MAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,YACF;CAEA,MAAM,EAAC,mBAAkB;CACzB,OAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,WACA,MACA,YACF;AACF;AAEA,IAAM,8CAA8C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkKpD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,IAAY;CAU9C,WAAoB,SAAS;CAE7B,aAAsB,mBACpB,eACA,gBACA,kCACF;CACA,iBAA0B,mBACxB,eACA,WACA,qDACF;CAEA;CACA;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,kBACA,WACA,MACA,eAAe,YACf;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EACxD,KAAKC,SAAS;EACd,KAAKC,YAAY;EACjB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,UAAU,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,KAAKS,SAAS,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;EAAE,CAAC,IACxE,QAAO,KAAK,KAAK,GAAG,GACpB,IACF;EACA,KAAKR,aAAa,IAAI,UAAU,IAAI,EAClC,oCACE,KAAK,mCACT,CAAC;EACD,KAAKC,8BAA8B;EACnC,KAAKQ,aAAa;EAClB,KAAKP,aAAa;EAClB,KAAKC,SAAS,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,YAAY;EAC/D,KAAKO,gBAAgB,EAAC,KAAK,SAAQ;CACrC;CAEA,MAAM,MAAM;EACV,KAAKhB,IAAI,OAAO,wBAAwB;EAExC,KAAKM,WAAW,qBAAqB;EAErC,MAAM,YAAY,MAAM,KAAKF,QAAQ,iBAAiB;EACtD,IAAI,WACF,KAAKY,cAAc,YAAY;EAKjC,MAAM,KAAKX,QAAQ,gBAAgB,KAAKU,UAAU;EAClD,KAAKA,aAAa;EAIlB,MAAM,sBAAsB,wBAAwB,KAAK;EAEzD,OAAO,KAAKN,OAAO,UAAU,GAAG;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;GACrB,IAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,KAAKJ,QAAQ,uCAAuC;IAC5D,MAAM,SAAS,MAAM,KAAKD,QAAQ,YAChC,eACA,gBACF;IACA,KAAKC,QAAQ,IAAI,EAAE,OAAM,MAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;IAEtD,KAAKS,UAAU;IACf,IACE,KAAKL,OAAO,aAAa,IACzB,6CAIA,KAAKF,4BAA4B,QAC/B,KAAKP,KACL,eACA,oBAAoB,eACtB;IAEF,YAAY;IAEZ,WAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;KACpB,QAAQ,MAAR;MACE,KAAK;OACH,IAAI,IAAI,KACN,KAAKK,QAAQ,OAAO,MAAM;OAE5B,IAAI,IAAI,WAAW;QAKjB,KAAKW,cAAc,YAAY,IAAI;QACnC,KAAKV,WAAW,WAAW,KAAKU,aAAa;OAC/C;OACA;MACF,KAAK;OACH,MAAM,KAAKC,sBAAsB,GAAG;OACpC;MACF,KAAK;OACH,YAAY,OAAO,GAAG;OACtB;MACF,KAAK;OACH,IAAI,cAAc,OAAO,GAAG,WAC1B,MAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,WAC9E;OAEF,KAAKL,WAAW,IAAI,CAAC;OACrB;MACF;OACE,IAAI,SAAS,QACX,KAAKC,eAAe,IAAI,CAAC;OAE3B,IAAI,cAAc,MAChB,MAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,kCAC7B;OAEF;KACJ;KAEA,MAAM,OAAO,KAAKR,QAAQ,MAAM,WAAW,MAAM;KACjD,MAAM,QAA2B;MAAC;MAAW,OAAO,GAAG;MAAK;KAAI;KAChE,kBAAkB,KAAK;KACvB,IAAI,iBAAiB,qBAEnB,KAAKC,WAAW,QAAQ,KAAK;UACxB;MAQL,MAAM,YAAY,KAAKA,WAAW,uBAAuB,KAAK;MAC9D,MAAM,OAAO,QAAQ,OAAO,SAAS;MACrC,iBAAiB;KACnB;KAEA,IAAI,SAAS,YAAY,SAAS,YAChC,YAAY;KAId,MAAM,eAAe,KAAKD,QAAQ,aAAa;KAC/C,IAAI,cACF,MAAM,OAAO,QAAQ,OAAO,YAAY;IAE5C;GACF,SAAS,GAAG;IACV,MAAM;GACR,UAAU;IACR,KAAKS,SAAS,QAAQ,OAAO;IAC7B,KAAKA,UAAU,KAAA;GACjB;GAGA,IAAI,WAAW;IACb,KAAKd,IAAI,OAAO,oCAAoC,WAAW;IAC/D,KAAKK,QAAQ,MAAM;IACnB,KAAKC,WAAW,QAAQ;KAAC;KAAW;KAAY;IAAa,CAAC;GAChE;GAGA,MAAM,QAAQ,IAAI;IAChB,KAAKD,QAAQ,KAAK;IAClB,KAAKI,OAAO,QAAQ,KAAKT,KAAK,GAAG;IACjC,KAAKS,OAAO,aAAa,8CACrB,qBACE,KAAKT,KACL,uBAAuB,KAAKA,KAAK,eAAe,GAAG,CACrD,IACA;GACN,CAAC;EACH;EAEA,KAAKM,WAAW,oBAAoB;EACpC,KAAKN,IAAI,OAAO,wBAAwB;CAC1C;CAEA,MAAMiB,sBAAsB,KAA6B;EACvD,KAAKjB,IAAI,OAAO,4BAA4B,GAAG;EAC/C,MAAM,EAAC,QAAO;EAEd,QAAQ,KAAR;GACE,KAAK;IACH,MAAM,kBAAkB,KAAKE,WAAW,KAAKD,MAAM;IACnD,MAAM,wBACJ,KAAKD,KACL,eACA,IAAI,WAAW,mBACf,IAAI,YACN;IACA,IAAI,KAAKQ,YAAY;KACnB,KAAKR,IAAI,OAAO,8BAA8B;KAC9C,MAAM,KAAK,KAAK,IAAI,gBAAgB,CAAC;IACvC;IACA;GACF,SACE,YAAY,GAAG;EACnB;CACF;CAEA,UAAU,KAAiD;EACzD,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;EAC/D,IAAI,SAAS,WACX,KAAKW,SAAS,QAAQ;EAExB,MAAM,aAAa,aAAa,OAAe,EAC7C,eAAe,KAAKL,WAAW,OAAO,UAAU,EAClD,CAAC;EACD,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,KAAKU,aACb;EACA,IAAI,mBAAmB,KAAKb,iBAAiB;GAC3C,KAAKH,IAAI,OACP,2CAA2C,gBAC7C;GACA,WAAW,MACT,GACA,8BACE,KAAKG,gBACN,cAAc,eAAe,EAChC;EACF,OAAO;GACL,KAAKH,IAAI,QAAQ,qBAAqB,WAAW,IAAI;GAErD,KAAKM,WAAW,IAAI,UAAU;GAC9B,KAAKD,QAAQ,QAAQ,YAAY,IAAI;EACvC;EACA,OAAO,QAAQ,QAAQ,UAAU;CACnC;CAEA,gBAAgB,WAAmB;EACjC,MAAM,WAAW,KAAKK,mBAAmB;EACzC,KAAKA,mBAAmB,IAAI,SAAS;EAErC,IAAI,aAAa,GACf,KAAKD,OAAO,iBAAiB,KAAKS,iBAAiB,GAAG,gBAAgB;CAE1E;CAEA,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,KAAKb,QAAQ,0BAA0B;EAClE,IAAI,CAAC,cACH,KAAKL,IAAI,OACP,8EACF;EAEF,OAAO;GACL,gBAAgB,KAAKG;GACrB,cAAc,gBAAgB,KAAKA;EACrC;CACF;;;;;;CAOA,MAAMe,mBAAkC;EACtC,MAAM,UAAU,CAAC,GAAG,KAAKR,kBAAkB;EAC3C,IAAI,QAAQ,WAAW,GAAG;GACxB,KAAKV,IAAI,OAAO,4CAA4C;GAC5D;EACF;EACA,MAAM,UAAU,CAAC,GAAG,KAAKM,WAAW,QAAQ,CAAC;EAC7C,IAAI,QAAQ,WAAW,GAAG;GAGxB,KAAKN,IAAI,OAAO,mCAAmC;GACnD;EACF;EACA,IAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,IAAI,kBAAkB,iBACpB,KAAKA,IAAI,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,EAChF;QACK;IACL,KAAKA,IAAI,OAAO,0BAA0B,gBAAgB,KAAK;IAC/D,MAAM,QAAQ,YAAY,IAAI;IAC9B,MAAM,UAAU,MAAM,KAAKK,QAAQ,mBAAmB,eAAe;IACrE,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;IACrD,KAAKL,IAAI,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,KAClE;IACA,KAAKU,mBAAmB,OAAO,eAAe;GAChD;EACF,SAAS,GAAG;GACV,KAAKV,IAAI,OAAO,4BAA4B,CAAC;EAC/C,UAAU;GACR,IAAI,KAAKU,mBAAmB,MAE1B,KAAKD,OAAO,iBAAiB,KAAKS,iBAAiB,GAAG,gBAAgB;EAE1E;CACF;CAEA,MAAM,KAAK,KAAe;EACxB,KAAKT,OAAO,KAAK,KAAKT,KAAK,GAAG;EAC9B,KAAKc,SAAS,QAAQ,OAAO;EAC7B,MAAM,QAAQ,WAAW,CAAC,KAAKT,QAAQ,KAAK,GAAG,KAAKD,QAAQ,KAAK,CAAC,CAAC;CACrE;AACF;AAiBA,IAAM,mBAAmB,6BAA6B;AAEtD,IAAM,gBAAgB,KAAK,UAAU,CACnC,YACA,EAAC,KAAK,WAAU,CAClB,CAAoB"}
+{"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#changeCounter","#stream","#purgeLock","#latestStatus","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import {getDefaultHighWaterMark} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseOrAbort} from '../../../../shared/src/promise-race.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n  type Rollback,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n  type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Status,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {\n  Storer,\n  type PurgeLock,\n  type TuningOptions as StorerOptions,\n} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\nexport type TuningOptions = StorerOptions & {\n  flowControlConsensusTimeoutProportion: number;\n  flowControlSlowSubscriberGracePeriodMs?: number | undefined;\n};\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  replicationStatusPublisher: ReplicationStatusPublisher,\n  subscriptionState: SubscriptionState,\n  purgeLock: PurgeLock | null,\n  autoReset: boolean,\n  opts: TuningOptions,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n    setTimeoutFn,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    replicationStatusPublisher,\n    purgeLock,\n    autoReset,\n    opts,\n    setTimeoutFn,\n  );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n *\n * Additionally, the ChangeStreamData is eagerly stringified once, after which\n * the string is passed to the changeLog and all subscribers, eliminating\n * redundant stringification and reducing GC churn.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n  readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n  readonly #changeCounter = getOrCreateCounter(\n    'replication',\n    'changes',\n    'Count of replicated changes (DML or DDL statements)',\n  );\n\n  #latestStatus: Status;\n  #purgeLock: PurgeLock | null;\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    replicationStatusPublisher: ReplicationStatusPublisher,\n    initialPurgeLock: PurgeLock | null,\n    autoReset: boolean,\n    opts: TuningOptions,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      opts,\n    );\n    this.#forwarder = new Forwarder(lc, {\n      flowControlConsensusTimeoutProportion:\n        opts.flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodMs:\n        opts.flowControlSlowSubscriberGracePeriodMs,\n    });\n    this.#replicationStatusPublisher = replicationStatusPublisher;\n    this.#purgeLock = initialPurgeLock;\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n    this.#latestStatus = {tag: 'status'};\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    this.#forwarder.startProgressMonitor();\n\n    const lagReport = await this.#source.startLagReporter();\n    if (lagReport) {\n      this.#latestStatus.lagReport = lagReport;\n    }\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership(this.#purgeLock);\n    this.#purgeLock = null;\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        if (\n          this.#state.resetBackoff() >\n          REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n        ) {\n          // After recovering from a backoff for which a replication status\n          // error was published, publish an OK status\n          this.#replicationStatusPublisher.publish(\n            this.#lc,\n            'Replicating',\n            `Replicating from ${lastWatermark}`,\n          );\n        }\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              if (msg.lagReport) {\n                // Lag reports are not stored in the cdc change log, but rather\n                // only forwarded on \"live\" connections. When a new subscriber\n                // is catching up, it is initialized with the #latestStatus\n                // from which it can measure lag while catching up.\n                this.#latestStatus.lagReport = msg.lagReport;\n                this.#forwarder.sendStatus(this.#latestStatus);\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (type === 'data') {\n                this.#changeCounter.add(1);\n              }\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          const json = this.#storer.store(watermark, change);\n          const entry: WatermarkedChange = [watermark, change[1].tag, json];\n          unflushedBytes += json.length;\n          if (unflushedBytes < flushBytesThreshold) {\n            // pipeline changes until flushBytesThreshold\n            this.#forwarder.forward(entry);\n          } else {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            const forwarded = this.#forwarder.forwardWithFlowControl(entry);\n            await promiseOrAbort(\n              forwarded,\n              stream.changes.signal,\n              this.#state.signal,\n            );\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await promiseOrAbort(\n              readyForMore,\n              stream.changes.signal,\n              this.#state.signal,\n            );\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, 'rollback', ROLLBACK_JSON]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n\n    this.#forwarder.stopProgressMonitor();\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<string>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<string>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n      () => this.#latestStatus,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await Promise.allSettled([this.#storer.stop(), this.#source.stop()]);\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n\nconst ROLLBACK_JSON = JSON.stringify([\n  'rollback',\n  {tag: 'rollback'},\n] satisfies Rollback);\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAiEA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,WACA,MACA,eAAe,YACiB;CAEhC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,MAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,YACF;CAEA,MAAM,EAAC,mBAAkB;CACzB,OAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,WACA,MACA,YACF;AACF;AAEA,IAAM,8CAA8C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkKpD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,IAAY;CAU9C,WAAoB,SAAS;CAE7B,aAAsB,mBACpB,eACA,gBACA,kCACF;CACA,iBAA0B,mBACxB,eACA,WACA,qDACF;CAEA;CACA;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,kBACA,WACA,MACA,eAAe,YACf;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EACxD,KAAKC,SAAS;EACd,KAAKC,YAAY;EACjB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,UAAU,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,KAAKS,SAAS,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;EAAE,CAAC,IACxE,QAAO,KAAK,KAAK,GAAG,GACpB,IACF;EACA,KAAKR,aAAa,IAAI,UAAU,IAAI;GAClC,uCACE,KAAK;GACP,wCACE,KAAK;EACT,CAAC;EACD,KAAKC,8BAA8B;EACnC,KAAKQ,aAAa;EAClB,KAAKP,aAAa;EAClB,KAAKC,SAAS,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,YAAY;EAC/D,KAAKO,gBAAgB,EAAC,KAAK,SAAQ;CACrC;CAEA,MAAM,MAAM;EACV,KAAKhB,IAAI,OAAO,wBAAwB;EAExC,KAAKM,WAAW,qBAAqB;EAErC,MAAM,YAAY,MAAM,KAAKF,QAAQ,iBAAiB;EACtD,IAAI,WACF,KAAKY,cAAc,YAAY;EAKjC,MAAM,KAAKX,QAAQ,gBAAgB,KAAKU,UAAU;EAClD,KAAKA,aAAa;EAIlB,MAAM,sBAAsB,wBAAwB,KAAK;EAEzD,OAAO,KAAKN,OAAO,UAAU,GAAG;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;GACrB,IAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,KAAKJ,QAAQ,uCAAuC;IAC5D,MAAM,SAAS,MAAM,KAAKD,QAAQ,YAChC,eACA,gBACF;IACA,KAAKC,QAAQ,IAAI,EAAE,OAAM,MAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;IAEtD,KAAKS,UAAU;IACf,IACE,KAAKL,OAAO,aAAa,IACzB,6CAIA,KAAKF,4BAA4B,QAC/B,KAAKP,KACL,eACA,oBAAoB,eACtB;IAEF,YAAY;IAEZ,WAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;KACpB,QAAQ,MAAR;MACE,KAAK;OACH,IAAI,IAAI,KACN,KAAKK,QAAQ,OAAO,MAAM;OAE5B,IAAI,IAAI,WAAW;QAKjB,KAAKW,cAAc,YAAY,IAAI;QACnC,KAAKV,WAAW,WAAW,KAAKU,aAAa;OAC/C;OACA;MACF,KAAK;OACH,MAAM,KAAKC,sBAAsB,GAAG;OACpC;MACF,KAAK;OACH,YAAY,OAAO,GAAG;OACtB;MACF,KAAK;OACH,IAAI,cAAc,OAAO,GAAG,WAC1B,MAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,WAC9E;OAEF,KAAKL,WAAW,IAAI,CAAC;OACrB;MACF;OACE,IAAI,SAAS,QACX,KAAKC,eAAe,IAAI,CAAC;OAE3B,IAAI,cAAc,MAChB,MAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,kCAC7B;OAEF;KACJ;KAEA,MAAM,OAAO,KAAKR,QAAQ,MAAM,WAAW,MAAM;KACjD,MAAM,QAA2B;MAAC;MAAW,OAAO,GAAG;MAAK;KAAI;KAChE,kBAAkB,KAAK;KACvB,IAAI,iBAAiB,qBAEnB,KAAKC,WAAW,QAAQ,KAAK;UACxB;MASL,MAAM,eADY,KAAKA,WAAW,uBAAuB,KAEvD,GACA,OAAO,QAAQ,QACf,KAAKG,OAAO,MACd;MACA,iBAAiB;KACnB;KAEA,IAAI,SAAS,YAAY,SAAS,YAChC,YAAY;KAId,MAAM,eAAe,KAAKJ,QAAQ,aAAa;KAC/C,IAAI,cACF,MAAM,eACJ,cACA,OAAO,QAAQ,QACf,KAAKI,OAAO,MACd;IAEJ;GACF,SAAS,GAAG;IACV,MAAM;GACR,UAAU;IACR,KAAKK,SAAS,QAAQ,OAAO;IAC7B,KAAKA,UAAU,KAAA;GACjB;GAGA,IAAI,WAAW;IACb,KAAKd,IAAI,OAAO,oCAAoC,WAAW;IAC/D,KAAKK,QAAQ,MAAM;IACnB,KAAKC,WAAW,QAAQ;KAAC;KAAW;KAAY;IAAa,CAAC;GAChE;GAGA,MAAM,QAAQ,IAAI;IAChB,KAAKD,QAAQ,KAAK;IAClB,KAAKI,OAAO,QAAQ,KAAKT,KAAK,GAAG;IACjC,KAAKS,OAAO,aAAa,8CACrB,qBACE,KAAKT,KACL,uBAAuB,KAAKA,KAAK,eAAe,GAAG,CACrD,IACA;GACN,CAAC;EACH;EAEA,KAAKM,WAAW,oBAAoB;EACpC,KAAKN,IAAI,OAAO,wBAAwB;CAC1C;CAEA,MAAMiB,sBAAsB,KAA6B;EACvD,KAAKjB,IAAI,OAAO,4BAA4B,GAAG;EAC/C,MAAM,EAAC,QAAO;EAEd,QAAQ,KAAR;GACE,KAAK;IACH,MAAM,kBAAkB,KAAKE,WAAW,KAAKD,MAAM;IACnD,MAAM,wBACJ,KAAKD,KACL,eACA,IAAI,WAAW,mBACf,IAAI,YACN;IACA,IAAI,KAAKQ,YAAY;KACnB,KAAKR,IAAI,OAAO,8BAA8B;KAC9C,MAAM,KAAK,KAAK,IAAI,gBAAgB,CAAC;IACvC;IACA;GACF,SACE,YAAY,GAAG;EACnB;CACF;CAEA,UAAU,KAAiD;EACzD,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;EAC/D,IAAI,SAAS,WACX,KAAKW,SAAS,QAAQ;EAExB,MAAM,aAAa,aAAa,OAAe,EAC7C,eAAe,KAAKL,WAAW,OAAO,UAAU,EAClD,CAAC;EACD,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,KAAKU,aACb;EACA,IAAI,mBAAmB,KAAKb,iBAAiB;GAC3C,KAAKH,IAAI,OACP,2CAA2C,gBAC7C;GACA,WAAW,MACT,GACA,8BACE,KAAKG,gBACN,cAAc,eAAe,EAChC;EACF,OAAO;GACL,KAAKH,IAAI,QAAQ,qBAAqB,WAAW,IAAI;GAErD,KAAKM,WAAW,IAAI,UAAU;GAC9B,KAAKD,QAAQ,QAAQ,YAAY,IAAI;EACvC;EACA,OAAO,QAAQ,QAAQ,UAAU;CACnC;CAEA,gBAAgB,WAAmB;EACjC,MAAM,WAAW,KAAKK,mBAAmB;EACzC,KAAKA,mBAAmB,IAAI,SAAS;EAErC,IAAI,aAAa,GACf,KAAKD,OAAO,iBAAiB,KAAKS,iBAAiB,GAAG,gBAAgB;CAE1E;CAEA,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,KAAKb,QAAQ,0BAA0B;EAClE,IAAI,CAAC,cACH,KAAKL,IAAI,OACP,8EACF;EAEF,OAAO;GACL,gBAAgB,KAAKG;GACrB,cAAc,gBAAgB,KAAKA;EACrC;CACF;;;;;;CAOA,MAAMe,mBAAkC;EACtC,MAAM,UAAU,CAAC,GAAG,KAAKR,kBAAkB;EAC3C,IAAI,QAAQ,WAAW,GAAG;GACxB,KAAKV,IAAI,OAAO,4CAA4C;GAC5D;EACF;EACA,MAAM,UAAU,CAAC,GAAG,KAAKM,WAAW,QAAQ,CAAC;EAC7C,IAAI,QAAQ,WAAW,GAAG;GAGxB,KAAKN,IAAI,OAAO,mCAAmC;GACnD;EACF;EACA,IAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,IAAI,kBAAkB,iBACpB,KAAKA,IAAI,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,EAChF;QACK;IACL,KAAKA,IAAI,OAAO,0BAA0B,gBAAgB,KAAK;IAC/D,MAAM,QAAQ,YAAY,IAAI;IAC9B,MAAM,UAAU,MAAM,KAAKK,QAAQ,mBAAmB,eAAe;IACrE,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;IACrD,KAAKL,IAAI,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,KAClE;IACA,KAAKU,mBAAmB,OAAO,eAAe;GAChD;EACF,SAAS,GAAG;GACV,KAAKV,IAAI,OAAO,4BAA4B,CAAC;EAC/C,UAAU;GACR,IAAI,KAAKU,mBAAmB,MAE1B,KAAKD,OAAO,iBAAiB,KAAKS,iBAAiB,GAAG,gBAAgB;EAE1E;CACF;CAEA,MAAM,KAAK,KAAe;EACxB,KAAKT,OAAO,KAAK,KAAKT,KAAK,GAAG;EAC9B,KAAKc,SAAS,QAAQ,OAAO;EAC7B,MAAM,QAAQ,WAAW,CAAC,KAAKT,QAAQ,KAAK,GAAG,KAAKD,QAAQ,KAAK,CAAC,CAAC;CACrE;AACF;AAiBA,IAAM,mBAAmB,6BAA6B;AAEtD,IAAM,gBAAgB,KAAK,UAAU,CACnC,YACA,EAAC,KAAK,WAAU,CAClB,CAAoB"}

package/out/zero-cache/src/services/change-streamer/error-type-enum.d.ts

@@ -1,7 +1,15 @@
 export declare const Unknown = 0;
 export declare const WrongReplicaVersion = 1;
 export declare const WatermarkTooOld = 2;
+/**
+ * Sent on an initially live stream before disconnecting it because
+ * it is too far behind. How the subscriber handles this is unspecified
+ * by the server; the current implementation shuts down the process
+ * with the assumption that it is in a pathological zombie state.
+ */
+export declare const StreamTooFarBehind = 3;
 export type Unknown = typeof Unknown;
 export type WrongReplicaVersion = typeof WrongReplicaVersion;
 export type WatermarkTooOld = typeof WatermarkTooOld;
+export type StreamTooFarBehind = typeof StreamTooFarBehind;
 //# sourceMappingURL=error-type-enum.d.ts.map

package/out/zero-cache/src/services/change-streamer/error-type-enum.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"error-type-enum.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/error-type-enum.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,IAAI,CAAC;AACzB,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,eAAO,MAAM,eAAe,IAAI,CAAC;AAEjC,MAAM,MAAM,OAAO,GAAG,OAAO,OAAO,CAAC;AACrC,MAAM,MAAM,mBAAmB,GAAG,OAAO,mBAAmB,CAAC;AAC7D,MAAM,MAAM,eAAe,GAAG,OAAO,eAAe,CAAC"}
+{"version":3,"file":"error-type-enum.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/error-type-enum.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,IAAI,CAAC;AACzB,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,eAAO,MAAM,eAAe,IAAI,CAAC;AACjC;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,IAAI,CAAC;AAEpC,MAAM,MAAM,OAAO,GAAG,OAAO,OAAO,CAAC;AACrC,MAAM,MAAM,mBAAmB,GAAG,OAAO,mBAAmB,CAAC;AAC7D,MAAM,MAAM,eAAe,GAAG,OAAO,eAAe,CAAC;AACrD,MAAM,MAAM,kBAAkB,GAAG,OAAO,kBAAkB,CAAC"}

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts

@@ -3,12 +3,24 @@ import type { WatermarkedChange } from './change-streamer-service.ts';
 import type { Status } from './change-streamer.ts';
 import type { Subscriber } from './subscriber.ts';
 export type ProgressMonitorOptions = {
-    flowControlConsensusPaddingSeconds: number;
+    flowControlConsensusTimeoutProportion: number;
+    flowControlSlowSubscriberGracePeriodMs?: number | undefined;
+    setTimeoutFn?: typeof setTimeout | undefined;
+    clearTimeoutFn?: typeof clearTimeout | undefined;
 };
 export declare class Forwarder {
     #private;
     constructor(lc: LogContext, opts?: ProgressMonitorOptions);
     startProgressMonitor(): void;
+    /**
+     * Runs one progress-monitor tick: samples each active subscriber's change
+     * rate, logs slow broadcasts, and disconnects subscribers that have been
+     * continuously lagging for longer than the configured grace period.
+     *
+     * Invoked on an interval by {@link startProgressMonitor()}; exposed (rather
+     * than private) so tests can drive a single tick with an explicit `now`.
+     */
+    checkSubscriberProgress(now: number): void;
     stopProgressMonitor(): void;
     /**
      * `add()` is called in lock step with `Storer.catchup()` so that the

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAY,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AAC/E,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,sBAAsB,CAAC;AACjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,sBAAsB,GAAG;IACnC,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF,qBAAa,SAAS;;gBAqBlB,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAAgE;IAkDxE,oBAAoB;IAsBpB,mBAAmB;IAInB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;IAKhC,UAAU,CAAC,MAAM,EAAE,MAAM;IAMzB;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB;IAiErD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;CAQvB"}
+{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAY,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AAC/E,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,sBAAsB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAkB,MAAM,iBAAiB,CAAC;AAEjE,MAAM,MAAM,sBAAsB,GAAG;IACnC,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE5D,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IAC7C,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;CAClD,CAAC;AAEF,qBAAa,SAAS;;gBAyBlB,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAEL;IA+DH,oBAAoB;IAQpB;;;;;;;OAOG;IACH,uBAAuB,CAAC,GAAG,EAAE,MAAM;IAyCnC,mBAAmB;IAInB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;IAKhC,UAAU,CAAC,MAAM,EAAE,MAAM;IAMzB;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB;IAsErD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;CAQvB"}

package/out/zero-cache/src/services/change-streamer/forwarder.js

@@ -4,17 +4,27 @@ import { Broadcast } from "./broadcast.js";
 //#region ../zero-cache/src/services/change-streamer/forwarder.ts
 var Forwarder = class {
 	#lc;
-	#progressMonitorOptions;
+	#setTimeout;
+	#clearTimeout;
 	#active = /* @__PURE__ */ new Set();
 	#queued = /* @__PURE__ */ new Set();
+	#earlyReleaseOptions;
+	#flowControlSlowSubscriberGracePeriodMs;
 	#flowControlWaits = getOrCreateCounter("replication", "flow_control.waits", "Flow-control checkpoints completed, labeled by release mode.");
 	#flowControlWaitTime = getOrCreateLatencyHistogram("replication", "flow_control.wait_duration", "Time replication waits at flow-control checkpoints.");
 	#inTransaction = false;
 	#currentBroadcast;
 	#progressMonitor;
-	constructor(lc, opts = { flowControlConsensusPaddingSeconds: 1 }) {
-		this.#lc = lc.withContext("component", "progress-monitor");
-		this.#progressMonitorOptions = opts;
+	constructor(lc, opts = { flowControlConsensusTimeoutProportion: 2 }) {
+		this.#lc = lc.withContext("component", "subscriber-progress-monitor");
+		this.#setTimeout = opts.setTimeoutFn ?? setTimeout;
+		this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;
+		this.#earlyReleaseOptions = opts.flowControlConsensusTimeoutProportion < 0 ? void 0 : {
+			consensusTimeoutProportion: opts.flowControlConsensusTimeoutProportion,
+			setTimeoutFn: this.#setTimeout,
+			clearTimeoutFn: this.#clearTimeout
+		};
+		this.#flowControlSlowSubscriberGracePeriodMs = opts.flowControlSlowSubscriberGracePeriodMs;
 		getOrCreateGauge("replication", "flow_control.active_subscribers", "Active change-stream subscribers receiving live changes.").addCallback((result) => result.observe(this.#active.size));
 		getOrCreateGauge("replication", "flow_control.queued_subscribers", "Change-stream subscribers waiting for the current transaction to finish before activation.").addCallback((result) => result.observe(this.#queued.size));
 		getOrCreateGauge("replication", "flow_control.pending_messages", "Downstream change-stream messages not yet acked by subscribers.").addCallback((result) => result.observe(this.#subscriberStats().pendingMessages));
@@ -30,14 +40,34 @@ var Forwarder = class {
 	}
 	startProgressMonitor() {
 		clearInterval(this.#progressMonitor);
-		this.#progressMonitor = setInterval(this.#trackProgress, 1e3);
+		this.#progressMonitor = setInterval(() => this.checkSubscriberProgress(performance.now()), 1e3);
+	}
+	/**
+	* Runs one progress-monitor tick: samples each active subscriber's change
+	* rate, logs slow broadcasts, and disconnects subscribers that have been
+	* continuously lagging for longer than the configured grace period.
+	*
+	* Invoked on an interval by {@link startProgressMonitor()}; exposed (rather
+	* than private) so tests can drive a single tick with an explicit `now`.
+	*/
+	checkSubscriberProgress(now) {
+		let timedOut;
+		let slowestOnTime;
+		for (const sub of this.#active) {
+			const stats = sub.sampleProcessRate(now).getStats();
+			if (stats.missedLastTimeout) (timedOut ??= /* @__PURE__ */ new Map()).set(sub, stats);
+			else if (stats.processRate <= (slowestOnTime?.processRate ?? Infinity)) slowestOnTime = stats;
+		}
+		this.#currentBroadcast?.logSlowRequests(now);
+		if (this.#flowControlSlowSubscriberGracePeriodMs && timedOut?.size && slowestOnTime) for (const [sub, stats] of timedOut.entries()) {
+			const rate = stats.processRate < slowestOnTime.processRate ? "lagging" : "catching-up";
+			const laggingDuration = sub.reportChangeRate(now, rate);
+			if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {
+				this.#lc.warn?.(`subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`, { stats });
+				sub.close(3, `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`);
+			}
+		}
 	}
-	#trackProgress = () => {
-		const now = performance.now();
-		for (const sub of this.#active) sub.sampleProcessRate(now);
-		const { flowControlConsensusPaddingSeconds } = this.#progressMonitorOptions;
-		if (flowControlConsensusPaddingSeconds >= 0) this.#currentBroadcast?.checkProgress(this.#lc, flowControlConsensusPaddingSeconds * 1e3, now);
-	};
 	stopProgressMonitor() {
 		clearInterval(this.#progressMonitor);
 	}
@@ -77,7 +107,7 @@ var Forwarder = class {
 	*/
 	async forwardWithFlowControl(entry) {
 		const start = performance.now();
-		const broadcast = new Broadcast(this.#active.values(), entry);
+		const broadcast = new Broadcast(this.#lc, this.#active.values(), entry, this.#earlyReleaseOptions);
 		this.#updateActiveSubscribers(entry[1]);
 		this.#currentBroadcast = broadcast;
 		try {

package/out/zero-cache/src/services/change-streamer/forwarder.js.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.js","names":["#lc","#progressMonitorOptions","#active","#queued","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#trackProgress","#currentBroadcast","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast} from './broadcast.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer-service.ts';\nimport type {Status} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusPaddingSeconds: number;\n};\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #progressMonitorOptions: ProgressMonitorOptions;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  readonly #flowControlWaits = getOrCreateCounter(\n    'replication',\n    'flow_control.waits',\n    'Flow-control checkpoints completed, labeled by release mode.',\n  );\n  readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n    'replication',\n    'flow_control.wait_duration',\n    'Time replication waits at flow-control checkpoints.',\n  );\n  #inTransaction = false;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {flowControlConsensusPaddingSeconds: 1},\n  ) {\n    this.#lc = lc.withContext('component', 'progress-monitor');\n    this.#progressMonitorOptions = opts;\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.active_subscribers',\n      'Active change-stream subscribers receiving live changes.',\n    ).addCallback(result => result.observe(this.#active.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.queued_subscribers',\n      'Change-stream subscribers waiting for the current transaction to finish before activation.',\n    ).addCallback(result => result.observe(this.#queued.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.pending_messages',\n      'Downstream change-stream messages not yet acked by subscribers.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().pendingMessages),\n    );\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.backlog_messages',\n      'Live change-stream messages buffered while subscribers catch up.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogMessages),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n      description:\n        'Live change-stream bytes buffered while subscribers catch up.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogBytes),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n      description:\n        'Maximum live change-stream bytes buffered by a single subscriber.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().maxBacklogBytes),\n    );\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(this.#trackProgress, 1000);\n  }\n\n  readonly #trackProgress = () => {\n    const now = performance.now();\n    for (const sub of this.#active) {\n      sub.sampleProcessRate(now);\n    }\n\n    const {flowControlConsensusPaddingSeconds} = this.#progressMonitorOptions;\n    // A negative number disables early flow control release.\n    if (flowControlConsensusPaddingSeconds >= 0) {\n      this.#currentBroadcast?.checkProgress(\n        this.#lc,\n        flowControlConsensusPaddingSeconds * 1000,\n        now,\n      );\n    }\n  };\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward is fire-and-forget, with no flow control. The\n   * change-streamer should call and await {@link forwardWithFlowControl()}\n   * occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    Broadcast.withoutTracking(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n  }\n\n  sendStatus(status: Status) {\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    const start = performance.now();\n    const broadcast = new Broadcast(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    try {\n      await broadcast.done;\n      const attributes = {'release.mode': broadcast.releaseMode};\n      this.#flowControlWaits.add(1, attributes);\n      this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n    } finally {\n      // Technically #currentBroadcast may have changed, so only\n      // unset if it if is still the same.\n      if (this.#currentBroadcast === broadcast) {\n        this.#currentBroadcast = undefined;\n      }\n    }\n  }\n\n  #subscriberStats() {\n    let pendingMessages = 0;\n    let backlogMessages = 0;\n    let backlogBytes = 0;\n    let maxBacklogBytes = 0;\n\n    for (const sub of joinIterables(this.#active, this.#queued)) {\n      const stats = sub.getStats();\n      pendingMessages += stats.pending;\n      backlogMessages += stats.backlog;\n      backlogBytes += stats.backlogBytes;\n      maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n    }\n\n    return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;;AAgBA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB;CACA;CAEA,YACE,IACA,OAA+B,EAAC,oCAAoC,EAAC,GACrE;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKC,0BAA0B;EAE/B,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKG,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,YAAY,KAAKC,gBAAgB,GAAI;CAC/D;CAEA,uBAAgC;EAC9B,MAAM,MAAM,YAAY,IAAI;EAC5B,KAAK,MAAM,OAAO,KAAKN,SACrB,IAAI,kBAAkB,GAAG;EAG3B,MAAM,EAAC,uCAAsC,KAAKD;EAElD,IAAI,sCAAsC,GACxC,KAAKQ,mBAAmB,cACtB,KAAKT,KACL,qCAAqC,KACrC,GACF;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKO,gBAAgB;CACrC;;;;;;CAOA,IAAI,KAAiB;EACnB,IAAI,KAAKG,gBACP,KAAKP,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKA,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,UAAU,gBAAgB,KAAKD,QAAQ,OAAO,GAAG,KAAK;EACtD,KAAKS,yBAAyB,MAAM,EAAE;CACxC;CAEA,WAAW,QAAgB;EACzB,KAAK,MAAM,OAAO,KAAKT,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UAAU,KAAKA,QAAQ,OAAO,GAAG,KAAK;EAC5D,KAAKS,yBAAyB,MAAM,EAAE;EAGtC,KAAKF,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKL,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKI,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKP,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKO,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKP,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}
+{"version":3,"file":"forwarder.js","names":["#lc","#setTimeout","#clearTimeout","#active","#queued","#earlyReleaseOptions","#flowControlSlowSubscriberGracePeriodMs","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#currentBroadcast","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer-service.ts';\nimport type {Status} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusTimeoutProportion: number;\n  flowControlSlowSubscriberGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  readonly #earlyReleaseOptions: EarlyReleaseOptions | undefined;\n  readonly #flowControlSlowSubscriberGracePeriodMs: number | undefined;\n\n  readonly #flowControlWaits = getOrCreateCounter(\n    'replication',\n    'flow_control.waits',\n    'Flow-control checkpoints completed, labeled by release mode.',\n  );\n  readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n    'replication',\n    'flow_control.wait_duration',\n    'Time replication waits at flow-control checkpoints.',\n  );\n  #inTransaction = false;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {\n      flowControlConsensusTimeoutProportion: 2.0,\n    },\n  ) {\n    this.#lc = lc.withContext('component', 'subscriber-progress-monitor');\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    this.#earlyReleaseOptions =\n      opts.flowControlConsensusTimeoutProportion < 0\n        ? undefined\n        : {\n            consensusTimeoutProportion:\n              opts.flowControlConsensusTimeoutProportion,\n            setTimeoutFn: this.#setTimeout,\n            clearTimeoutFn: this.#clearTimeout,\n          };\n    this.#flowControlSlowSubscriberGracePeriodMs =\n      opts.flowControlSlowSubscriberGracePeriodMs;\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.active_subscribers',\n      'Active change-stream subscribers receiving live changes.',\n    ).addCallback(result => result.observe(this.#active.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.queued_subscribers',\n      'Change-stream subscribers waiting for the current transaction to finish before activation.',\n    ).addCallback(result => result.observe(this.#queued.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.pending_messages',\n      'Downstream change-stream messages not yet acked by subscribers.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().pendingMessages),\n    );\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.backlog_messages',\n      'Live change-stream messages buffered while subscribers catch up.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogMessages),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n      description:\n        'Live change-stream bytes buffered while subscribers catch up.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogBytes),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n      description:\n        'Maximum live change-stream bytes buffered by a single subscriber.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().maxBacklogBytes),\n    );\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(\n      () => this.checkSubscriberProgress(performance.now()),\n      1000,\n    );\n  }\n\n  /**\n   * Runs one progress-monitor tick: samples each active subscriber's change\n   * rate, logs slow broadcasts, and disconnects subscribers that have been\n   * continuously lagging for longer than the configured grace period.\n   *\n   * Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n   * than private) so tests can drive a single tick with an explicit `now`.\n   */\n  checkSubscriberProgress(now: number) {\n    let timedOut: Map<Subscriber, SubscriberStats> | undefined;\n    let slowestOnTime: SubscriberStats | undefined;\n\n    for (const sub of this.#active) {\n      const stats = sub.sampleProcessRate(now).getStats();\n      if (stats.missedLastTimeout) {\n        (timedOut ??= new Map()).set(sub, stats);\n      } else if (\n        stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n      ) {\n        slowestOnTime = stats;\n      }\n    }\n    this.#currentBroadcast?.logSlowRequests(now);\n\n    if (\n      this.#flowControlSlowSubscriberGracePeriodMs &&\n      timedOut?.size &&\n      slowestOnTime\n    ) {\n      for (const [sub, stats] of timedOut.entries()) {\n        const rate =\n          stats.processRate < slowestOnTime.processRate\n            ? 'lagging'\n            : 'catching-up';\n        const laggingDuration = sub.reportChangeRate(now, rate);\n        if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n          this.#lc.warn?.(\n            `subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n            {stats},\n          );\n          sub.close(\n            StreamTooFarBehind,\n            `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n          );\n        }\n      }\n    }\n  }\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward is fire-and-forget, with no flow control. The\n   * change-streamer should call and await {@link forwardWithFlowControl()}\n   * occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    Broadcast.withoutTracking(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n  }\n\n  sendStatus(status: Status) {\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    const start = performance.now();\n    const broadcast = new Broadcast(\n      this.#lc,\n      this.#active.values(),\n      entry,\n      this.#earlyReleaseOptions,\n    );\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    try {\n      await broadcast.done;\n      const attributes = {'release.mode': broadcast.releaseMode};\n      this.#flowControlWaits.add(1, attributes);\n      this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n    } finally {\n      // Technically #currentBroadcast may have changed, so only\n      // unset if it if is still the same.\n      if (this.#currentBroadcast === broadcast) {\n        this.#currentBroadcast = undefined;\n      }\n    }\n  }\n\n  #subscriberStats() {\n    let pendingMessages = 0;\n    let backlogMessages = 0;\n    let backlogBytes = 0;\n    let maxBacklogBytes = 0;\n\n    for (const sub of joinIterables(this.#active, this.#queued)) {\n      const stats = sub.getStats();\n      pendingMessages += stats.pending;\n      backlogMessages += stats.backlog;\n      backlogBytes += stats.backlogBytes;\n      maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n    }\n\n    return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;;AAqBA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB;CACA;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,6BAA6B;EACpE,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,KAAKG,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAKJ;GACnB,gBAAgB,KAAKC;EACvB;EACN,KAAKI,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKH,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKK,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAKP,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,EAAE,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAKQ,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAKL,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAKA,yCAAyC;IACnE,KAAKN,IAAI,OACP,cAAc,IAAI,GAAG,kBAAkB,gBAAgB,oBACvD,EAAC,MAAK,CACR;IACA,IAAI,MAAA,GAEF,oBAAoB,KAAKM,wCAAwC,MAAM,gBAAgB,IACzF;GACF;EACF;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;CACrC;;;;;;CAOA,IAAI,KAAiB;EACnB,IAAI,KAAKE,gBACP,KAAKR,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKA,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,UAAU,gBAAgB,KAAKD,QAAQ,OAAO,GAAG,KAAK;EACtD,KAAKU,yBAAyB,MAAM,EAAE;CACxC;CAEA,WAAW,QAAgB;EACzB,KAAK,MAAM,OAAO,KAAKV,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAKH,KACL,KAAKG,QAAQ,OAAO,GACpB,OACA,KAAKE,oBACP;EACA,KAAKQ,yBAAyB,MAAM,EAAE;EAGtC,KAAKF,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKJ,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKG,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKR,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKQ,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKR,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/storer.d.ts

@@ -1,4 +1,5 @@
 import type { LogContext } from '@rocicorp/logger';
+import { type JSONObject } from '../../../../shared/src/bigint-json.ts';
 import { TransactionPool } from '../../db/transaction-pool.ts';
 import { type PostgresDB } from '../../types/pg.ts';
 import { type ShardID } from '../../types/shards.ts';
@@ -100,4 +101,37 @@ export declare class PurgeLocker {
     constructor(lc: LogContext, shard: ShardID, db: PostgresDB);
     acquire(): Promise<PurgeLock | null>;
 }
+type TaskKey = JSONObject;
+type TaskDoneFn = () => void;
+/**
+ * Periodic monitor for detecting the pathological condition in which a db
+ * query hangs forever because of a connection problem, such as a half-closed
+ * socket.
+ *
+ * Use this for db calls that recur or accumulate progress within a single
+ * logical operation on the main storer loop (e.g. the queue-processing loop,
+ * or a multi-batch catchup read), where "no progress for N ms" is the
+ * meaningful signal. For a one-off db call -- including ones made before
+ * {@link Storer.run} starts this monitor's polling -- use a plain timeout
+ * (see `Storer.#withTimeout`) instead.
+ *
+ * The failure handler should stop the server. The ProgressMonitor will
+ * assume this and stop itself after calling it.
+ *
+ * Internal to the Storer, but exported for testing.
+ */
+export declare class ProgressMonitor {
+    #private;
+    constructor(lc: LogContext, failureThresholdMs: number, onFailure: (err: Error) => void);
+    start(): void;
+    checkTaskProgress(now: Date): void;
+    /**
+     * Starts a task to be tracked.
+     *
+     * @return a callback to invoke when the task is done.
+     */
+    trackTask(task: TaskKey, now?: Date): TaskDoneFn;
+    stop(): void;
+}
+export {};
 //# sourceMappingURL=storer.d.ts.map

package/out/zero-cache/src/services/change-streamer/storer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,EAAC,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EACpB,KAAK,MAAM,EAKZ,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EACV,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAS3C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAuChD,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAoBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACvD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa;IAiCZ,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI;IA2B5C,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;KACrC,CAAC;IAkCI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAOzD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyCtD;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB;IAkB/C,KAAK;IAIL,MAAM,CAAC,CAAC,EAAE,uBAAuB;IAIjC,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAOpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IA6FzC;;;OAGG;IACG,GAAG;IA2eT;;;OAGG;IACG,YAAY;IAQlB;;;;OAIG;IACG,IAAI;CAaX;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,UAmB/B;AA+BD,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAG5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM;IAUb,OAAO;CAWd;AAED,qBAAa,WAAW;;gBAKV,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU;IAWpD,OAAO;CA4Bd"}
+{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,uCAAuC,CAAC;AAM/C,OAAO,EAAC,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EACpB,KAAK,MAAM,EAKZ,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EACV,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAS3C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAuChD,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAqBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACvD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa;IA+DZ,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI;IA8B5C,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;KACrC,CAAC;IAqCI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAazD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyCtD;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB;IAkB/C,KAAK;IAIL,MAAM,CAAC,CAAC,EAAE,uBAAuB;IAIjC,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAMpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IA2EzC;;;OAGG;IACG,GAAG;IAshBT;;;OAGG;IACG,YAAY;IAQlB;;;;OAIG;IACG,IAAI;CAcX;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,UAmB/B;AA+BD,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAG5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM;IAUb,OAAO;CAWd;AAED,qBAAa,WAAW;;gBAKV,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU;IAWpD,OAAO;CA4Bd;AAKD,KAAK,OAAO,GAAG,UAAU,CAAC;AAE1B,KAAK,UAAU,GAAG,MAAM,IAAI,CAAC;AAE7B;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,eAAe;;gBASxB,EAAE,EAAE,UAAU,EACd,kBAAkB,EAAE,MAAM,EAC1B,SAAS,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI;IAOjC,KAAK;IASL,iBAAiB,CAAC,GAAG,EAAE,IAAI;IAkB3B;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,OAAa,GAAG,UAAU;IAOtD,IAAI;CAIL"}