@rocicorp/zero 1.10.0-canary.2 → 1.10.0-canary.21

package/out/zero-cache/src/services/change-streamer/storer.js.map

@@ -1 +1 @@
-{"version":3,"file":"storer.js","names":["#lc","#shard","#taskID","#discoveryAddress","#discoveryProtocol","#db","#replicaVersion","#onCommitted","#onFatal","#queue","#backPressureThresholdBytes","#statementTimeoutMs","#changeLogBatchSize","#drainTimeoutMs","#cdc","#approximateQueuedBytes","#running","#readyForMore","#resetBackpressureTimeout","#backpressureTimeout","#stopped","#processQueue","#cancelQueueEntries","#startCatchup","#flushChangeLog","#trackBackfillMetadata","#maybeReleaseBackPressure","#catchup","#cookieStmts","#upsertTableMetadataStmt","#upsertColumnBackfillStmt","#tx","#released"],"sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {getHeapStatistics} from 'node:v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {orTimeout} from '../../types/timeout.ts';\nimport {\n  backfillRequestSchema,\n  isDataChange,\n  isSchemaChange,\n  type BackfillID,\n  type BackfillRequest,\n  type Change,\n  type DataChange,\n  type Identifier,\n  type SchemaChange,\n  type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {\n  type ChangeStreamData,\n  type Commit,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {UpstreamStatusMessage} from '../change-source/protocol/current/status.ts';\nimport {\n  cookieOps,\n  type BackfillCookie,\n  type CookieOp,\n  type CookieSet,\n  type TableMetadataCookie,\n} from '../replicator/change-log-cookies.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport {\n  extractChangeSubstring,\n  reconstructWatermarkedChange,\n  serializeChangeStreamData,\n  type ChangeLogEntry,\n} from './change-log-codec.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n  AutoResetSignal,\n  markResetRequired,\n  type BackfillingColumn,\n  type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n  subscriber: Subscriber;\n  mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n  | [\n      'change',\n      watermark: string,\n      json: string,\n      orig: Exclude<Change, DataChange> | null, // null for DataChanges\n    ]\n  | ['ready', callback: () => void]\n  | ['subscriber', SubscriberAndMode]\n  | ['abort']\n  | 'stop';\n\ntype PendingTransaction = {\n  pool: TransactionPool;\n  preCommitWatermark: string;\n  pos: number;\n  startingReplicationState: Promise<ReplicationOwner>;\n  ack: boolean;\n  // changeLog rows buffered for the next multi-row INSERT flush.\n  batch: ChangeLogRow[];\n  // The most recently issued flush (or metadata) process, awaited to bound\n  // pipeline depth and to order the commit-time replicationState update.\n  lastFlush: Promise<unknown> | undefined;\n};\n\ntype ReplicationOwner = {\n  owner: string | null;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\nexport type TuningOptions = {\n  backPressureLimitHeapProportion: number;\n  statementTimeoutMs: number;\n  changeLogBatchSize: number;\n  drainTimeoutMs?: number | undefined;\n};\n\n/**\n * A single `changeLog` row, accumulated in {@link PendingTransaction.batch}\n * and written via `json_to_recordset()` (see `#flushChangeLog()`).\n */\ntype ChangeLogRow = {\n  watermark: string;\n  precommit: string | null;\n  pos: number;\n  change: string;\n};\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n *   `replicaVersion`, which is the version at which initial sync\n *   was performed, and any attempts to catchup from a different\n *   point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n *   a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n *   subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n  readonly id = 'storer';\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #taskID: string;\n  readonly #discoveryAddress: string;\n  readonly #discoveryProtocol: string;\n  readonly #db: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #onCommitted: (c: Commit) => void;\n  readonly #onFatal: (err: Error) => void;\n  readonly #queue = new Queue<QueueEntry>();\n  readonly #backPressureThresholdBytes: number;\n  readonly #statementTimeoutMs: number;\n  readonly #changeLogBatchSize: number;\n  readonly #drainTimeoutMs: number;\n\n  #approximateQueuedBytes = 0;\n  #running = false;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    db: PostgresDB,\n    replicaVersion: string,\n    onCommitted: (c: Commit | UpstreamStatusMessage) => void,\n    onFatal: (err: Error) => void,\n    {\n      backPressureLimitHeapProportion,\n      statementTimeoutMs,\n      changeLogBatchSize,\n      drainTimeoutMs = 30_000,\n    }: TuningOptions,\n  ) {\n    this.#lc = lc.withContext('component', 'change-log');\n    this.#shard = shard;\n    this.#taskID = taskID;\n    this.#discoveryAddress = discoveryAddress;\n    this.#discoveryProtocol = discoveryProtocol;\n    this.#db = db;\n    this.#replicaVersion = replicaVersion;\n    this.#onCommitted = onCommitted;\n    this.#onFatal = onFatal;\n    this.#statementTimeoutMs = statementTimeoutMs;\n    this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);\n    this.#drainTimeoutMs = drainTimeoutMs;\n\n    const heapStats = getHeapStatistics();\n    this.#backPressureThresholdBytes =\n      (heapStats.heap_size_limit - heapStats.used_heap_size) *\n      backPressureLimitHeapProportion;\n\n    this.#lc.info?.(\n      `Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of ` +\n        `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) ` +\n        `to absorb upstream spikes`,\n      {heapStats},\n    );\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async assumeOwnership(purgeLock?: PurgeLock | null) {\n    const db = this.#db;\n    const owner = this.#taskID;\n    const ownerAddress = this.#discoveryAddress;\n    const ownerProtocol = this.#discoveryProtocol;\n    // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n    const addressWithProtocol =\n      ownerProtocol === 'ws'\n        ? ownerAddress\n        : `${ownerProtocol}://${ownerAddress}`;\n    this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n    const start = performance.now();\n    await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n    const elapsed = (performance.now() - start).toFixed(2);\n    this.#lc.info?.(\n      `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n    );\n\n    if (purgeLock) {\n      // Once ownership has been assumed, any initial purge-lock preventing the\n      // purging of change-log records can be released, as a change-streamer\n      // that was attempting to purge records will correspondingly abort on the\n      // ownership check.\n      void purgeLock.release();\n    }\n  }\n\n  /**\n   * The whole of what a stream connection starts from: where to resume, what to\n   * ask the change source to backfill, and the raw cookie set those requests\n   * were derived from.\n   *\n   * All four statements run in one REPEATABLE READ transaction, so they see one\n   * snapshot. That is invariant 15 — the resume watermark and the cookies must\n   * come from the same store, read at the same position — and it is why the\n   * cookies are read here rather than by a second call: a cookie set paired\n   * with a watermark it was not folded to loses every backfill that completed\n   * in the interval, and those rows are not replayable from the slot.\n   *\n   * `cookies` is not `backfillRequests` in another shape. The request list is\n   * driven off `backfilling`, so it drops the metadata of every table with no\n   * in-flight backfill — correct for starting a stream, lossy as the cookie\n   * snapshot that the SQLite change log continues folding onto.\n   */\n  async getStartStreamInitializationParameters(): Promise<{\n    lastWatermark: string;\n    backfillRequests: BackfillRequest[];\n    cookies: CookieSet;\n  }> {\n    const [[{lastWatermark}], result, tableMetadata, backfilling] = await runTx(\n      this.#db,\n      sql => [\n        sql<{lastWatermark: string}[]>`\n        SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n        // Formats a BackfillRequest using json_object_agg() to construct the\n        // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n        // to make it optional and possibly `null`.\n        sql`\n        SELECT\n            json_build_object(\n              'schema', b.\"schema\",\n              'name', b.\"table\",\n              'metadata', t.\"metadata\"\n            ) as \"table\",\n            json_object_agg(b.\"column\", b.\"backfill\")\n              as \"columns\"\n          FROM ${this.#cdc('backfilling')} as b\n          LEFT JOIN ${this.#cdc('tableMetadata')} as t\n          ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n          GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n        `,\n\n        // Ordered by primary key, so that this set and the SQLite change log's\n        // can be compared row for row. `COLLATE \"C\"` because that comparison is\n        // against a store whose TEXT columns sort by byte: a linguistic\n        // collation orders `foo-bar` and `foobar` differently than SQLite's\n        // BINARY does, which would read as a divergence rather than as the\n        // identical set it is.\n        sql<TableMetadataCookie[]>`\n        SELECT \"schema\", \"table\", \"metadata\" FROM ${this.#cdc('tableMetadata')}\n          ORDER BY \"schema\" COLLATE \"C\", \"table\" COLLATE \"C\"`,\n\n        sql<BackfillCookie[]>`\n        SELECT \"schema\", \"table\", \"column\", \"backfill\" FROM ${this.#cdc('backfilling')}\n          ORDER BY \"schema\" COLLATE \"C\", \"table\" COLLATE \"C\", \"column\" COLLATE \"C\"`,\n      ],\n      {mode: Mode.READONLY},\n    );\n\n    return {\n      lastWatermark,\n      backfillRequests: v.parse(result, backfillRequestsSchema),\n      cookies: {\n        tableMetadata: [...tableMetadata],\n        backfilling: [...backfilling],\n      },\n    };\n  }\n\n  async getMinWatermarkForCatchup(): Promise<string | null> {\n    const [{minWatermark}] = await this.#db<{minWatermark: string | null}[]>\n    /*sql*/ `\n      SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n    return minWatermark;\n  }\n\n  purgeRecordsBefore(watermark: string): Promise<number> {\n    return runTx(this.#db, async sql => {\n      // This NOWAIT pre-check is an optimization to abort the transaction\n      // (and release associated resources) early.\n      await sql<{watermark: string}[]>`\n          SELECT watermark FROM ${this.#cdc('changeLog')}\n            ORDER BY watermark, pos LIMIT 1\n            FOR UPDATE NOWAIT\n        `;\n      // If the row is purge-locked by an incoming replication-manager, it\n      // will assume ownership of the change-log before releasing the lock.\n      // This DELETE blocks until the lock is released, allowing the change\n      // in ownership to be reliably detected (and the transaction aborted)\n      // in the subsequent check.\n      const [{deleted}] = await sql<{deleted: bigint}[]>`\n        -- The backup watermark can be ahead of the durable changeLog if the\n        -- storer is behind but the backup replica has consumed forwarded\n        -- changes. Preserve the latest durable changeLog transaction as the\n        -- catchup boundary instead of assuming the backup watermark exists.\n        -- The storer inserts each changeLog transaction atomically, so any\n        -- durable row for a watermark implies the full transaction is durable.\n        WITH keep AS (\n          SELECT max(watermark) AS watermark\n          FROM ${this.#cdc('changeLog')}\n        ), purged AS (\n          DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n            AND watermark < (SELECT watermark FROM keep)\n            RETURNING watermark, pos\n        ) SELECT COUNT(*) as deleted FROM purged;`;\n\n      const [{owner}] = await sql<ReplicationOwner[]>`\n        SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR SHARE`;\n      if (owner !== this.#taskID) {\n        throw new AbortError(\n          `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n        );\n      }\n      return Number(deleted);\n    });\n  }\n\n  /**\n   * @returns The JSON stringified stream message to be sent downstream.\n   */\n  store(watermark: string, data: ChangeStreamData) {\n    // Eagerly stringify the JSON payload to:\n    // - avoid redundant stringification when fanning out to subscribers\n    // - efficiently estimate the amount of memory the payload consumes\n    const json = serializeChangeStreamData(data);\n    this.#approximateQueuedBytes += json.length;\n\n    const change = data[1];\n    this.#queue.enqueue([\n      'change',\n      watermark,\n      json,\n      isDataChange(change) ? null : change, // drop DataChanges to save memory\n    ]);\n\n    return json;\n  }\n\n  abort() {\n    this.#queue.enqueue(['abort']);\n  }\n\n  catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n    this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n  }\n\n  #readyForMore: Resolver<void> | null = null;\n  #backpressureTimeout: NodeJS.Timeout | undefined;\n\n  readyForMore(): Promise<void> | undefined {\n    if (!this.#running) {\n      return undefined;\n    }\n    if (\n      this.#readyForMore === null &&\n      this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n    ) {\n      this.#lc.warn?.(\n        `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\\n` +\n          `\\n` +\n          `To inspect changeLog backlog in your change DB:\\n` +\n          `  SELECT\\n` +\n          `    (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\\n` +\n          `    change->>'tag' AS operation,\\n` +\n          `    COUNT(*) AS count\\n` +\n          `  FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n          `  GROUP BY 1, 2\\n` +\n          `  ORDER BY 3 DESC\\n` +\n          `  LIMIT 20;`,\n      );\n      this.#readyForMore = resolver();\n      this.#resetBackpressureTimeout(true);\n    }\n    return this.#readyForMore?.promise;\n  }\n\n  #resetBackpressureTimeout(reschedule: boolean) {\n    clearTimeout(this.#backpressureTimeout);\n    this.#backpressureTimeout = reschedule\n      ? setTimeout(() => {\n          this.#readyForMore?.reject(\n            new AbortError(\n              `No statement processed within the last ${this.#statementTimeoutMs}ms. Considering the change-log to be wedged.`,\n            ),\n          );\n          this.#readyForMore = null;\n        }, this.#statementTimeoutMs)\n      : undefined;\n  }\n\n  #maybeReleaseBackPressure() {\n    if (this.#readyForMore !== null) {\n      // Wait for at least 20% of the threshold to free up.\n      if (\n        this.#approximateQueuedBytes <\n        this.#backPressureThresholdBytes * 0.8\n      ) {\n        this.#lc.info?.(\n          `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`,\n        );\n        this.#readyForMore.resolve();\n        this.#readyForMore = null;\n        this.#resetBackpressureTimeout(false);\n      } else {\n        this.#resetBackpressureTimeout(true);\n      }\n    }\n  }\n\n  /**\n   * Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.\n   *\n   * Uses `json_to_recordset()` so the batch is a single JSON parameter: the\n   * statement text stays constant regardless of batch size, avoiding the\n   * unbounded prepared-statement variants (and Postgres memory growth) that a\n   * multi-row INSERT would produce. See rocicorp/mono#3511.\n   *\n   * Returns (and updates) {@link PendingTransaction.lastFlush}; a no-op when the\n   * batch is empty.\n   */\n  #flushChangeLog(tx: PendingTransaction): Promise<unknown> | undefined {\n    const {batch} = tx;\n    if (batch.length === 0) {\n      return tx.lastFlush;\n    }\n    tx.batch = [];\n    tx.lastFlush = tx.pool.process(sql => [\n      sql`\n        INSERT INTO ${this.#cdc('changeLog')} (\"watermark\", \"pos\", \"change\", \"precommit\")\n        SELECT \"watermark\", \"pos\", \"change\"::json, \"precommit\"\n          FROM json_to_recordset(${batch}) AS x(\n            \"watermark\" TEXT,\n            \"pos\" INT8,\n            \"change\" TEXT,\n            \"precommit\" TEXT\n          )`,\n    ]);\n    return tx.lastFlush;\n  }\n\n  #stopped = promiseVoid;\n\n  /**\n   * Runs the storer loop until {@link stop()} is called, or an error is thrown.\n   * Once {@link run()} completes, it can be called again.\n   */\n  async run() {\n    assert(!this.#running, `storer is already running`);\n\n    const {promise: stopped, resolve: signalStopped} = resolver();\n    this.#running = true;\n    this.#stopped = stopped;\n\n    this.#lc.info?.('starting storer');\n    let err: unknown;\n    try {\n      await this.#processQueue();\n    } catch (e) {\n      err = e; // used in finally\n      throw e;\n    } finally {\n      // Release any pending backpressure so the upstream can proceed\n      if (this.#readyForMore !== null) {\n        this.#readyForMore.resolve();\n        this.#readyForMore = null;\n      }\n      this.#resetBackpressureTimeout(false);\n      this.#cancelQueueEntries(\n        this.#queue.drain().filter(entry => entry !== undefined),\n        err,\n      );\n      this.#running = false;\n      signalStopped();\n      this.#lc.info?.('storer stopped');\n    }\n  }\n\n  #cancelQueueEntries(queue: QueueEntry[], e: unknown) {\n    if (queue.length === 0) {\n      return;\n    }\n    this.#lc.info?.(\n      `canceling ${queue.length} entries from the changeLog queue`,\n    );\n    const err = e instanceof Error ? e : new AbortError('server shutting down');\n    for (const entry of queue) {\n      if (entry === 'stop') {\n        continue;\n      }\n      const type = entry[0];\n      switch (type) {\n        case 'subscriber': {\n          // Disconnect subscribers waiting to be caught up so that they can\n          // reconnect and try again.\n          const {subscriber} = entry[1];\n          this.#lc.info?.(`disconnecting ${subscriber.id}`);\n          subscriber.fail(err);\n          break;\n        }\n      }\n    }\n  }\n\n  async #processQueue() {\n    let tx: PendingTransaction | null = null;\n    let msg: QueueEntry | false;\n\n    const catchupQueue: SubscriberAndMode[] = [];\n    try {\n      while ((msg = await this.#queue.dequeue()) !== 'stop') {\n        const [msgType] = msg;\n        switch (msgType) {\n          case 'ready': {\n            const signalReady = msg[1];\n            signalReady();\n            continue;\n          }\n          case 'subscriber': {\n            const subscriber = msg[1];\n            if (tx) {\n              catchupQueue.push(subscriber); // Wait for the current tx to complete.\n            } else {\n              await this.#startCatchup([subscriber]); // Catch up immediately.\n            }\n            continue;\n          }\n          case 'abort': {\n            if (tx) {\n              tx.pool.abort();\n              await tx.pool.done();\n              tx = null;\n            }\n            continue;\n          }\n        }\n        // msgType === 'change'\n        const [_, watermark, json, change] = msg;\n        const tag = change?.tag;\n        this.#approximateQueuedBytes -= json.length;\n\n        if (tag === 'begin') {\n          assert(!tx, 'received BEGIN in the middle of a transaction');\n          const {promise, resolve, reject} = resolver<ReplicationOwner>();\n          void promise.catch(() => {}); // handle rejections before the await\n          tx = {\n            pool: new TransactionPool(\n              this.#lc.withContext('watermark', watermark),\n              {\n                mode: Mode.READ_COMMITTED,\n                statementResponseTimeout: this.#statementTimeoutMs,\n              },\n            ),\n            preCommitWatermark: watermark,\n            pos: 0,\n            startingReplicationState: promise,\n            ack: !change.skipAck,\n            batch: [],\n            lastFlush: undefined,\n          };\n          tx.pool.run(this.#db);\n          // Acquire a lock on the replicationState row to detect and/or prevent\n          // a concurrent ownership change.\n          void tx.pool.process(tx => {\n            tx<ReplicationOwner[]> /*sql*/ `\n          SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n              ([result]) => resolve(result),\n              reject,\n            );\n            return [];\n          });\n        } else {\n          assert(tx, () => `received change outside of transaction: ${json}`);\n          tx.pos++;\n        }\n\n        const entry: ChangeLogRow = {\n          watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n          precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n          pos: tx.pos,\n          // For backwards compatibility, only the change message is stored\n          // in the cdc changeLog.\n          change: extractChangeSubstring(json, tag),\n        };\n\n        if (change !== null && isSchemaChange(change)) {\n          // Schema changes carry backfill / table-metadata statements that\n          // must be applied in stream order relative to the changeLog rows.\n          // Flush any buffered rows first, then write this row together with\n          // its metadata statements as a single unit (preserving the previous\n          // per-change ordering for schema changes).\n          await this.#flushChangeLog(tx);\n          tx.lastFlush = tx.pool.process(sql => [\n            sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n            ...this.#trackBackfillMetadata(sql, change),\n          ]);\n        } else {\n          // Accumulate plain changeLog rows (begin, data changes, commit) and\n          // write them as a single multi-row INSERT. Collapsing the per-change\n          // single-row INSERTs into batches is the dominant cost reduction for\n          // large transactions, where the previous one-statement-per-change\n          // path dominated the upstream replication lag.\n          tx.batch.push(entry);\n          if (tx.batch.length >= this.#changeLogBatchSize) {\n            // Bound pipeline depth (and thus memory) by awaiting the previous\n            // flush before issuing the next. This is the batched analog of the\n            // previous per-100-statement backpressure await, and likewise\n            // guards against memory blowup on very large transactions.\n            const prevFlush = tx.lastFlush;\n            void this.#flushChangeLog(tx);\n            await prevFlush;\n          }\n        }\n        this.#maybeReleaseBackPressure();\n\n        if (tag === 'commit') {\n          // Flush any remaining buffered changeLog rows (including this commit\n          // row) before updating the replication state, so the state update is\n          // ordered after all changeLog inserts for this transaction.\n          void this.#flushChangeLog(tx);\n\n          const {owner} = await tx.startingReplicationState;\n          if (owner !== this.#taskID) {\n            // Ownership change reflected in the replicationState read in 'begin'.\n            tx.pool.fail(\n              new AbortError(\n                `changeLog ownership has been assumed by ${owner}`,\n              ),\n            );\n          } else {\n            // Update the replication state.\n            const lastWatermark = watermark;\n            void tx.pool.process(tx => [\n              tx`\n            UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n            ]);\n            tx.pool.setDone();\n          }\n\n          await tx.pool.done();\n\n          // ACK the LSN to the upstream Postgres.\n          if (tx.ack) {\n            this.#onCommitted(['commit', change, {watermark}]);\n          }\n          tx = null;\n\n          // Before beginning the next transaction, open a READONLY snapshot to\n          // concurrently catchup any queued subscribers.\n          await this.#startCatchup(catchupQueue.splice(0));\n        } else if (tag === 'rollback') {\n          // Aborted transactions are not stored in the changeLog. Abort the current tx\n          // and process catchup of subscribers that were waiting for it to end.\n          tx.pool.abort();\n          await tx.pool.done();\n          tx = null;\n\n          await this.#startCatchup(catchupQueue.splice(0));\n        }\n      }\n    } catch (e) {\n      catchupQueue.forEach(({subscriber}) => subscriber.fail(e));\n      throw e;\n    }\n  }\n\n  async #startCatchup(subs: SubscriberAndMode[]) {\n    if (subs.length === 0) {\n      return;\n    }\n\n    const lc = this.#lc.withContext('pool', 'catchup');\n    // TODO: Consider setting initialWorkers to accomodate parallel\n    //       catchups of multiple subscribers. The tricky part is\n    //       staying within the db's max connections.\n    const reader = new TransactionPool(lc, {mode: Mode.READONLY});\n    reader.run(this.#db);\n\n    let lastWatermark: string | undefined;\n    try {\n      // Ensure that the transaction has started (and is thus holding a snapshot\n      // of the database) before continuing on to commit more changes. This is\n      // done by performing a single read on the db, which determines the\n      // snapshot for the REPEATABLE_READ transaction.\n      [{lastWatermark}] = await reader.processReadTask(\n        sql => sql<{lastWatermark: string}[]>`\n        SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}\n      `,\n      );\n    } catch (e) {\n      subs.map(({subscriber}) => subscriber.fail(e));\n      throw e;\n    }\n\n    // Run the actual catchup queries in the background. Errors are handled in\n    // #catchup() by disconnecting the associated subscriber.\n    void Promise.all(\n      subs.map(sub =>\n        this.#catchup(\n          lc.withContext('subscriber', sub.subscriber.id),\n          sub,\n          lastWatermark,\n          reader,\n        ),\n      ),\n    ).finally(() => reader.setDone());\n  }\n\n  async #catchup(\n    lc: LogContext,\n    {subscriber: sub, mode}: SubscriberAndMode,\n    lastWatermark: string,\n    reader: TransactionPool,\n  ) {\n    try {\n      lc.info?.(`starting catchup`);\n      await reader.processReadTask(async tx => {\n        lc.info?.(`catching up`);\n        const start = Date.now();\n\n        // When starting from initial-sync, there won't be a change with a watermark\n        // equal to the replica version. This is the empty changeLog scenario.\n        let watermarkFound = sub.watermark === this.#replicaVersion;\n        let count = 0;\n        let lastBatchConsumed: Promise<unknown> | undefined;\n\n        for await (const entries of tx<ChangeLogEntry[]> /*sql*/ `\n          SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc('changeLog')}\n           WHERE watermark >= ${sub.watermark}\n             AND watermark <= ${lastWatermark}\n           ORDER BY watermark, pos`.cursor(2000)) {\n          // Wait for the last batch of entries to be consumed by the\n          // subscriber before sending down the current batch. This pipelining\n          // allows one batch of changes to be received from the change-db\n          // while the previous batch of changes are sent to the subscriber,\n          // resulting in flow control that caps the number of changes\n          // referenced in memory to 2 * batch-size.\n          const start = performance.now();\n          await lastBatchConsumed;\n          const elapsed = performance.now() - start;\n          if (lastBatchConsumed) {\n            lc[elapsed > 100 ? 'info' : 'debug']?.(\n              `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n            );\n          }\n\n          for (const entry of entries) {\n            if (entry.watermark === sub.watermark) {\n              // This should be the first entry.\n              // Catchup starts from *after* the watermark.\n              watermarkFound = true;\n            } else if (watermarkFound) {\n              lastBatchConsumed = sub.catchup(\n                reconstructWatermarkedChange(entry),\n              );\n              count++;\n            } else if (mode === 'backup') {\n              throw new AutoResetSignal(\n                `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n              );\n            } else {\n              lc.warn?.(\n                `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n              );\n              sub.close(\n                ErrorType.WatermarkTooOld,\n                `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n              );\n              return;\n            }\n          }\n        }\n        if (watermarkFound) {\n          await lastBatchConsumed;\n          lc.info?.(\n            `caught up ${sub.id} with ${count} changes (${\n              Date.now() - start\n            } ms)`,\n          );\n        } else {\n          // The subscriber is ahead of the latest durable changeLog entry\n          // (lastWatermark). This can legitimately happen: changes are\n          // forwarded to subscribers (the backup replica and view-syncers)\n          // concurrently with — and can outrun — the durable store, so a\n          // replica may briefly lead the change DB after the storer falls\n          // behind or the change-streamer restarts. No catchup is possible or\n          // needed; once the change DB catches back up, forwarding resumes and\n          // the subscriber dedups any watermarks it already has. Unlike the\n          // AutoResetSignal / WatermarkTooOld cases above, this is not a gap in\n          // replication history, so the subscriber is simply marked caught up.\n          lc.warn?.(\n            `subscriber ${sub.id} at watermark ${sub.watermark} is ahead of ` +\n              `the latest durable watermark ${lastWatermark}; waiting for the ` +\n              `change DB to catch up`,\n          );\n        }\n        // Start draining messages buffered during catchup. The returned promise\n        // is intentionally not awaited here: while the drain is in progress,\n        // new sends keep appending to the subscriber backlog and inherit its\n        // byte-based backpressure.\n        void sub.setCaughtUp();\n      });\n    } catch (err) {\n      lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n      if (err instanceof AutoResetSignal) {\n        await markResetRequired(this.#db, this.#shard);\n        this.#onFatal(err);\n      }\n      sub.fail(err);\n    }\n  }\n\n  /**\n   * Returns the db statements necessary to track backfill and table metadata\n   * presented in the `change`, if any.\n   *\n   * This is the Postgres interpreter of {@link cookieOps}, which is also\n   * interpreted against the SQLite change log by `ChangeLogCookieWriter`. The\n   * two stores have to agree on every transition forever, and the failure mode\n   * if they drift is a backfill that is silently never re-requested — so the\n   * decision of what a schema change *means* lives in one place and only its\n   * transport lives here.\n   */\n  #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n    return cookieOps(change).flatMap(op => this.#cookieStmts(sql, op));\n  }\n\n  #cookieStmts(sql: PostgresTransaction, op: CookieOp): PendingQuery<Row[]>[] {\n    switch (op.op) {\n      case 'upsert-metadata':\n        return [this.#upsertTableMetadataStmt(sql, op.table, op.metadata)];\n\n      case 'upsert-backfill':\n        return [\n          this.#upsertColumnBackfillStmt(sql, op.table, op.column, op.backfill),\n        ];\n\n      case 'rename-table': {\n        const {old} = op;\n        const row = {schema: op.new.schema, table: op.new.name};\n        return [\n          sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n          sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n        ];\n      }\n\n      case 'drop-table': {\n        const {schema, name} = op.table;\n        return [\n          sql`DELETE FROM ${this.#cdc('tableMetadata')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n        ];\n      }\n\n      case 'rename-column': {\n        const {schema, name: table} = op.table;\n        return [\n          sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${op.new}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${op.old}`,\n        ];\n      }\n\n      case 'drop-column': {\n        const {schema, name} = op.table;\n        return [\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${op.column}`,\n        ];\n      }\n\n      case 'complete-backfill': {\n        const {schema, name: table} = op.table;\n        return [\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql([...op.columns])}`,\n        ];\n      }\n\n      default:\n        unreachable(op);\n    }\n  }\n\n  #upsertTableMetadataStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    metadata: TableMetadata,\n  ) {\n    const row: TableMetadataRow = {schema, table, metadata};\n    return sql`\n        INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\")\n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  #upsertColumnBackfillStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    column: string,\n    backfill: BackfillID,\n  ) {\n    const row: BackfillingColumn = {schema, table, column, backfill};\n    return sql`\n        INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\", \"column\")\n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  /**\n   * Waits until all currently queued entries have been processed.\n   * This is only used in tests.\n   */\n  async allProcessed() {\n    if (this.#running) {\n      const {promise, resolve} = resolver();\n      this.#queue.enqueue(['ready', resolve]);\n      await promise;\n    }\n  }\n\n  /**\n   * Stops the storer and waits up to a drain timeout for entries to drain,\n   * throwing an exception if it doesn't drain in time, in order to abort\n   * the server when PG (or the connection to it) appears to be wedged.\n   */\n  async stop() {\n    if (this.#running) {\n      this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n      this.abort(); // for cleanliness, abort any open transactions\n      this.#queue.enqueue('stop');\n    }\n    if (\n      (await orTimeout(this.#stopped, this.#drainTimeoutMs)) === 'timed-out'\n    ) {\n      throw new AbortError(\n        `changeLog did not drain within ${this.#drainTimeoutMs}ms`,\n      );\n    }\n  }\n}\n\nexport class PurgeLock {\n  readonly #lc: LogContext;\n  readonly #tx: TransactionPool;\n  readonly replicaVersion: string;\n  readonly minWatermark: string;\n\n  constructor(\n    lc: LogContext,\n    tx: TransactionPool,\n    replicaVersion: string,\n    watermark: string,\n  ) {\n    this.#lc = lc;\n    this.#tx = tx;\n    this.replicaVersion = replicaVersion;\n    this.minWatermark = watermark;\n  }\n\n  #released = false;\n\n  async release() {\n    if (this.#released) {\n      return;\n    }\n    this.#released = true;\n    this.#tx.setDone();\n    await this.#tx\n      .done()\n      .catch(e => this.#lc.warn?.(`error from purge-lock release`, e));\n    this.#lc.info?.(`released purge lock on ${this.minWatermark}`);\n  }\n}\n\nexport class PurgeLocker {\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #db: PostgresDB;\n\n  constructor(lc: LogContext, shard: ShardID, db: PostgresDB) {\n    this.#lc = lc.withContext('component', 'purge-locker');\n    this.#shard = shard;\n    this.#db = db;\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async acquire() {\n    const tx = new TransactionPool(this.#lc, {mode: Mode.READ_COMMITTED}).run(\n      this.#db,\n    );\n    const row = await tx.processReadTask(\n      sql => sql<{watermark: string}[]>`\n      SELECT watermark FROM ${this.#cdc('changeLog')}\n        ORDER BY watermark, pos LIMIT 1\n        FOR SHARE \n    `,\n    );\n    if (row.length === 0) {\n      this.#lc.info?.(`changeLog is empty. No rows to purge-lock.`);\n      tx.setDone();\n      await tx.done();\n      return null;\n    }\n    const [{watermark}] = row;\n    const [{replicaVersion}] = await tx.processReadTask(\n      sql => sql<{replicaVersion: string}[]>`\n        SELECT \"replicaVersion\" FROM ${this.#cdc('replicationConfig')}\n      `,\n    );\n    this.#lc.info?.(\n      `locked watermark ${watermark} from being purged from replica@${replicaVersion}`,\n    );\n    return new PurgeLock(this.#lc, tx, replicaVersion, watermark);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA0FA,IAAM,yBAAyB,eAAE,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkD5D,IAAa,SAAb,MAAuC;CACrC,KAAc;CACd;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,MAAkB;CACxC;CACA;CACA;CACA;CAEA,0BAA0B;CAC1B,WAAW;CAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,aACA,SACA,EACE,iCACA,oBACA,oBACA,iBAAiB,OAEnB;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAKC,SAAS;EACd,KAAKC,UAAU;EACf,KAAKC,oBAAoB;EACzB,KAAKC,qBAAqB;EAC1B,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;EACpB,KAAKC,WAAW;EAChB,KAAKG,sBAAsB;EAC3B,KAAKC,sBAAsB,KAAK,IAAI,GAAG,kBAAkB;EACzD,KAAKC,kBAAkB;EAEvB,MAAM,YAAY,kBAAkB;EACpC,KAAKH,+BACF,UAAU,kBAAkB,UAAU,kBACvC;EAEF,KAAKV,IAAI,OACP,gBAAgB,KAAKU,8BAA8B,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAC5C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAE/E,EAAC,UAAS,CACZ;CACF;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKL,IAAI,GAAG,UAAU,KAAKJ,MAAM,EAAE,GAAG,OAAO;CACtD;CAEA,MAAM,gBAAgB,WAA8B;EAClD,MAAM,KAAK,KAAKI;EAChB,MAAM,QAAQ,KAAKH;EACnB,MAAM,eAAe,KAAKC;EAC1B,MAAM,gBAAgB,KAAKC;EAE3B,MAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,cAAc,KAAK;EAC5B,KAAKJ,IAAI,OAAO,yBAAyB,qBAAqB;EAC9D,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,EAAE,UAAU,KAAKc,KAAK,kBAAkB,EAAE,OAAO,GAAG;GAAC;GAAO,cAAc;EAAmB,CAAC;EACpG,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;EACrD,KAAKd,IAAI,OACP,wBAAwB,oBAAoB,IAAI,QAAQ,KAC1D;EAEA,IAAI,WAKF,UAAe,QAAQ;CAE3B;;;;;;;;;;;;;;;;;;CAmBA,MAAM,yCAIH;EACD,MAAM,CAAC,CAAC,EAAC,kBAAiB,QAAQ,eAAe,eAAe,MAAM,MACpE,KAAKK,MACL,QAAO;GACL,GAA8B;sCACA,KAAKS,KAAK,kBAAkB;GAK1D,GAAG;;;;;;;;;iBASM,KAAKA,KAAK,aAAa,EAAE;sBACpB,KAAKA,KAAK,eAAe,EAAE;;;;GAWzC,GAA0B;oDACkB,KAAKA,KAAK,eAAe,EAAE;;GAGvE,GAAqB;8DACiC,KAAKA,KAAK,aAAa,EAAE;;EAEjF,GACA,EAAC,MAAM,SAAa,CACtB;EAEA,OAAO;GACL;GACA,kBAAkB,MAAQ,QAAQ,sBAAsB;GACxD,SAAS;IACP,eAAe,CAAC,GAAG,aAAa;IAChC,aAAa,CAAC,GAAG,WAAW;GAC9B;EACF;CACF;CAEA,MAAM,4BAAoD;EACxD,MAAM,CAAC,EAAC,kBAAiB,MAAM,KAAKT,GAC5B;qDACyC,KAAKS,KAAK,WAAW;EACtE,OAAO;CACT;CAEA,mBAAmB,WAAoC;EACrD,OAAO,MAAM,KAAKT,KAAK,OAAM,QAAO;GAGlC,MAAM,GAA0B;kCACJ,KAAKS,KAAK,WAAW,EAAE;;;;GASnD,MAAM,CAAC,EAAC,aAAY,MAAM,GAAwB;;;;;;;;;iBASvC,KAAKA,KAAK,WAAW,EAAE;;wBAEhB,KAAKA,KAAK,WAAW,EAAE,qBAAqB,UAAU;;;;GAKxE,MAAM,CAAC,EAAC,WAAU,MAAM,GAAuB;8BACvB,KAAKA,KAAK,kBAAkB,EAAE;GACtD,IAAI,UAAU,KAAKZ,SACjB,MAAM,IAAI,WACR,+BAA+B,UAAU,uCAAuC,OAClF;GAEF,OAAO,OAAO,OAAO;EACvB,CAAC;CACH;;;;CAKA,MAAM,WAAmB,MAAwB;EAI/C,MAAM,OAAO,0BAA0B,IAAI;EAC3C,KAAKa,2BAA2B,KAAK;EAErC,MAAM,SAAS,KAAK;EACpB,KAAKN,OAAO,QAAQ;GAClB;GACA;GACA;GACA,aAAa,MAAM,IAAI,OAAO;EAChC,CAAC;EAED,OAAO;CACT;CAEA,QAAQ;EACN,KAAKA,OAAO,QAAQ,CAAC,OAAO,CAAC;CAC/B;CAEA,QAAQ,YAAwB,MAAsB;EACpD,KAAKA,OAAO,QAAQ,CAAC,cAAc;GAAC;GAAY;EAAI,CAAC,CAAC;CACxD;CAEA,gBAAuC;CACvC;CAEA,eAA0C;EACxC,IAAI,CAAC,KAAKO,UACR;EAEF,IACE,KAAKC,kBAAkB,QACvB,KAAKF,0BAA0B,KAAKL,6BACpC;GACA,KAAKV,IAAI,OACP,+BAA+B,KAAKS,OAAO,KAAK,EAAE,qBAAqB,KAAKM,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,2SAW9H;GACA,KAAKE,gBAAgB,SAAS;GAC9B,KAAKC,0BAA0B,IAAI;EACrC;EACA,OAAO,KAAKD,eAAe;CAC7B;CAEA,0BAA0B,YAAqB;EAC7C,aAAa,KAAKE,oBAAoB;EACtC,KAAKA,uBAAuB,aACxB,iBAAiB;GACf,KAAKF,eAAe,OAClB,IAAI,WACF,0CAA0C,KAAKN,oBAAoB,6CACrE,CACF;GACA,KAAKM,gBAAgB;EACvB,GAAG,KAAKN,mBAAmB,IAC3B,KAAA;CACN;CAEA,4BAA4B;EAC1B,IAAI,KAAKM,kBAAkB,MAEzB,IACE,KAAKF,0BACL,KAAKL,8BAA8B,IACnC;GACA,KAAKV,IAAI,OACP,gCAAgC,KAAKS,OAAO,KAAK,EAAE,qBAAqB,KAAKM,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,KAC/H;GACA,KAAKE,cAAc,QAAQ;GAC3B,KAAKA,gBAAgB;GACrB,KAAKC,0BAA0B,KAAK;EACtC,OACE,KAAKA,0BAA0B,IAAI;CAGzC;;;;;;;;;;;;CAaA,gBAAgB,IAAsD;EACpE,MAAM,EAAC,UAAS;EAChB,IAAI,MAAM,WAAW,GACnB,OAAO,GAAG;EAEZ,GAAG,QAAQ,CAAC;EACZ,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG;sBACa,KAAKJ,KAAK,WAAW,EAAE;;mCAEV,MAAM;;;;;YAMrC,CAAC;EACD,OAAO,GAAG;CACZ;CAEA,WAAW;;;;;CAMX,MAAM,MAAM;EACV,OAAO,CAAC,KAAKE,UAAU,2BAA2B;EAElD,MAAM,EAAC,SAAS,SAAS,SAAS,kBAAiB,SAAS;EAC5D,KAAKA,WAAW;EAChB,KAAKI,WAAW;EAEhB,KAAKpB,IAAI,OAAO,iBAAiB;EACjC,IAAI;EACJ,IAAI;GACF,MAAM,KAAKqB,cAAc;EAC3B,SAAS,GAAG;GACV,MAAM;GACN,MAAM;EACR,UAAU;GAER,IAAI,KAAKJ,kBAAkB,MAAM;IAC/B,KAAKA,cAAc,QAAQ;IAC3B,KAAKA,gBAAgB;GACvB;GACA,KAAKC,0BAA0B,KAAK;GACpC,KAAKI,oBACH,KAAKb,OAAO,MAAM,EAAE,QAAO,UAAS,UAAU,KAAA,CAAS,GACvD,GACF;GACA,KAAKO,WAAW;GAChB,cAAc;GACd,KAAKhB,IAAI,OAAO,gBAAgB;EAClC;CACF;CAEA,oBAAoB,OAAqB,GAAY;EACnD,IAAI,MAAM,WAAW,GACnB;EAEF,KAAKA,IAAI,OACP,aAAa,MAAM,OAAO,kCAC5B;EACA,MAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,WAAW,sBAAsB;EAC1E,KAAK,MAAM,SAAS,OAAO;GACzB,IAAI,UAAU,QACZ;GAGF,QADa,MAAM,IACnB;IACE,KAAK,cAAc;KAGjB,MAAM,EAAC,eAAc,MAAM;KAC3B,KAAKA,IAAI,OAAO,iBAAiB,WAAW,IAAI;KAChD,WAAW,KAAK,GAAG;KACnB;IACF;GACF;EACF;CACF;CAEA,MAAMqB,gBAAgB;EACpB,IAAI,KAAgC;EACpC,IAAI;EAEJ,MAAM,eAAoC,CAAC;EAC3C,IAAI;GACF,QAAQ,MAAM,MAAM,KAAKZ,OAAO,QAAQ,OAAO,QAAQ;IACrD,MAAM,CAAC,WAAW;IAClB,QAAQ,SAAR;KACE,KAAK,SAAS;MACZ,MAAM,cAAc,IAAI;MACxB,YAAY;MACZ;KACF;KACA,KAAK,cAAc;MACjB,MAAM,aAAa,IAAI;MACvB,IAAI,IACF,aAAa,KAAK,UAAU;WAE5B,MAAM,KAAKc,cAAc,CAAC,UAAU,CAAC;MAEvC;KACF;KACA,KAAK;MACH,IAAI,IAAI;OACN,GAAG,KAAK,MAAM;OACd,MAAM,GAAG,KAAK,KAAK;OACnB,KAAK;MACP;MACA;IAEJ;IAEA,MAAM,CAAC,GAAG,WAAW,MAAM,UAAU;IACrC,MAAM,MAAM,QAAQ;IACpB,KAAKR,2BAA2B,KAAK;IAErC,IAAI,QAAQ,SAAS;KACnB,OAAO,CAAC,IAAI,+CAA+C;KAC3D,MAAM,EAAC,SAAS,SAAS,WAAU,SAA2B;KAC9D,QAAa,YAAY,CAAC,CAAC;KAC3B,KAAK;MACH,MAAM,IAAI,gBACR,KAAKf,IAAI,YAAY,aAAa,SAAS,GAC3C;OACE,MAAM;OACN,0BAA0B,KAAKW;MACjC,CACF;MACA,oBAAoB;MACpB,KAAK;MACL,0BAA0B;MAC1B,KAAK,CAAC,OAAO;MACb,OAAO,CAAC;MACR,WAAW,KAAA;KACb;KACA,GAAG,KAAK,IAAI,KAAKN,GAAG;KAGpB,GAAQ,KAAK,SAAQ,OAAM;MACzB,EAA+B;gCACX,KAAKS,KAAK,kBAAkB,EAAE,aAAa,MAC5D,CAAC,YAAY,QAAQ,MAAM,GAC5B,MACF;MACA,OAAO,CAAC;KACV,CAAC;IACH,OAAO;KACL,OAAO,UAAU,2CAA2C,MAAM;KAClE,GAAG;IACL;IAEA,MAAM,QAAsB;KAC1B,WAAW,QAAQ,WAAW,YAAY,GAAG;KAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;KACtD,KAAK,GAAG;KAGR,QAAQ,uBAAuB,MAAM,GAAG;IAC1C;IAEA,IAAI,WAAW,QAAQ,eAAe,MAAM,GAAG;KAM7C,MAAM,KAAKU,gBAAgB,EAAE;KAC7B,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG,eAAe,KAAKV,KAAK,WAAW,EAAE,GAAG,IAAI,KAAK,KACrD,GAAG,KAAKW,uBAAuB,KAAK,MAAM,CAC5C,CAAC;IACH,OAAO;KAML,GAAG,MAAM,KAAK,KAAK;KACnB,IAAI,GAAG,MAAM,UAAU,KAAKb,qBAAqB;MAK/C,MAAM,YAAY,GAAG;MACrB,KAAUY,gBAAgB,EAAE;MAC5B,MAAM;KACR;IACF;IACA,KAAKE,0BAA0B;IAE/B,IAAI,QAAQ,UAAU;KAIpB,KAAUF,gBAAgB,EAAE;KAE5B,MAAM,EAAC,UAAS,MAAM,GAAG;KACzB,IAAI,UAAU,KAAKtB,SAEjB,GAAG,KAAK,KACN,IAAI,WACF,2CAA2C,OAC7C,CACF;UACK;MAEL,MAAM,gBAAgB;MACtB,GAAQ,KAAK,SAAQ,OAAM,CACzB,EAAE;qBACK,KAAKY,KAAK,kBAAkB,EAAE,OAAO,GAAG,EAAC,cAAa,CAAC,GAChE,CAAC;MACD,GAAG,KAAK,QAAQ;KAClB;KAEA,MAAM,GAAG,KAAK,KAAK;KAGnB,IAAI,GAAG,KACL,KAAKP,aAAa;MAAC;MAAU;MAAQ,EAAC,UAAS;KAAC,CAAC;KAEnD,KAAK;KAIL,MAAM,KAAKgB,cAAc,aAAa,OAAO,CAAC,CAAC;IACjD,OAAO,IAAI,QAAQ,YAAY;KAG7B,GAAG,KAAK,MAAM;KACd,MAAM,GAAG,KAAK,KAAK;KACnB,KAAK;KAEL,MAAM,KAAKA,cAAc,aAAa,OAAO,CAAC,CAAC;IACjD;GACF;EACF,SAAS,GAAG;GACV,aAAa,SAAS,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GACzD,MAAM;EACR;CACF;CAEA,MAAMA,cAAc,MAA2B;EAC7C,IAAI,KAAK,WAAW,GAClB;EAGF,MAAM,KAAK,KAAKvB,IAAI,YAAY,QAAQ,SAAS;EAIjD,MAAM,SAAS,IAAI,gBAAgB,IAAI,EAAC,MAAM,SAAa,CAAC;EAC5D,OAAO,IAAI,KAAKK,GAAG;EAEnB,IAAI;EACJ,IAAI;GAKF,CAAC,CAAC,kBAAkB,MAAM,OAAO,iBAC/B,QAAO,GAA8B;sCACP,KAAKS,KAAK,kBAAkB,EAAE;OAE9D;EACF,SAAS,GAAG;GACV,KAAK,KAAK,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GAC7C,MAAM;EACR;EAIA,QAAa,IACX,KAAK,KAAI,QACP,KAAKa,SACH,GAAG,YAAY,cAAc,IAAI,WAAW,EAAE,GAC9C,KACA,eACA,MACF,CACF,CACF,EAAE,cAAc,OAAO,QAAQ,CAAC;CAClC;CAEA,MAAMA,SACJ,IACA,EAAC,YAAY,KAAK,QAClB,eACA,QACA;EACA,IAAI;GACF,GAAG,OAAO,kBAAkB;GAC5B,MAAM,OAAO,gBAAgB,OAAM,OAAM;IACvC,GAAG,OAAO,aAAa;IACvB,MAAM,QAAQ,KAAK,IAAI;IAIvB,IAAI,iBAAiB,IAAI,cAAc,KAAKrB;IAC5C,IAAI,QAAQ;IACZ,IAAI;IAEJ,WAAW,MAAM,WAAW,EAA6B;sEACK,KAAKQ,KAAK,WAAW,EAAE;gCAC7D,IAAI,UAAU;gCACd,cAAc;oCACV,OAAO,GAAI,GAAG;KAOxC,MAAM,QAAQ,YAAY,IAAI;KAC9B,MAAM;KACN,MAAM,UAAU,YAAY,IAAI,IAAI;KACpC,IAAI,mBACF,GAAG,UAAU,MAAM,SAAS,WAC1B,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,IAAI,GAAG,0CAChD;KAGF,KAAK,MAAM,SAAS,SAClB,IAAI,MAAM,cAAc,IAAI,WAG1B,iBAAiB;UACZ,IAAI,gBAAgB;MACzB,oBAAoB,IAAI,QACtB,6BAA6B,KAAK,CACpC;MACA;KACF,OAAO,IAAI,SAAS,UAClB,MAAM,IAAI,gBACR,+BAA+B,IAAI,UAAU,wBAAwB,MAAM,UAAU,EACvF;UACK;MACL,GAAG,OACD,qCAAqC,IAAI,UAAU,wBAAwB,MAAM,UAAU,EAC7F;MACA,IAAI,MACF,GACA,mCAAmC,MAAM,UAAU,cAAc,IAAI,UAAU,EACjF;MACA;KACF;IAEJ;IACA,IAAI,gBAAgB;KAClB,MAAM;KACN,GAAG,OACD,aAAa,IAAI,GAAG,QAAQ,MAAM,YAChC,KAAK,IAAI,IAAI,MACd,KACH;IACF,OAWE,GAAG,OACD,cAAc,IAAI,GAAG,gBAAgB,IAAI,UAAU,4CACjB,cAAc,wCAElD;IAMF,IAAS,YAAY;GACvB,CAAC;EACH,SAAS,KAAK;GACZ,GAAG,QAAQ,sCAAsC,IAAI,MAAM,GAAG;GAC9D,IAAI,eAAe,iBAAiB;IAClC,MAAM,kBAAkB,KAAKT,KAAK,KAAKJ,MAAM;IAC7C,KAAKO,SAAS,GAAG;GACnB;GACA,IAAI,KAAK,GAAG;EACd;CACF;;;;;;;;;;;;CAaA,uBAAuB,KAA0B,QAAsB;EACrE,OAAO,UAAU,MAAM,EAAE,SAAQ,OAAM,KAAKoB,aAAa,KAAK,EAAE,CAAC;CACnE;CAEA,aAAa,KAA0B,IAAqC;EAC1E,QAAQ,GAAG,IAAX;GACE,KAAK,mBACH,OAAO,CAAC,KAAKC,yBAAyB,KAAK,GAAG,OAAO,GAAG,QAAQ,CAAC;GAEnE,KAAK,mBACH,OAAO,CACL,KAAKC,0BAA0B,KAAK,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,CACtE;GAEF,KAAK,gBAAgB;IACnB,MAAM,EAAC,QAAO;IACd,MAAM,MAAM;KAAC,QAAQ,GAAG,IAAI;KAAQ,OAAO,GAAG,IAAI;IAAI;IACtD,OAAO,CACL,GAAG,UAAU,KAAKhB,KAAK,eAAe,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC/B,IAAI,OAAO,iBAAiB,IAAI,QACzD,GAAG,UAAU,KAAKA,KAAK,aAAa,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC7B,IAAI,OAAO,iBAAiB,IAAI,MAC3D;GACF;GAEA,KAAK,cAAc;IACjB,MAAM,EAAC,QAAQ,SAAQ,GAAG;IAC1B,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,eAAe,EAAE;mCACpB,OAAO,iBAAiB,QACjD,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MACnD;GACF;GAEA,KAAK,iBAAiB;IACpB,MAAM,EAAC,QAAQ,MAAM,UAAS,GAAG;IACjC,OAAO,CACL,GAAG,UAAU,KAAKA,KAAK,aAAa,EAAE,kBAAkB,GAAG,IAAI;mCACtC,OAAO,iBAAiB,MAAM,kBAAkB,GAAG,KAC9E;GACF;GAEA,KAAK,eAAe;IAClB,MAAM,EAAC,QAAQ,SAAQ,GAAG;IAC1B,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,KAAK,kBAAkB,GAAG,QAC7E;GACF;GAEA,KAAK,qBAAqB;IACxB,MAAM,EAAC,QAAQ,MAAM,UAAS,GAAG;IACjC,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MAAM,mBAAmB,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAC/F;GACF;GAEA,SACE,YAAY,EAAE;EAClB;CACF;CAEA,yBACE,KACA,EAAC,QAAQ,MAAM,SACf,UACA;EACA,MAAM,MAAwB;GAAC;GAAQ;GAAO;EAAQ;EACtD,OAAO,GAAG;sBACQ,KAAKA,KAAK,eAAe,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEnC,IAAI,GAAG,EAAE;;CAEjC;CAEA,0BACE,KACA,EAAC,QAAQ,MAAM,SACf,QACA,UACA;EACA,MAAM,MAAyB;GAAC;GAAQ;GAAO;GAAQ;EAAQ;EAC/D,OAAO,GAAG;sBACQ,KAAKA,KAAK,aAAa,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEjC,IAAI,GAAG,EAAE;;CAEjC;;;;;CAMA,MAAM,eAAe;EACnB,IAAI,KAAKE,UAAU;GACjB,MAAM,EAAC,SAAS,YAAW,SAAS;GACpC,KAAKP,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;GACtC,MAAM;EACR;CACF;;;;;;CAOA,MAAM,OAAO;EACX,IAAI,KAAKO,UAAU;GACjB,KAAKhB,IAAI,OAAO,YAAY,KAAKS,OAAO,KAAK,EAAE,mBAAmB;GAClE,KAAK,MAAM;GACX,KAAKA,OAAO,QAAQ,MAAM;EAC5B;EACA,IACG,MAAM,UAAU,KAAKW,UAAU,KAAKP,eAAe,MAAO,aAE3D,MAAM,IAAI,WACR,kCAAkC,KAAKA,gBAAgB,GACzD;CAEJ;AACF;AAEA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CAEA,YACE,IACA,IACA,gBACA,WACA;EACA,KAAKb,MAAM;EACX,KAAK+B,MAAM;EACX,KAAK,iBAAiB;EACtB,KAAK,eAAe;CACtB;CAEA,YAAY;CAEZ,MAAM,UAAU;EACd,IAAI,KAAKC,WACP;EAEF,KAAKA,YAAY;EACjB,KAAKD,IAAI,QAAQ;EACjB,MAAM,KAAKA,IACR,KAAK,EACL,OAAM,MAAK,KAAK/B,IAAI,OAAO,iCAAiC,CAAC,CAAC;EACjE,KAAKA,IAAI,OAAO,0BAA0B,KAAK,cAAc;CAC/D;AACF;AAEA,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CAEA,YAAY,IAAgB,OAAgB,IAAgB;EAC1D,KAAKA,MAAM,GAAG,YAAY,aAAa,cAAc;EACrD,KAAKC,SAAS;EACd,KAAKI,MAAM;CACb;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKA,IAAI,GAAG,UAAU,KAAKJ,MAAM,EAAE,GAAG,OAAO;CACtD;CAEA,MAAM,UAAU;EACd,MAAM,KAAK,IAAI,gBAAgB,KAAKD,KAAK,EAAC,MAAM,eAAmB,CAAC,EAAE,IACpE,KAAKK,GACP;EACA,MAAM,MAAM,MAAM,GAAG,iBACnB,QAAO,GAA0B;8BACT,KAAKS,KAAK,WAAW,EAAE;;;KAIjD;EACA,IAAI,IAAI,WAAW,GAAG;GACpB,KAAKd,IAAI,OAAO,4CAA4C;GAC5D,GAAG,QAAQ;GACX,MAAM,GAAG,KAAK;GACd,OAAO;EACT;EACA,MAAM,CAAC,EAAC,eAAc;EACtB,MAAM,CAAC,EAAC,oBAAmB,MAAM,GAAG,iBAClC,QAAO,GAA+B;uCACL,KAAKc,KAAK,mBAAmB,EAAE;OAElE;EACA,KAAKd,IAAI,OACP,oBAAoB,UAAU,kCAAkC,gBAClE;EACA,OAAO,IAAI,UAAU,KAAKA,KAAK,IAAI,gBAAgB,SAAS;CAC9D;AACF"}
+{"version":3,"file":"storer.js","names":["#lc","#shard","#taskID","#discoveryAddress","#discoveryProtocol","#makeConnectionPool","#db","#inserter","#purger","#replicaVersion","#onCommitted","#onFatal","#queue","#backPressureThresholdBytes","#statementTimeoutMs","#changeLogBatchSize","#drainTimeoutMs","#progressMonitor","#withTimeout","#cdc","#approximateQueuedBytes","#running","#readyForMore","#stopped","#processQueue","#cancelQueueEntries","#startCatchup","#flushChangeLog","#trackBackfillMetadata","#maybeReleaseBackPressure","#catchup","#cookieStmts","#upsertTableMetadataStmt","#upsertColumnBackfillStmt","#tx","#released","#failureThresholdMs","#onFailure","#taskUpdates","#timer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {getHeapStatistics} from 'node:v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {\n  BigIntJSON,\n  type JSONObject,\n} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {sharedSnapshot, TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {orTimeout} from '../../types/timeout.ts';\nimport {\n  backfillRequestSchema,\n  isDataChange,\n  isSchemaChange,\n  type BackfillID,\n  type BackfillRequest,\n  type Change,\n  type DataChange,\n  type Identifier,\n  type SchemaChange,\n  type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {\n  type ChangeStreamData,\n  type Commit,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {UpstreamStatusMessage} from '../change-source/protocol/current/status.ts';\nimport {\n  cookieOps,\n  type BackfillCookie,\n  type CookieOp,\n  type CookieSet,\n  type TableMetadataCookie,\n} from '../replicator/change-log-cookies.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport {\n  extractChangeSubstring,\n  reconstructWatermarkedChange,\n  serializeChangeStreamData,\n  type ChangeLogEntry,\n} from './change-log-codec.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n  AutoResetSignal,\n  markResetRequired,\n  type BackfillingColumn,\n  type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\n/**\n * Factory for creating dynamic connection pools used by the Storer to\n * isolate concurrent transactions into separate pg clients.\n */\nexport type PostgresDBProvider = (\n  applicationName: string,\n  maxConns: number,\n) => PostgresDB;\n\ntype SubscriberAndMode = {\n  subscriber: Subscriber;\n  mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n  | [\n      'change',\n      watermark: string,\n      json: string,\n      orig: Exclude<Change, DataChange> | null, // null for DataChanges\n    ]\n  | ['ready', callback: () => void]\n  | ['subscriber', SubscriberAndMode]\n  | ['abort']\n  | 'stop';\n\ntype PendingTransaction = {\n  pool: TransactionPool;\n  preCommitWatermark: string;\n  pos: number;\n  startingReplicationState: Promise<ReplicationOwner>;\n  ack: boolean;\n  // changeLog rows buffered for the next multi-row INSERT flush.\n  batch: ChangeLogRow[];\n  // The most recently issued flush (or metadata) process, awaited to bound\n  // pipeline depth and to order the commit-time replicationState update.\n  lastFlush: Promise<unknown> | undefined;\n};\n\ntype ReplicationOwner = {\n  owner: string | null;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\nexport type TuningOptions = {\n  backPressureLimitHeapProportion: number;\n  statementTimeoutMs: number;\n  changeLogBatchSize: number;\n  drainTimeoutMs?: number | undefined;\n};\n\n/**\n * A single `changeLog` row, accumulated in {@link PendingTransaction.batch}\n * and written via `json_to_recordset()` (see `#flushChangeLog()`).\n */\ntype ChangeLogRow = {\n  watermark: string;\n  precommit: string | null;\n  pos: number;\n  change: string;\n};\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n *   `replicaVersion`, which is the version at which initial sync\n *   was performed, and any attempts to catchup from a different\n *   point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n *   a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n *   subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n  readonly id = 'storer';\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #taskID: string;\n  readonly #discoveryAddress: string;\n  readonly #discoveryProtocol: string;\n  readonly #makeConnectionPool: PostgresDBProvider;\n\n  // Isolated connection pools for steady-state, potentially concurrent\n  // queries, in order to avoid a postgres.js connection-swapping bug:\n  // https://github.com/porsager/postgres/issues/1204\n  readonly #db: PostgresDB;\n  readonly #inserter: PostgresDB;\n  readonly #purger: PostgresDB;\n\n  readonly #replicaVersion: string;\n  readonly #onCommitted: (c: Commit) => void;\n  readonly #onFatal: (err: Error) => void;\n  readonly #queue = new Queue<QueueEntry>();\n  readonly #backPressureThresholdBytes: number;\n  readonly #statementTimeoutMs: number;\n  readonly #changeLogBatchSize: number;\n  readonly #drainTimeoutMs: number;\n  readonly #progressMonitor: ProgressMonitor;\n\n  #approximateQueuedBytes = 0;\n  #running = false;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    dbProvider: PostgresDBProvider,\n    replicaVersion: string,\n    onCommitted: (c: Commit | UpstreamStatusMessage) => void,\n    onFatal: (err: Error) => void,\n    {\n      backPressureLimitHeapProportion,\n      statementTimeoutMs,\n      changeLogBatchSize,\n      drainTimeoutMs = 30_000,\n    }: TuningOptions,\n  ) {\n    this.#lc = lc.withContext('component', 'change-log');\n    this.#shard = shard;\n    this.#taskID = taskID;\n    this.#discoveryAddress = discoveryAddress;\n    this.#discoveryProtocol = discoveryProtocol;\n    this.#makeConnectionPool = dbProvider;\n    this.#db = dbProvider('change-stream-init', 3);\n    this.#inserter = dbProvider('change-log-inserter', 1);\n    this.#purger = dbProvider('change-log-purger', 1);\n    this.#replicaVersion = replicaVersion;\n    this.#onCommitted = onCommitted;\n    this.#onFatal = onFatal;\n    this.#statementTimeoutMs = statementTimeoutMs;\n    this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);\n    this.#drainTimeoutMs = drainTimeoutMs;\n    this.#progressMonitor = new ProgressMonitor(\n      lc,\n      statementTimeoutMs,\n      onFatal,\n    );\n\n    const heapStats = getHeapStatistics();\n    this.#backPressureThresholdBytes =\n      (heapStats.heap_size_limit - heapStats.used_heap_size) *\n      backPressureLimitHeapProportion;\n\n    this.#lc.info?.(\n      `Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of ` +\n        `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) ` +\n        `to absorb upstream spikes`,\n      {heapStats},\n    );\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  /**\n   * Bounds a one-off db call (i.e. not part of the main storer loop or the\n   * background catchup read, which are tracked by the ProgressMonitor\n   * instead) with a plain timeout. This covers calls like\n   * {@link assumeOwnership} and {@link getStartStreamInitializationParameters}\n   * that are made by the caller *before* {@link run()} -- and thus before the\n   * ProgressMonitor's polling starts -- as well as ones made well after,\n   * where a continuously-polling watchdog would be overkill for a single\n   * bounded db round trip.\n   *\n   * Does not cancel `promise` on timeout (there is no cancellation mechanism\n   * for a postgres.js query); it just stops waiting for it, and swallows a\n   * later rejection so it doesn't surface as an unhandled rejection.\n   */\n  async #withTimeout<T>(name: string, promise: Promise<T>): Promise<T> {\n    const result = await orTimeout(promise, this.#statementTimeoutMs);\n    if (result === 'timed-out') {\n      void promise.catch(() => {});\n      throw new AbortError(\n        `${name} did not complete within ${this.#statementTimeoutMs}ms`,\n      );\n    }\n    return result;\n  }\n\n  async assumeOwnership(purgeLock?: PurgeLock | null) {\n    const db = this.#db;\n    const owner = this.#taskID;\n    const ownerAddress = this.#discoveryAddress;\n    const ownerProtocol = this.#discoveryProtocol;\n    // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n    const addressWithProtocol =\n      ownerProtocol === 'ws'\n        ? ownerAddress\n        : `${ownerProtocol}://${ownerAddress}`;\n    this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n    const start = performance.now();\n    await this.#withTimeout(\n      'assume-ownership',\n      db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`,\n    );\n    const elapsed = (performance.now() - start).toFixed(2);\n    this.#lc.info?.(\n      `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n    );\n\n    if (purgeLock) {\n      // Once ownership has been assumed, any initial purge-lock preventing the\n      // purging of change-log records can be released, as a change-streamer\n      // that was attempting to purge records will correspondingly abort on the\n      // ownership check.\n      void purgeLock.release();\n    }\n  }\n\n  /**\n   * The whole of what a stream connection starts from: where to resume, what to\n   * ask the change source to backfill, and the raw cookie set those requests\n   * were derived from.\n   *\n   * All four statements run in one REPEATABLE READ transaction, so they see one\n   * snapshot. That is invariant 15 — the resume watermark and the cookies must\n   * come from the same store, read at the same position — and it is why the\n   * cookies are read here rather than by a second call: a cookie set paired\n   * with a watermark it was not folded to loses every backfill that completed\n   * in the interval, and those rows are not replayable from the slot.\n   *\n   * `cookies` is not `backfillRequests` in another shape. The request list is\n   * driven off `backfilling`, so it drops the metadata of every table with no\n   * in-flight backfill — correct for starting a stream, lossy as the cookie\n   * snapshot that the SQLite change log continues folding onto.\n   */\n  async getStartStreamInitializationParameters(): Promise<{\n    lastWatermark: string;\n    backfillRequests: BackfillRequest[];\n    cookies: CookieSet;\n  }> {\n    const [[{lastWatermark}], result, tableMetadata, backfilling] =\n      await this.#withTimeout(\n        'get-stream-params',\n        runTx(\n          this.#db,\n          sql => [\n            sql<{lastWatermark: string}[]>`\n        SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n            // Formats a BackfillRequest using json_object_agg() to construct the\n            // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n            // to make it optional and possibly `null`.\n            sql`\n        SELECT\n            json_build_object(\n              'schema', b.\"schema\",\n              'name', b.\"table\",\n              'metadata', t.\"metadata\"\n            ) as \"table\",\n            json_object_agg(b.\"column\", b.\"backfill\")\n              as \"columns\"\n          FROM ${this.#cdc('backfilling')} as b\n          LEFT JOIN ${this.#cdc('tableMetadata')} as t\n          ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n          GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n        `,\n\n            // Ordered by primary key, so that this set and the SQLite change log's\n            // can be compared row for row. `COLLATE \"C\"` because that comparison is\n            // against a store whose TEXT columns sort by byte: a linguistic\n            // collation orders `foo-bar` and `foobar` differently than SQLite's\n            // BINARY does, which would read as a divergence rather than as the\n            // identical set it is.\n            sql<TableMetadataCookie[]>`\n        SELECT \"schema\", \"table\", \"metadata\" FROM ${this.#cdc('tableMetadata')}\n          ORDER BY \"schema\" COLLATE \"C\", \"table\" COLLATE \"C\"`,\n\n            sql<BackfillCookie[]>`\n        SELECT \"schema\", \"table\", \"column\", \"backfill\" FROM ${this.#cdc('backfilling')}\n          ORDER BY \"schema\" COLLATE \"C\", \"table\" COLLATE \"C\", \"column\" COLLATE \"C\"`,\n          ],\n          {mode: Mode.READONLY},\n        ),\n      );\n\n    return {\n      lastWatermark,\n      backfillRequests: v.parse(result, backfillRequestsSchema),\n      cookies: {\n        tableMetadata: [...tableMetadata],\n        backfilling: [...backfilling],\n      },\n    };\n  }\n\n  async getMinWatermarkForCatchup(): Promise<string | null> {\n    const [{minWatermark}] = await this.#withTimeout(\n      'get-min-watermark',\n      this.#db<{minWatermark: string | null}[]> /*sql*/ `\n      SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`,\n    );\n    return minWatermark;\n  }\n\n  /**\n   * Returns the retained Postgres catchup bounds.\n   * `minWatermark` is the oldest transaction, or `null` for a new replica.\n   * `lastWatermark` is the durable head.\n   */\n  async getCatchupBounds(): Promise<{\n    minWatermark: string | null;\n    lastWatermark: string;\n  }> {\n    const [bounds] = await this.#withTimeout(\n      'get-catchup-bounds',\n      this.#db<{minWatermark: string | null; lastWatermark: string}[]> /*sql*/ `\n      SELECT\n        (SELECT min(watermark) FROM ${this.#cdc('changeLog')}) as \"minWatermark\",\n        (SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}) as \"lastWatermark\"`,\n    );\n    return bounds;\n  }\n\n  /**\n   * Lists at most `limit` committed transaction watermarks in\n   * `(afterWatermark, throughWatermark]`, in ascending order.\n   */\n  async listCommitWatermarks(\n    afterWatermark: string,\n    throughWatermark: string,\n    limit: number,\n  ): Promise<string[]> {\n    const rows = await this.#db<{watermark: string}[]> /*sql*/ `\n      SELECT watermark FROM ${this.#cdc('changeLog')}\n       WHERE precommit IS NOT NULL\n         AND watermark > ${afterWatermark}\n         AND watermark <= ${throughWatermark}\n       ORDER BY watermark\n       LIMIT ${limit}`;\n    return rows.map(({watermark}) => watermark);\n  }\n\n  /**\n   * Reads `(afterWatermark, throughWatermark]` in stream order and bounded batches.\n   * This query matches the subscriber catchup query.\n   *\n   * Both queries fail when an escaped NUL reaches `change->'tag'`.\n   * Postgres cannot convert that value to text.\n   */\n  readCatchupRange(\n    afterWatermark: string,\n    throughWatermark: string,\n    batchRows = 2000,\n  ): AsyncIterable<ChangeLogEntry[]> {\n    assert(\n      Number.isSafeInteger(batchRows) && batchRows > 0,\n      'Postgres change log batch size must be a positive safe integer',\n    );\n    return this.#db<ChangeLogEntry[]> /*sql*/ `\n      SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc('changeLog')}\n       WHERE watermark > ${afterWatermark}\n         AND watermark <= ${throughWatermark}\n       ORDER BY watermark, pos`.cursor(batchRows) as AsyncIterable<\n      ChangeLogEntry[]\n    >;\n  }\n\n  // Note: These are the only db `await`s that are not tracked by the\n  //       ProgressMonitor, as the delete can legitimately block and/or take\n  //       an arbitrary amount of time. Luckily, this is a background call and\n  //       thus cannot cause the storer loop to hang.\n  purgeRecordsBefore(watermark: string): Promise<number> {\n    return runTx(this.#purger, async sql => {\n      // This NOWAIT pre-check is an optimization to abort the transaction\n      // (and release associated resources) early.\n      await sql<{watermark: string}[]>`\n          SELECT watermark FROM ${this.#cdc('changeLog')}\n            ORDER BY watermark, pos LIMIT 1\n            FOR UPDATE NOWAIT\n        `;\n      // If the row is purge-locked by an incoming replication-manager, it\n      // will assume ownership of the change-log before releasing the lock.\n      // This DELETE blocks until the lock is released, allowing the change\n      // in ownership to be reliably detected (and the transaction aborted)\n      // in the subsequent check.\n      const [{deleted}] = await sql<{deleted: bigint}[]>`\n        -- The backup watermark can be ahead of the durable changeLog if the\n        -- storer is behind but the backup replica has consumed forwarded\n        -- changes. Preserve the latest durable changeLog transaction as the\n        -- catchup boundary instead of assuming the backup watermark exists.\n        -- The storer inserts each changeLog transaction atomically, so any\n        -- durable row for a watermark implies the full transaction is durable.\n        WITH keep AS (\n          SELECT max(watermark) AS watermark\n          FROM ${this.#cdc('changeLog')}\n        ), purged AS (\n          DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n            AND watermark < (SELECT watermark FROM keep)\n            RETURNING watermark, pos\n        ) SELECT COUNT(*) as deleted FROM purged;`;\n\n      const [{owner}] = await sql<ReplicationOwner[]>`\n        SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR SHARE`;\n      if (owner !== this.#taskID) {\n        throw new AbortError(\n          `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n        );\n      }\n      return Number(deleted);\n    });\n  }\n\n  /**\n   * @returns The JSON stringified stream message to be sent downstream.\n   */\n  store(watermark: string, data: ChangeStreamData) {\n    // Eagerly stringify the JSON payload to:\n    // - avoid redundant stringification when fanning out to subscribers\n    // - efficiently estimate the amount of memory the payload consumes\n    const json = serializeChangeStreamData(data);\n    this.#approximateQueuedBytes += json.length;\n\n    const change = data[1];\n    this.#queue.enqueue([\n      'change',\n      watermark,\n      json,\n      isDataChange(change) ? null : change, // drop DataChanges to save memory\n    ]);\n\n    return json;\n  }\n\n  abort() {\n    this.#queue.enqueue(['abort']);\n  }\n\n  catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n    this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n  }\n\n  #readyForMore: Resolver<void> | null = null;\n\n  readyForMore(): Promise<void> | undefined {\n    if (!this.#running) {\n      return undefined;\n    }\n    if (\n      this.#readyForMore === null &&\n      this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n    ) {\n      this.#lc.warn?.(\n        `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\\n` +\n          `\\n` +\n          `To inspect changeLog backlog in your change DB:\\n` +\n          `  SELECT\\n` +\n          `    (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\\n` +\n          `    change->>'tag' AS operation,\\n` +\n          `    COUNT(*) AS count\\n` +\n          `  FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n          `  GROUP BY 1, 2\\n` +\n          `  ORDER BY 3 DESC\\n` +\n          `  LIMIT 20;`,\n      );\n      this.#readyForMore = resolver();\n    }\n    return this.#readyForMore?.promise;\n  }\n\n  #maybeReleaseBackPressure() {\n    if (this.#readyForMore !== null) {\n      // Wait for at least 20% of the threshold to free up.\n      if (\n        this.#approximateQueuedBytes <\n        this.#backPressureThresholdBytes * 0.8\n      ) {\n        this.#lc.info?.(\n          `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`,\n        );\n        this.#readyForMore.resolve();\n        this.#readyForMore = null;\n      }\n    }\n  }\n\n  /**\n   * Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.\n   *\n   * Uses `json_to_recordset()` so the batch is a single JSON parameter: the\n   * statement text stays constant regardless of batch size, avoiding the\n   * unbounded prepared-statement variants (and Postgres memory growth) that a\n   * multi-row INSERT would produce. See rocicorp/mono#3511.\n   *\n   * Returns (and updates) {@link PendingTransaction.lastFlush}; a no-op when the\n   * batch is empty.\n   */\n  #flushChangeLog(tx: PendingTransaction): Promise<unknown> | undefined {\n    const {batch} = tx;\n    if (batch.length === 0) {\n      return tx.lastFlush;\n    }\n    tx.batch = [];\n    tx.lastFlush = tx.pool.process(sql => [\n      sql`\n        INSERT INTO ${this.#cdc('changeLog')} (\"watermark\", \"pos\", \"change\", \"precommit\")\n        SELECT \"watermark\", \"pos\", \"change\"::json, \"precommit\"\n          FROM json_to_recordset(${batch}) AS x(\n            \"watermark\" TEXT,\n            \"pos\" INT8,\n            \"change\" TEXT,\n            \"precommit\" TEXT\n          )`,\n    ]);\n    return tx.lastFlush;\n  }\n\n  #stopped = promiseVoid;\n\n  /**\n   * Runs the storer loop until {@link stop()} is called, or an error is thrown.\n   * Once {@link run()} completes, it can be called again.\n   */\n  async run() {\n    assert(!this.#running, `storer is already running`);\n\n    const {promise: stopped, resolve: signalStopped} = resolver();\n    this.#running = true;\n    this.#stopped = stopped;\n\n    this.#lc.info?.('starting storer');\n    this.#progressMonitor.start(); // Note: This is stopped in stop()\n    let err: unknown;\n    try {\n      await this.#processQueue();\n    } catch (e) {\n      err = e; // used in finally\n      throw e;\n    } finally {\n      // Release any pending backpressure so the upstream can proceed\n      if (this.#readyForMore !== null) {\n        this.#readyForMore.resolve();\n        this.#readyForMore = null;\n      }\n      this.#cancelQueueEntries(\n        this.#queue.drain().filter(entry => entry !== undefined),\n        err,\n      );\n      this.#running = false;\n      signalStopped();\n      this.#lc.info?.('storer stopped');\n    }\n  }\n\n  #cancelQueueEntries(queue: QueueEntry[], e: unknown) {\n    if (queue.length === 0) {\n      return;\n    }\n    this.#lc.info?.(\n      `canceling ${queue.length} entries from the changeLog queue`,\n    );\n    const err = e instanceof Error ? e : new AbortError('server shutting down');\n    for (const entry of queue) {\n      if (entry === 'stop') {\n        continue;\n      }\n      const type = entry[0];\n      switch (type) {\n        case 'subscriber': {\n          // Disconnect subscribers waiting to be caught up so that they can\n          // reconnect and try again.\n          const {subscriber} = entry[1];\n          this.#lc.info?.(`disconnecting ${subscriber.id}`);\n          subscriber.fail(err);\n          break;\n        }\n      }\n    }\n  }\n\n  async #processQueue() {\n    let tx: PendingTransaction | null = null;\n    let msg: QueueEntry | false;\n\n    // Track the progress of each (previous and next) queue entry before it is\n    // processed in the loop.\n    let lastTaskDone: TaskDoneFn | undefined;\n    const nextMessage = async () => {\n      lastTaskDone?.();\n      lastTaskDone = undefined;\n      const entry = await this.#queue.dequeue();\n      if (entry !== 'stop') {\n        lastTaskDone = this.#progressMonitor.trackTask({\n          task: 'queue-entry',\n          // Note: TaskKeys are logged, so only include entry[0] (the type) to\n          // avoid logging application data.\n          type: entry[0],\n        });\n      }\n      return entry;\n    };\n\n    const catchupQueue: SubscriberAndMode[] = [];\n    try {\n      while ((msg = await nextMessage()) !== 'stop') {\n        const [msgType] = msg;\n        switch (msgType) {\n          case 'ready': {\n            const signalReady = msg[1];\n            signalReady();\n            continue;\n          }\n          case 'subscriber': {\n            const subscriber = msg[1];\n            if (tx) {\n              catchupQueue.push(subscriber); // Wait for the current tx to complete.\n            } else {\n              await this.#startCatchup([subscriber]); // Catch up immediately.\n            }\n            continue;\n          }\n          case 'abort': {\n            if (tx) {\n              tx.pool.abort();\n              await tx.pool.done();\n              tx = null;\n            }\n            continue;\n          }\n        }\n        // msgType === 'change'\n        const [_, watermark, json, change] = msg;\n        const tag = change?.tag;\n        this.#approximateQueuedBytes -= json.length;\n\n        if (tag === 'begin') {\n          assert(!tx, 'received BEGIN in the middle of a transaction');\n          const {promise, resolve, reject} = resolver<ReplicationOwner>();\n          void promise.catch(() => {}); // handle rejections before the await\n          tx = {\n            pool: new TransactionPool(\n              this.#lc.withContext('watermark', watermark),\n              {mode: Mode.READ_COMMITTED},\n            ),\n            preCommitWatermark: watermark,\n            pos: 0,\n            startingReplicationState: promise,\n            ack: !change.skipAck,\n            batch: [],\n            lastFlush: undefined,\n          };\n          tx.pool.run(this.#inserter);\n          // Acquire a lock on the replicationState row to detect and/or prevent\n          // a concurrent ownership change.\n          void tx.pool.process(tx => {\n            tx<ReplicationOwner[]> /*sql*/ `\n          SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n              ([result]) => resolve(result),\n              reject,\n            );\n            return [];\n          });\n        } else {\n          assert(tx, () => `received change outside of transaction: ${json}`);\n          tx.pos++;\n        }\n\n        const entry: ChangeLogRow = {\n          watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n          precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n          pos: tx.pos,\n          // For backwards compatibility, only the change message is stored\n          // in the cdc changeLog.\n          change: extractChangeSubstring(json, tag),\n        };\n\n        if (change !== null && isSchemaChange(change)) {\n          // Schema changes carry backfill / table-metadata statements that\n          // must be applied in stream order relative to the changeLog rows.\n          // Flush any buffered rows first, then write this row together with\n          // its metadata statements as a single unit (preserving the previous\n          // per-change ordering for schema changes).\n          await this.#flushChangeLog(tx);\n          tx.lastFlush = tx.pool.process(sql => [\n            sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n            ...this.#trackBackfillMetadata(sql, change),\n          ]);\n        } else {\n          // Accumulate plain changeLog rows (begin, data changes, commit) and\n          // write them as a single multi-row INSERT. Collapsing the per-change\n          // single-row INSERTs into batches is the dominant cost reduction for\n          // large transactions, where the previous one-statement-per-change\n          // path dominated the upstream replication lag.\n          tx.batch.push(entry);\n          if (tx.batch.length >= this.#changeLogBatchSize) {\n            // Bound pipeline depth (and thus memory) by awaiting the previous\n            // flush before issuing the next. This is the batched analog of the\n            // previous per-100-statement backpressure await, and likewise\n            // guards against memory blowup on very large transactions.\n            const prevFlush = tx.lastFlush;\n            void this.#flushChangeLog(tx);\n            await prevFlush;\n          }\n        }\n        this.#maybeReleaseBackPressure();\n\n        if (tag === 'commit') {\n          // Flush any remaining buffered changeLog rows (including this commit\n          // row) before updating the replication state, so the state update is\n          // ordered after all changeLog inserts for this transaction.\n          void this.#flushChangeLog(tx);\n\n          const {owner} = await tx.startingReplicationState;\n          if (owner !== this.#taskID) {\n            // Ownership change reflected in the replicationState read in 'begin'.\n            tx.pool.fail(\n              new AbortError(\n                `changeLog ownership has been assumed by ${owner}`,\n              ),\n            );\n          } else {\n            // Update the replication state.\n            const lastWatermark = watermark;\n            void tx.pool.process(tx => [\n              tx`\n            UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n            ]);\n            tx.pool.setDone();\n          }\n\n          await tx.pool.done();\n\n          // ACK the LSN to the upstream Postgres.\n          if (tx.ack) {\n            this.#onCommitted(['commit', change, {watermark}]);\n          }\n          tx = null;\n\n          // Before beginning the next transaction, open a READONLY snapshot to\n          // concurrently catchup any queued subscribers.\n          await this.#startCatchup(catchupQueue.splice(0));\n        } else if (tag === 'rollback') {\n          // Aborted transactions are not stored in the changeLog. Abort the current tx\n          // and process catchup of subscribers that were waiting for it to end.\n          tx.pool.abort();\n          await tx.pool.done();\n          tx = null;\n\n          await this.#startCatchup(catchupQueue.splice(0));\n        }\n      }\n    } catch (e) {\n      catchupQueue.forEach(({subscriber}) => subscriber.fail(e));\n      throw e;\n    }\n  }\n\n  async #startCatchup(subs: SubscriberAndMode[]) {\n    const numCatchups = subs.length;\n    if (numCatchups === 0) {\n      return;\n    }\n\n    const lc = this.#lc.withContext('pool', 'catchup');\n    const {init, cleanup, snapshotID} = sharedSnapshot();\n    const reader = new TransactionPool(lc, {\n      mode: Mode.READONLY,\n      init,\n      cleanup,\n      initialWorkers: subs.length,\n    });\n\n    // A dynamic connection pool is created for catchup, sized to the number\n    // of subscribers being caught up.\n    const catchupConns = this.#makeConnectionPool(\n      'subscriber-catchup',\n      numCatchups,\n    );\n    reader.run(catchupConns);\n\n    let lastWatermark: string | undefined;\n    const catchupSnapshotted = this.#progressMonitor.trackTask({\n      task: 'capture-catchup-snapshot',\n      subscribers: subs.map(({subscriber: s}) => s.id),\n    });\n    try {\n      // Ensure that the transaction has started (and is thus holding a snapshot\n      // of the database) before continuing on to commit more changes. This is\n      // done by performing a single read on the db, which determines the\n      // snapshot for the REPEATABLE_READ transaction.\n      [{lastWatermark}] = await reader.processReadTask(\n        sql => sql<{lastWatermark: string}[]>`\n        SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}\n      `,\n      );\n      lc.info?.(\n        `snapshotted db at ${lastWatermark} (${await snapshotID}) ` +\n          `to catchup ${numCatchups} subscriber(s)`,\n      );\n    } catch (e) {\n      subs.map(({subscriber}) => subscriber.fail(e));\n      throw e;\n    } finally {\n      catchupSnapshotted();\n    }\n\n    // Run the actual catchup queries in the background. Errors are handled in\n    // #catchup() by disconnecting the associated subscriber.\n    void Promise.allSettled(\n      subs.map(sub =>\n        this.#catchup(\n          lc.withContext('subscriber', sub.subscriber.id),\n          sub,\n          lastWatermark,\n          reader,\n        ),\n      ),\n    ).finally(() => {\n      reader.setDone();\n      void catchupConns.end().catch(() => {});\n    });\n  }\n\n  async #catchup(\n    lc: LogContext,\n    {subscriber: sub, mode}: SubscriberAndMode,\n    lastWatermark: string,\n    reader: TransactionPool,\n  ) {\n    let entriesReceived = 0;\n    let catchupProgressed = this.#progressMonitor.trackTask({\n      task: 'catchup',\n      subscriber: sub.id,\n      entriesReceived,\n    });\n    try {\n      lc.info?.(`starting catchup`);\n      await reader.processReadTask(async tx => {\n        lc.info?.(`catching up`);\n        const start = Date.now();\n\n        // When starting from initial-sync, there won't be a change with a watermark\n        // equal to the replica version. This is the empty changeLog scenario.\n        let watermarkFound = sub.watermark === this.#replicaVersion;\n        let count = 0;\n        let lastBatchConsumed: Promise<unknown> | undefined;\n\n        for await (const entries of tx<ChangeLogEntry[]> /*sql*/ `\n          SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc('changeLog')}\n           WHERE watermark >= ${sub.watermark}\n             AND watermark <= ${lastWatermark}\n           ORDER BY watermark, pos`.cursor(2000)) {\n          catchupProgressed();\n          entriesReceived += entries.length;\n\n          try {\n            // Wait for the last batch of entries to be consumed by the\n            // subscriber before sending down the current batch. This pipelining\n            // allows one batch of changes to be received from the change-db\n            // while the previous batch of changes are sent to the subscriber,\n            // resulting in flow control that caps the number of changes\n            // referenced in memory to 2 * batch-size.\n            const start = performance.now();\n            await lastBatchConsumed;\n            const elapsed = performance.now() - start;\n            if (lastBatchConsumed) {\n              lc[elapsed > 100 ? 'info' : 'debug']?.(\n                `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n              );\n            }\n\n            for (const entry of entries) {\n              if (entry.watermark === sub.watermark) {\n                // This should be the first entry.\n                // Catchup starts from *after* the watermark.\n                watermarkFound = true;\n              } else if (watermarkFound) {\n                lastBatchConsumed = sub.catchup(\n                  reconstructWatermarkedChange(entry),\n                );\n                count++;\n              } else if (mode === 'backup') {\n                throw new AutoResetSignal(\n                  `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n                );\n              } else {\n                lc.warn?.(\n                  `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n                );\n                sub.close(\n                  ErrorType.WatermarkTooOld,\n                  `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n                );\n                return;\n              }\n            }\n          } finally {\n            // Track the db's read of the next chunk of changes.\n            catchupProgressed = this.#progressMonitor.trackTask({\n              task: 'catchup',\n              subscriber: sub.id,\n              entriesReceived,\n            });\n          }\n        }\n        catchupProgressed();\n\n        if (watermarkFound) {\n          await lastBatchConsumed;\n          lc.info?.(\n            `caught up ${sub.id} with ${count} changes (${\n              Date.now() - start\n            } ms)`,\n          );\n        } else {\n          // The subscriber is ahead of the latest durable changeLog entry\n          // (lastWatermark). This can legitimately happen: changes are\n          // forwarded to subscribers (the backup replica and view-syncers)\n          // concurrently with — and can outrun — the durable store, so a\n          // replica may briefly lead the change DB after the storer falls\n          // behind or the change-streamer restarts. No catchup is possible or\n          // needed; once the change DB catches back up, forwarding resumes and\n          // the subscriber dedups any watermarks it already has. Unlike the\n          // AutoResetSignal / WatermarkTooOld cases above, this is not a gap in\n          // replication history, so the subscriber is simply marked caught up.\n          lc.warn?.(\n            `subscriber ${sub.id} at watermark ${sub.watermark} is ahead of ` +\n              `the latest durable watermark ${lastWatermark}; waiting for the ` +\n              `change DB to catch up`,\n          );\n        }\n        // Start draining messages buffered during catchup. The returned promise\n        // is intentionally not awaited here: while the drain is in progress,\n        // new sends keep appending to the subscriber backlog and inherit its\n        // byte-based backpressure.\n        void sub.setCaughtUp();\n      });\n    } catch (err) {\n      lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n      if (err instanceof AutoResetSignal) {\n        await markResetRequired(this.#db, this.#shard);\n        this.#onFatal(err);\n      }\n      sub.fail(err);\n    } finally {\n      catchupProgressed();\n    }\n  }\n\n  /**\n   * Returns the db statements necessary to track backfill and table metadata\n   * presented in the `change`, if any.\n   *\n   * This is the Postgres interpreter of {@link cookieOps}, which is also\n   * interpreted against the SQLite change log by `ChangeLogCookieWriter`. The\n   * two stores have to agree on every transition forever, and the failure mode\n   * if they drift is a backfill that is silently never re-requested — so the\n   * decision of what a schema change *means* lives in one place and only its\n   * transport lives here.\n   */\n  #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n    return cookieOps(change).flatMap(op => this.#cookieStmts(sql, op));\n  }\n\n  #cookieStmts(sql: PostgresTransaction, op: CookieOp): PendingQuery<Row[]>[] {\n    switch (op.op) {\n      case 'upsert-metadata':\n        return [this.#upsertTableMetadataStmt(sql, op.table, op.metadata)];\n\n      case 'upsert-backfill':\n        return [\n          this.#upsertColumnBackfillStmt(sql, op.table, op.column, op.backfill),\n        ];\n\n      case 'rename-table': {\n        const {old} = op;\n        const row = {schema: op.new.schema, table: op.new.name};\n        return [\n          sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n          sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n        ];\n      }\n\n      case 'drop-table': {\n        const {schema, name} = op.table;\n        return [\n          sql`DELETE FROM ${this.#cdc('tableMetadata')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n        ];\n      }\n\n      case 'rename-column': {\n        const {schema, name: table} = op.table;\n        return [\n          sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${op.new}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${op.old}`,\n        ];\n      }\n\n      case 'drop-column': {\n        const {schema, name} = op.table;\n        return [\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${op.column}`,\n        ];\n      }\n\n      case 'complete-backfill': {\n        const {schema, name: table} = op.table;\n        return [\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql([...op.columns])}`,\n        ];\n      }\n\n      default:\n        unreachable(op);\n    }\n  }\n\n  #upsertTableMetadataStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    metadata: TableMetadata,\n  ) {\n    const row: TableMetadataRow = {schema, table, metadata};\n    return sql`\n        INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\")\n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  #upsertColumnBackfillStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    column: string,\n    backfill: BackfillID,\n  ) {\n    const row: BackfillingColumn = {schema, table, column, backfill};\n    return sql`\n        INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\", \"column\")\n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  /**\n   * Waits until all currently queued entries have been processed.\n   * This is only used in tests.\n   */\n  async allProcessed() {\n    if (this.#running) {\n      const {promise, resolve} = resolver();\n      this.#queue.enqueue(['ready', resolve]);\n      await promise;\n    }\n  }\n\n  /**\n   * Stops the storer and waits up to a drain timeout for entries to drain,\n   * throwing an exception if it doesn't drain in time, in order to abort\n   * the server when PG (or the connection to it) appears to be wedged.\n   */\n  async stop() {\n    if (this.#running) {\n      this.#progressMonitor.stop();\n      this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n      this.abort(); // for cleanliness, abort any open transactions\n      this.#queue.enqueue('stop');\n    }\n    if (\n      (await orTimeout(this.#stopped, this.#drainTimeoutMs)) === 'timed-out'\n    ) {\n      throw new AbortError(\n        `changeLog did not drain within ${this.#drainTimeoutMs}ms`,\n      );\n    }\n  }\n}\n\nexport class PurgeLock {\n  readonly #lc: LogContext;\n  readonly #tx: TransactionPool;\n  readonly replicaVersion: string;\n  readonly minWatermark: string;\n\n  constructor(\n    lc: LogContext,\n    tx: TransactionPool,\n    replicaVersion: string,\n    watermark: string,\n  ) {\n    this.#lc = lc;\n    this.#tx = tx;\n    this.replicaVersion = replicaVersion;\n    this.minWatermark = watermark;\n  }\n\n  #released = false;\n\n  async release() {\n    if (this.#released) {\n      return;\n    }\n    this.#released = true;\n    this.#tx.setDone();\n    await this.#tx\n      .done()\n      .catch(e => this.#lc.warn?.(`error from purge-lock release`, e));\n    this.#lc.info?.(`released purge lock on ${this.minWatermark}`);\n  }\n}\n\nexport class PurgeLocker {\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #db: PostgresDB;\n\n  constructor(lc: LogContext, shard: ShardID, db: PostgresDB) {\n    this.#lc = lc.withContext('component', 'purge-locker');\n    this.#shard = shard;\n    this.#db = db;\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async acquire() {\n    const tx = new TransactionPool(this.#lc, {mode: Mode.READ_COMMITTED}).run(\n      this.#db,\n    );\n    const row = await tx.processReadTask(\n      sql => sql<{watermark: string}[]>`\n      SELECT watermark FROM ${this.#cdc('changeLog')}\n        ORDER BY watermark, pos LIMIT 1\n        FOR SHARE \n    `,\n    );\n    if (row.length === 0) {\n      this.#lc.info?.(`changeLog is empty. No rows to purge-lock.`);\n      tx.setDone();\n      await tx.done();\n      return null;\n    }\n    const [{watermark}] = row;\n    const [{replicaVersion}] = await tx.processReadTask(\n      sql => sql<{replicaVersion: string}[]>`\n        SELECT \"replicaVersion\" FROM ${this.#cdc('replicationConfig')}\n      `,\n    );\n    this.#lc.info?.(\n      `locked watermark ${watermark} from being purged from replica@${replicaVersion}`,\n    );\n    return new PurgeLock(this.#lc, tx, replicaVersion, watermark);\n  }\n}\n\n// A TaskKey, keyed by identity, representing an db task monitored by the\n// ProgressMonitor. The contents are stringified in error messages if the\n// task fails to make progress for more than a failure threshold interval.\ntype TaskKey = JSONObject;\n\ntype TaskDoneFn = () => void;\n\n/**\n * Periodic monitor for detecting the pathological condition in which a db\n * query hangs forever because of a connection problem, such as a half-closed\n * socket.\n *\n * Use this for db calls that recur or accumulate progress within a single\n * logical operation on the main storer loop (e.g. the queue-processing loop,\n * or a multi-batch catchup read), where \"no progress for N ms\" is the\n * meaningful signal. For a one-off db call -- including ones made before\n * {@link Storer.run} starts this monitor's polling -- use a plain timeout\n * (see `Storer.#withTimeout`) instead.\n *\n * The failure handler should stop the server. The ProgressMonitor will\n * assume this and stop itself after calling it.\n *\n * Internal to the Storer, but exported for testing.\n */\nexport class ProgressMonitor {\n  readonly #lc: LogContext;\n  readonly #failureThresholdMs;\n  readonly #onFailure: (err: Error) => void;\n  readonly #taskUpdates = new Map<TaskKey, Date>();\n\n  #timer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    failureThresholdMs: number,\n    onFailure: (err: Error) => void,\n  ) {\n    this.#lc = lc.withContext('component', 'storer-progress-monitor');\n    this.#failureThresholdMs = failureThresholdMs;\n    this.#onFailure = onFailure;\n  }\n\n  start() {\n    clearInterval(this.#timer);\n    this.#timer = setInterval(\n      () => this.checkTaskProgress(new Date()),\n      this.#failureThresholdMs,\n    ).unref();\n  }\n\n  // Called by an internal periodic timer, but exported for testing.\n  checkTaskProgress(now: Date) {\n    for (const [task, lastUpdate] of this.#taskUpdates.entries()) {\n      if (now.getTime() - lastUpdate.getTime() >= this.#failureThresholdMs) {\n        this.#lc.error?.(\n          `Last task update was more than ${this.#failureThresholdMs}ms ago`,\n          {task, lastUpdate},\n        );\n        this.#onFailure(\n          new Error(\n            `Task failed to progress for over ${this.#failureThresholdMs}ms: ${BigIntJSON.stringify(task)}`,\n          ),\n        );\n        this.stop();\n        return;\n      }\n    }\n  }\n\n  /**\n   * Starts a task to be tracked.\n   *\n   * @return a callback to invoke when the task is done.\n   */\n  trackTask(task: TaskKey, now = new Date()): TaskDoneFn {\n    this.#taskUpdates.set(task, now);\n    return () => {\n      this.#taskUpdates.delete(task);\n    };\n  }\n\n  stop() {\n    clearInterval(this.#timer);\n    this.#taskUpdates.clear();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAuGA,IAAM,yBAAyB,eAAE,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkD5D,IAAa,SAAb,MAAuC;CACrC,KAAc;CACd;CACA;CACA;CACA;CACA;CACA;CAKA;CACA;CACA;CAEA;CACA;CACA;CACA,SAAkB,IAAI,MAAkB;CACxC;CACA;CACA;CACA;CACA;CAEA,0BAA0B;CAC1B,WAAW;CAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,YACA,gBACA,aACA,SACA,EACE,iCACA,oBACA,oBACA,iBAAiB,OAEnB;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAKC,SAAS;EACd,KAAKC,UAAU;EACf,KAAKC,oBAAoB;EACzB,KAAKC,qBAAqB;EAC1B,KAAKC,sBAAsB;EAC3B,KAAKC,MAAM,WAAW,sBAAsB,CAAC;EAC7C,KAAKC,YAAY,WAAW,uBAAuB,CAAC;EACpD,KAAKC,UAAU,WAAW,qBAAqB,CAAC;EAChD,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;EACpB,KAAKC,WAAW;EAChB,KAAKG,sBAAsB;EAC3B,KAAKC,sBAAsB,KAAK,IAAI,GAAG,kBAAkB;EACzD,KAAKC,kBAAkB;EACvB,KAAKC,mBAAmB,IAAI,gBAC1B,IACA,oBACA,OACF;EAEA,MAAM,YAAY,kBAAkB;EACpC,KAAKJ,+BACF,UAAU,kBAAkB,UAAU,kBACvC;EAEF,KAAKb,IAAI,OACP,gBAAgB,KAAKa,8BAA8B,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAC5C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAE/E,EAAC,UAAS,CACZ;CACF;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKP,IAAI,GAAG,UAAU,KAAKL,MAAM,EAAE,GAAG,OAAO;CACtD;;;;;;;;;;;;;;;CAgBA,MAAMiB,aAAgB,MAAc,SAAiC;EACnE,MAAM,SAAS,MAAM,UAAU,SAAS,KAAKJ,mBAAmB;EAChE,IAAI,WAAW,aAAa;GAC1B,QAAa,YAAY,CAAC,CAAC;GAC3B,MAAM,IAAI,WACR,GAAG,KAAK,2BAA2B,KAAKA,oBAAoB,GAC9D;EACF;EACA,OAAO;CACT;CAEA,MAAM,gBAAgB,WAA8B;EAClD,MAAM,KAAK,KAAKR;EAChB,MAAM,QAAQ,KAAKJ;EACnB,MAAM,eAAe,KAAKC;EAC1B,MAAM,gBAAgB,KAAKC;EAE3B,MAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,cAAc,KAAK;EAC5B,KAAKJ,IAAI,OAAO,yBAAyB,qBAAqB;EAC9D,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAKkB,aACT,oBACA,EAAE,UAAU,KAAKC,KAAK,kBAAkB,EAAE,OAAO,GAAG;GAAC;GAAO,cAAc;EAAmB,CAAC,GAChG;EACA,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;EACrD,KAAKnB,IAAI,OACP,wBAAwB,oBAAoB,IAAI,QAAQ,KAC1D;EAEA,IAAI,WAKF,UAAe,QAAQ;CAE3B;;;;;;;;;;;;;;;;;;CAmBA,MAAM,yCAIH;EACD,MAAM,CAAC,CAAC,EAAC,kBAAiB,QAAQ,eAAe,eAC/C,MAAM,KAAKkB,aACT,qBACA,MACE,KAAKZ,MACL,QAAO;GACL,GAA8B;sCACJ,KAAKa,KAAK,kBAAkB;GAKtD,GAAG;;;;;;;;;iBASE,KAAKA,KAAK,aAAa,EAAE;sBACpB,KAAKA,KAAK,eAAe,EAAE;;;;GAWrC,GAA0B;oDACc,KAAKA,KAAK,eAAe,EAAE;;GAGnE,GAAqB;8DAC6B,KAAKA,KAAK,aAAa,EAAE;;EAE7E,GACA,EAAC,MAAM,SAAa,CACtB,CACF;EAEF,OAAO;GACL;GACA,kBAAkB,MAAQ,QAAQ,sBAAsB;GACxD,SAAS;IACP,eAAe,CAAC,GAAG,aAAa;IAChC,aAAa,CAAC,GAAG,WAAW;GAC9B;EACF;CACF;CAEA,MAAM,4BAAoD;EACxD,MAAM,CAAC,EAAC,kBAAiB,MAAM,KAAKD,aAClC,qBACA,KAAKZ,GAA6C;qDACH,KAAKa,KAAK,WAAW,GACtE;EACA,OAAO;CACT;;;;;;CAOA,MAAM,mBAGH;EACD,MAAM,CAAC,UAAU,MAAM,KAAKD,aAC1B,sBACA,KAAKZ,GAAoE;;sCAEzC,KAAKa,KAAK,WAAW,EAAE;uCACtB,KAAKA,KAAK,kBAAkB,EAAE,qBACjE;EACA,OAAO;CACT;;;;;CAMA,MAAM,qBACJ,gBACA,kBACA,OACmB;EAQnB,QAAO,MAPY,KAAKb,GAAmC;8BACjC,KAAKa,KAAK,WAAW,EAAE;;2BAE1B,eAAe;4BACd,iBAAiB;;eAE9B,SACC,KAAK,EAAC,gBAAe,SAAS;CAC5C;;;;;;;;CASA,iBACE,gBACA,kBACA,YAAY,KACqB;EACjC,OACE,OAAO,cAAc,SAAS,KAAK,YAAY,GAC/C,gEACF;EACA,OAAO,KAAKb,GAA8B;kEACoB,KAAKa,KAAK,WAAW,EAAE;2BAC9D,eAAe;4BACd,iBAAiB;gCACb,OAAO,SAAS;CAG9C;CAMA,mBAAmB,WAAoC;EACrD,OAAO,MAAM,KAAKX,SAAS,OAAM,QAAO;GAGtC,MAAM,GAA0B;kCACJ,KAAKW,KAAK,WAAW,EAAE;;;;GASnD,MAAM,CAAC,EAAC,aAAY,MAAM,GAAwB;;;;;;;;;iBASvC,KAAKA,KAAK,WAAW,EAAE;;wBAEhB,KAAKA,KAAK,WAAW,EAAE,qBAAqB,UAAU;;;;GAKxE,MAAM,CAAC,EAAC,WAAU,MAAM,GAAuB;8BACvB,KAAKA,KAAK,kBAAkB,EAAE;GACtD,IAAI,UAAU,KAAKjB,SACjB,MAAM,IAAI,WACR,+BAA+B,UAAU,uCAAuC,OAClF;GAEF,OAAO,OAAO,OAAO;EACvB,CAAC;CACH;;;;CAKA,MAAM,WAAmB,MAAwB;EAI/C,MAAM,OAAO,0BAA0B,IAAI;EAC3C,KAAKkB,2BAA2B,KAAK;EAErC,MAAM,SAAS,KAAK;EACpB,KAAKR,OAAO,QAAQ;GAClB;GACA;GACA;GACA,aAAa,MAAM,IAAI,OAAO;EAChC,CAAC;EAED,OAAO;CACT;CAEA,QAAQ;EACN,KAAKA,OAAO,QAAQ,CAAC,OAAO,CAAC;CAC/B;CAEA,QAAQ,YAAwB,MAAsB;EACpD,KAAKA,OAAO,QAAQ,CAAC,cAAc;GAAC;GAAY;EAAI,CAAC,CAAC;CACxD;CAEA,gBAAuC;CAEvC,eAA0C;EACxC,IAAI,CAAC,KAAKS,UACR;EAEF,IACE,KAAKC,kBAAkB,QACvB,KAAKF,0BAA0B,KAAKP,6BACpC;GACA,KAAKb,IAAI,OACP,+BAA+B,KAAKY,OAAO,KAAK,EAAE,qBAAqB,KAAKQ,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,2SAW9H;GACA,KAAKE,gBAAgB,SAAS;EAChC;EACA,OAAO,KAAKA,eAAe;CAC7B;CAEA,4BAA4B;EAC1B,IAAI,KAAKA,kBAAkB;OAGvB,KAAKF,0BACL,KAAKP,8BAA8B,IACnC;IACA,KAAKb,IAAI,OACP,gCAAgC,KAAKY,OAAO,KAAK,EAAE,qBAAqB,KAAKQ,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,KAC/H;IACA,KAAKE,cAAc,QAAQ;IAC3B,KAAKA,gBAAgB;GACvB;;CAEJ;;;;;;;;;;;;CAaA,gBAAgB,IAAsD;EACpE,MAAM,EAAC,UAAS;EAChB,IAAI,MAAM,WAAW,GACnB,OAAO,GAAG;EAEZ,GAAG,QAAQ,CAAC;EACZ,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG;sBACa,KAAKH,KAAK,WAAW,EAAE;;mCAEV,MAAM;;;;;YAMrC,CAAC;EACD,OAAO,GAAG;CACZ;CAEA,WAAW;;;;;CAMX,MAAM,MAAM;EACV,OAAO,CAAC,KAAKE,UAAU,2BAA2B;EAElD,MAAM,EAAC,SAAS,SAAS,SAAS,kBAAiB,SAAS;EAC5D,KAAKA,WAAW;EAChB,KAAKE,WAAW;EAEhB,KAAKvB,IAAI,OAAO,iBAAiB;EACjC,KAAKiB,iBAAiB,MAAM;EAC5B,IAAI;EACJ,IAAI;GACF,MAAM,KAAKO,cAAc;EAC3B,SAAS,GAAG;GACV,MAAM;GACN,MAAM;EACR,UAAU;GAER,IAAI,KAAKF,kBAAkB,MAAM;IAC/B,KAAKA,cAAc,QAAQ;IAC3B,KAAKA,gBAAgB;GACvB;GACA,KAAKG,oBACH,KAAKb,OAAO,MAAM,EAAE,QAAO,UAAS,UAAU,KAAA,CAAS,GACvD,GACF;GACA,KAAKS,WAAW;GAChB,cAAc;GACd,KAAKrB,IAAI,OAAO,gBAAgB;EAClC;CACF;CAEA,oBAAoB,OAAqB,GAAY;EACnD,IAAI,MAAM,WAAW,GACnB;EAEF,KAAKA,IAAI,OACP,aAAa,MAAM,OAAO,kCAC5B;EACA,MAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,WAAW,sBAAsB;EAC1E,KAAK,MAAM,SAAS,OAAO;GACzB,IAAI,UAAU,QACZ;GAGF,QADa,MAAM,IACnB;IACE,KAAK,cAAc;KAGjB,MAAM,EAAC,eAAc,MAAM;KAC3B,KAAKA,IAAI,OAAO,iBAAiB,WAAW,IAAI;KAChD,WAAW,KAAK,GAAG;KACnB;IACF;GACF;EACF;CACF;CAEA,MAAMwB,gBAAgB;EACpB,IAAI,KAAgC;EACpC,IAAI;EAIJ,IAAI;EACJ,MAAM,cAAc,YAAY;GAC9B,eAAe;GACf,eAAe,KAAA;GACf,MAAM,QAAQ,MAAM,KAAKZ,OAAO,QAAQ;GACxC,IAAI,UAAU,QACZ,eAAe,KAAKK,iBAAiB,UAAU;IAC7C,MAAM;IAGN,MAAM,MAAM;GACd,CAAC;GAEH,OAAO;EACT;EAEA,MAAM,eAAoC,CAAC;EAC3C,IAAI;GACF,QAAQ,MAAM,MAAM,YAAY,OAAO,QAAQ;IAC7C,MAAM,CAAC,WAAW;IAClB,QAAQ,SAAR;KACE,KAAK,SAAS;MACZ,MAAM,cAAc,IAAI;MACxB,YAAY;MACZ;KACF;KACA,KAAK,cAAc;MACjB,MAAM,aAAa,IAAI;MACvB,IAAI,IACF,aAAa,KAAK,UAAU;WAE5B,MAAM,KAAKS,cAAc,CAAC,UAAU,CAAC;MAEvC;KACF;KACA,KAAK;MACH,IAAI,IAAI;OACN,GAAG,KAAK,MAAM;OACd,MAAM,GAAG,KAAK,KAAK;OACnB,KAAK;MACP;MACA;IAEJ;IAEA,MAAM,CAAC,GAAG,WAAW,MAAM,UAAU;IACrC,MAAM,MAAM,QAAQ;IACpB,KAAKN,2BAA2B,KAAK;IAErC,IAAI,QAAQ,SAAS;KACnB,OAAO,CAAC,IAAI,+CAA+C;KAC3D,MAAM,EAAC,SAAS,SAAS,WAAU,SAA2B;KAC9D,QAAa,YAAY,CAAC,CAAC;KAC3B,KAAK;MACH,MAAM,IAAI,gBACR,KAAKpB,IAAI,YAAY,aAAa,SAAS,GAC3C,EAAC,MAAM,eAAmB,CAC5B;MACA,oBAAoB;MACpB,KAAK;MACL,0BAA0B;MAC1B,KAAK,CAAC,OAAO;MACb,OAAO,CAAC;MACR,WAAW,KAAA;KACb;KACA,GAAG,KAAK,IAAI,KAAKO,SAAS;KAG1B,GAAQ,KAAK,SAAQ,OAAM;MACzB,EAA+B;gCACX,KAAKY,KAAK,kBAAkB,EAAE,aAAa,MAC5D,CAAC,YAAY,QAAQ,MAAM,GAC5B,MACF;MACA,OAAO,CAAC;KACV,CAAC;IACH,OAAO;KACL,OAAO,UAAU,2CAA2C,MAAM;KAClE,GAAG;IACL;IAEA,MAAM,QAAsB;KAC1B,WAAW,QAAQ,WAAW,YAAY,GAAG;KAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;KACtD,KAAK,GAAG;KAGR,QAAQ,uBAAuB,MAAM,GAAG;IAC1C;IAEA,IAAI,WAAW,QAAQ,eAAe,MAAM,GAAG;KAM7C,MAAM,KAAKQ,gBAAgB,EAAE;KAC7B,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG,eAAe,KAAKR,KAAK,WAAW,EAAE,GAAG,IAAI,KAAK,KACrD,GAAG,KAAKS,uBAAuB,KAAK,MAAM,CAC5C,CAAC;IACH,OAAO;KAML,GAAG,MAAM,KAAK,KAAK;KACnB,IAAI,GAAG,MAAM,UAAU,KAAKb,qBAAqB;MAK/C,MAAM,YAAY,GAAG;MACrB,KAAUY,gBAAgB,EAAE;MAC5B,MAAM;KACR;IACF;IACA,KAAKE,0BAA0B;IAE/B,IAAI,QAAQ,UAAU;KAIpB,KAAUF,gBAAgB,EAAE;KAE5B,MAAM,EAAC,UAAS,MAAM,GAAG;KACzB,IAAI,UAAU,KAAKzB,SAEjB,GAAG,KAAK,KACN,IAAI,WACF,2CAA2C,OAC7C,CACF;UACK;MAEL,MAAM,gBAAgB;MACtB,GAAQ,KAAK,SAAQ,OAAM,CACzB,EAAE;qBACK,KAAKiB,KAAK,kBAAkB,EAAE,OAAO,GAAG,EAAC,cAAa,CAAC,GAChE,CAAC;MACD,GAAG,KAAK,QAAQ;KAClB;KAEA,MAAM,GAAG,KAAK,KAAK;KAGnB,IAAI,GAAG,KACL,KAAKT,aAAa;MAAC;MAAU;MAAQ,EAAC,UAAS;KAAC,CAAC;KAEnD,KAAK;KAIL,MAAM,KAAKgB,cAAc,aAAa,OAAO,CAAC,CAAC;IACjD,OAAO,IAAI,QAAQ,YAAY;KAG7B,GAAG,KAAK,MAAM;KACd,MAAM,GAAG,KAAK,KAAK;KACnB,KAAK;KAEL,MAAM,KAAKA,cAAc,aAAa,OAAO,CAAC,CAAC;IACjD;GACF;EACF,SAAS,GAAG;GACV,aAAa,SAAS,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GACzD,MAAM;EACR;CACF;CAEA,MAAMA,cAAc,MAA2B;EAC7C,MAAM,cAAc,KAAK;EACzB,IAAI,gBAAgB,GAClB;EAGF,MAAM,KAAK,KAAK1B,IAAI,YAAY,QAAQ,SAAS;EACjD,MAAM,EAAC,MAAM,SAAS,eAAc,eAAe;EACnD,MAAM,SAAS,IAAI,gBAAgB,IAAI;GACrC,MAAM;GACN;GACA;GACA,gBAAgB,KAAK;EACvB,CAAC;EAID,MAAM,eAAe,KAAKK,oBACxB,sBACA,WACF;EACA,OAAO,IAAI,YAAY;EAEvB,IAAI;EACJ,MAAM,qBAAqB,KAAKY,iBAAiB,UAAU;GACzD,MAAM;GACN,aAAa,KAAK,KAAK,EAAC,YAAY,QAAO,EAAE,EAAE;EACjD,CAAC;EACD,IAAI;GAKF,CAAC,CAAC,kBAAkB,MAAM,OAAO,iBAC/B,QAAO,GAA8B;sCACP,KAAKE,KAAK,kBAAkB,EAAE;OAE9D;GACA,GAAG,OACD,qBAAqB,cAAc,IAAI,MAAM,WAAW,eACxC,YAAY,eAC9B;EACF,SAAS,GAAG;GACV,KAAK,KAAK,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GAC7C,MAAM;EACR,UAAU;GACR,mBAAmB;EACrB;EAIA,QAAa,WACX,KAAK,KAAI,QACP,KAAKW,SACH,GAAG,YAAY,cAAc,IAAI,WAAW,EAAE,GAC9C,KACA,eACA,MACF,CACF,CACF,EAAE,cAAc;GACd,OAAO,QAAQ;GACf,aAAkB,IAAI,EAAE,YAAY,CAAC,CAAC;EACxC,CAAC;CACH;CAEA,MAAMA,SACJ,IACA,EAAC,YAAY,KAAK,QAClB,eACA,QACA;EACA,IAAI,kBAAkB;EACtB,IAAI,oBAAoB,KAAKb,iBAAiB,UAAU;GACtD,MAAM;GACN,YAAY,IAAI;GAChB;EACF,CAAC;EACD,IAAI;GACF,GAAG,OAAO,kBAAkB;GAC5B,MAAM,OAAO,gBAAgB,OAAM,OAAM;IACvC,GAAG,OAAO,aAAa;IACvB,MAAM,QAAQ,KAAK,IAAI;IAIvB,IAAI,iBAAiB,IAAI,cAAc,KAAKR;IAC5C,IAAI,QAAQ;IACZ,IAAI;IAEJ,WAAW,MAAM,WAAW,EAA6B;sEACK,KAAKU,KAAK,WAAW,EAAE;gCAC7D,IAAI,UAAU;gCACd,cAAc;oCACV,OAAO,GAAI,GAAG;KACxC,kBAAkB;KAClB,mBAAmB,QAAQ;KAE3B,IAAI;MAOF,MAAM,QAAQ,YAAY,IAAI;MAC9B,MAAM;MACN,MAAM,UAAU,YAAY,IAAI,IAAI;MACpC,IAAI,mBACF,GAAG,UAAU,MAAM,SAAS,WAC1B,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,IAAI,GAAG,0CAChD;MAGF,KAAK,MAAM,SAAS,SAClB,IAAI,MAAM,cAAc,IAAI,WAG1B,iBAAiB;WACZ,IAAI,gBAAgB;OACzB,oBAAoB,IAAI,QACtB,6BAA6B,KAAK,CACpC;OACA;MACF,OAAO,IAAI,SAAS,UAClB,MAAM,IAAI,gBACR,+BAA+B,IAAI,UAAU,wBAAwB,MAAM,UAAU,EACvF;WACK;OACL,GAAG,OACD,qCAAqC,IAAI,UAAU,wBAAwB,MAAM,UAAU,EAC7F;OACA,IAAI,MACF,GACA,mCAAmC,MAAM,UAAU,cAAc,IAAI,UAAU,EACjF;OACA;MACF;KAEJ,UAAU;MAER,oBAAoB,KAAKF,iBAAiB,UAAU;OAClD,MAAM;OACN,YAAY,IAAI;OAChB;MACF,CAAC;KACH;IACF;IACA,kBAAkB;IAElB,IAAI,gBAAgB;KAClB,MAAM;KACN,GAAG,OACD,aAAa,IAAI,GAAG,QAAQ,MAAM,YAChC,KAAK,IAAI,IAAI,MACd,KACH;IACF,OAWE,GAAG,OACD,cAAc,IAAI,GAAG,gBAAgB,IAAI,UAAU,4CACjB,cAAc,wCAElD;IAMF,IAAS,YAAY;GACvB,CAAC;EACH,SAAS,KAAK;GACZ,GAAG,QAAQ,sCAAsC,IAAI,MAAM,GAAG;GAC9D,IAAI,eAAe,iBAAiB;IAClC,MAAM,kBAAkB,KAAKX,KAAK,KAAKL,MAAM;IAC7C,KAAKU,SAAS,GAAG;GACnB;GACA,IAAI,KAAK,GAAG;EACd,UAAU;GACR,kBAAkB;EACpB;CACF;;;;;;;;;;;;CAaA,uBAAuB,KAA0B,QAAsB;EACrE,OAAO,UAAU,MAAM,EAAE,SAAQ,OAAM,KAAKoB,aAAa,KAAK,EAAE,CAAC;CACnE;CAEA,aAAa,KAA0B,IAAqC;EAC1E,QAAQ,GAAG,IAAX;GACE,KAAK,mBACH,OAAO,CAAC,KAAKC,yBAAyB,KAAK,GAAG,OAAO,GAAG,QAAQ,CAAC;GAEnE,KAAK,mBACH,OAAO,CACL,KAAKC,0BAA0B,KAAK,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,CACtE;GAEF,KAAK,gBAAgB;IACnB,MAAM,EAAC,QAAO;IACd,MAAM,MAAM;KAAC,QAAQ,GAAG,IAAI;KAAQ,OAAO,GAAG,IAAI;IAAI;IACtD,OAAO,CACL,GAAG,UAAU,KAAKd,KAAK,eAAe,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC/B,IAAI,OAAO,iBAAiB,IAAI,QACzD,GAAG,UAAU,KAAKA,KAAK,aAAa,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC7B,IAAI,OAAO,iBAAiB,IAAI,MAC3D;GACF;GAEA,KAAK,cAAc;IACjB,MAAM,EAAC,QAAQ,SAAQ,GAAG;IAC1B,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,eAAe,EAAE;mCACpB,OAAO,iBAAiB,QACjD,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MACnD;GACF;GAEA,KAAK,iBAAiB;IACpB,MAAM,EAAC,QAAQ,MAAM,UAAS,GAAG;IACjC,OAAO,CACL,GAAG,UAAU,KAAKA,KAAK,aAAa,EAAE,kBAAkB,GAAG,IAAI;mCACtC,OAAO,iBAAiB,MAAM,kBAAkB,GAAG,KAC9E;GACF;GAEA,KAAK,eAAe;IAClB,MAAM,EAAC,QAAQ,SAAQ,GAAG;IAC1B,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,KAAK,kBAAkB,GAAG,QAC7E;GACF;GAEA,KAAK,qBAAqB;IACxB,MAAM,EAAC,QAAQ,MAAM,UAAS,GAAG;IACjC,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MAAM,mBAAmB,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAC/F;GACF;GAEA,SACE,YAAY,EAAE;EAClB;CACF;CAEA,yBACE,KACA,EAAC,QAAQ,MAAM,SACf,UACA;EACA,MAAM,MAAwB;GAAC;GAAQ;GAAO;EAAQ;EACtD,OAAO,GAAG;sBACQ,KAAKA,KAAK,eAAe,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEnC,IAAI,GAAG,EAAE;;CAEjC;CAEA,0BACE,KACA,EAAC,QAAQ,MAAM,SACf,QACA,UACA;EACA,MAAM,MAAyB;GAAC;GAAQ;GAAO;GAAQ;EAAQ;EAC/D,OAAO,GAAG;sBACQ,KAAKA,KAAK,aAAa,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEjC,IAAI,GAAG,EAAE;;CAEjC;;;;;CAMA,MAAM,eAAe;EACnB,IAAI,KAAKE,UAAU;GACjB,MAAM,EAAC,SAAS,YAAW,SAAS;GACpC,KAAKT,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;GACtC,MAAM;EACR;CACF;;;;;;CAOA,MAAM,OAAO;EACX,IAAI,KAAKS,UAAU;GACjB,KAAKJ,iBAAiB,KAAK;GAC3B,KAAKjB,IAAI,OAAO,YAAY,KAAKY,OAAO,KAAK,EAAE,mBAAmB;GAClE,KAAK,MAAM;GACX,KAAKA,OAAO,QAAQ,MAAM;EAC5B;EACA,IACG,MAAM,UAAU,KAAKW,UAAU,KAAKP,eAAe,MAAO,aAE3D,MAAM,IAAI,WACR,kCAAkC,KAAKA,gBAAgB,GACzD;CAEJ;AACF;AAEA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CAEA,YACE,IACA,IACA,gBACA,WACA;EACA,KAAKhB,MAAM;EACX,KAAKkC,MAAM;EACX,KAAK,iBAAiB;EACtB,KAAK,eAAe;CACtB;CAEA,YAAY;CAEZ,MAAM,UAAU;EACd,IAAI,KAAKC,WACP;EAEF,KAAKA,YAAY;EACjB,KAAKD,IAAI,QAAQ;EACjB,MAAM,KAAKA,IACR,KAAK,EACL,OAAM,MAAK,KAAKlC,IAAI,OAAO,iCAAiC,CAAC,CAAC;EACjE,KAAKA,IAAI,OAAO,0BAA0B,KAAK,cAAc;CAC/D;AACF;AAEA,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CAEA,YAAY,IAAgB,OAAgB,IAAgB;EAC1D,KAAKA,MAAM,GAAG,YAAY,aAAa,cAAc;EACrD,KAAKC,SAAS;EACd,KAAKK,MAAM;CACb;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKA,IAAI,GAAG,UAAU,KAAKL,MAAM,EAAE,GAAG,OAAO;CACtD;CAEA,MAAM,UAAU;EACd,MAAM,KAAK,IAAI,gBAAgB,KAAKD,KAAK,EAAC,MAAM,eAAmB,CAAC,EAAE,IACpE,KAAKM,GACP;EACA,MAAM,MAAM,MAAM,GAAG,iBACnB,QAAO,GAA0B;8BACT,KAAKa,KAAK,WAAW,EAAE;;;KAIjD;EACA,IAAI,IAAI,WAAW,GAAG;GACpB,KAAKnB,IAAI,OAAO,4CAA4C;GAC5D,GAAG,QAAQ;GACX,MAAM,GAAG,KAAK;GACd,OAAO;EACT;EACA,MAAM,CAAC,EAAC,eAAc;EACtB,MAAM,CAAC,EAAC,oBAAmB,MAAM,GAAG,iBAClC,QAAO,GAA+B;uCACL,KAAKmB,KAAK,mBAAmB,EAAE;OAElE;EACA,KAAKnB,IAAI,OACP,oBAAoB,UAAU,kCAAkC,gBAClE;EACA,OAAO,IAAI,UAAU,KAAKA,KAAK,IAAI,gBAAgB,SAAS;CAC9D;AACF;;;;;;;;;;;;;;;;;;AA0BA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA,+BAAwB,IAAI,IAAmB;CAE/C;CAEA,YACE,IACA,oBACA,WACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,yBAAyB;EAChE,KAAKoC,sBAAsB;EAC3B,KAAKC,aAAa;CACpB;CAEA,QAAQ;EACN,cAAc,KAAKE,MAAM;EACzB,KAAKA,SAAS,kBACN,KAAK,kCAAkB,IAAI,KAAK,CAAC,GACvC,KAAKH,mBACP,EAAE,MAAM;CACV;CAGA,kBAAkB,KAAW;EAC3B,KAAK,MAAM,CAAC,MAAM,eAAe,KAAKE,aAAa,QAAQ,GACzD,IAAI,IAAI,QAAQ,IAAI,WAAW,QAAQ,KAAK,KAAKF,qBAAqB;GACpE,KAAKpC,IAAI,QACP,kCAAkC,KAAKoC,oBAAoB,SAC3D;IAAC;IAAM;GAAU,CACnB;GACA,KAAKC,2BACH,IAAI,MACF,oCAAoC,KAAKD,oBAAoB,MAAM,WAAW,UAAU,IAAI,GAC9F,CACF;GACA,KAAK,KAAK;GACV;EACF;CAEJ;;;;;;CAOA,UAAU,MAAe,sBAAM,IAAI,KAAK,GAAe;EACrD,KAAKE,aAAa,IAAI,MAAM,GAAG;EAC/B,aAAa;GACX,KAAKA,aAAa,OAAO,IAAI;EAC/B;CACF;CAEA,OAAO;EACL,cAAc,KAAKC,MAAM;EACzB,KAAKD,aAAa,MAAM;CAC1B;AACF"}

package/out/zero-cache/src/services/change-streamer/subscriber.d.ts

@@ -1,5 +1,6 @@
 import type { Enum } from '../../../../shared/src/enum.ts';
 import type { Subscription } from '../../types/subscription.ts';
+import type { ReplicatorMode } from '../replicator/replicator.ts';
 import type { ChangeTag, Status, WatermarkedChange } from './change-streamer.ts';
 import type * as ErrorType from './error-type-enum.ts';
 type ErrorType = Enum<typeof ErrorType>;
@@ -35,7 +36,8 @@ export type SubscriberStats = {
 export declare class Subscriber {
     #private;
     readonly id: string;
-    constructor(protocolVersion: number, id: string, watermark: string, downstream: Subscription<string>, latestStatus: () => Status, options?: SubscriberOptions);
+    readonly mode: ReplicatorMode;
+    constructor(protocolVersion: number, id: string, mode: ReplicatorMode, watermark: string, downstream: Subscription<string>, latestStatus: () => Status, options?: SubscriberOptions);
     get watermark(): string;
     get acked(): string;
     /**
@@ -45,6 +47,11 @@ export declare class Subscriber {
      * subscriber to form a majority it stalls replication outright.
      */
     get backlogFull(): boolean;
+    /**
+     * @returns whether the subscriber is currently sending backlogged messages,
+     *          vs caught up and sending the "head" of the replication stream.
+     */
+    isBacklogged(): boolean;
     /**
      * Resolves the first time {@link backlogFull} becomes true, so that a caller
      * holding the subscriber in catchup can give up at the moment the subscriber

package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"subscriber.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AAKzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EACV,SAAS,EAET,MAAM,EACN,iBAAiB,EAClB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAEvD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAKxC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE1C;;;;OAIG;IACH,KAAK,CAAC,EAAE,CAAC,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;CACnD,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,IAAI,IAAI,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,OAAO,CAAC;CAC5B,CAAC;AAEF;;;;;;GAMG;AACH,qBAAa,UAAU;;IAErB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAiBlB,eAAe,EAAE,MAAM,EACvB,EAAE,EAAE,MAAM,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,YAAY,CAAC,MAAM,CAAC,EAChC,YAAY,EAAE,MAAM,MAAM,EAC1B,OAAO,GAAE,iBAAsB;IAgBjC,IAAI,SAAS,WAEZ;IAED,IAAI,KAAK,WAER;IAED;;;;;OAKG;IACH,IAAI,WAAW,YAId;IAED;;;;;OAKG;IACH,eAAe,IAAI,eAAe;IAclC,IAAI,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IA4B9C,UAAU,CAAC,MAAM,EAAE,MAAM;IAMzB,kEAAkE;IAC5D,OAAO,CAAC,MAAM,EAAE,iBAAiB;IAKvC;;;OAGG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAyE5B;;OAEG;IACH,IAAI,UAAU,WAEb;IAED;;;OAGG;IACH,IAAI,YAAY,WAEf;IAED;;;OAGG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,SAAK,GAAG,IAAI;IAQrD,QAAQ,IAAI,eAAe;IA8B3B,mBAAmB,CAAC,MAAM,EAAE,SAAS,GAAG,WAAW;IAOnD;;;;;;;;;;OAUG;IACH,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,aAAa;IAa/D,eAAe,CAAC,GAAG,EAAE,SAAS;IAS9B;;;;;;;;;;;OAWG;IACH,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO;IAInB,KAAK,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,MAAM;CA4G1C"}
+{"version":3,"file":"subscriber.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AAKzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EACV,SAAS,EAET,MAAM,EACN,iBAAiB,EAClB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAEvD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAKxC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE1C;;;;OAIG;IACH,KAAK,CAAC,EAAE,CAAC,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;CACnD,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,IAAI,IAAI,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,OAAO,CAAC;CAC5B,CAAC;AAEF;;;;;;GAMG;AACH,qBAAa,UAAU;;IAErB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;gBAiB5B,eAAe,EAAE,MAAM,EACvB,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,cAAc,EACpB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,YAAY,CAAC,MAAM,CAAC,EAChC,YAAY,EAAE,MAAM,MAAM,EAC1B,OAAO,GAAE,iBAAsB;IAiBjC,IAAI,SAAS,WAEZ;IAED,IAAI,KAAK,WAER;IAED;;;;;OAKG;IACH,IAAI,WAAW,YAId;IAED;;;OAGG;IACH,YAAY;IAIZ;;;;;OAKG;IACH,eAAe,IAAI,eAAe;IAclC,IAAI,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IA4B9C,UAAU,CAAC,MAAM,EAAE,MAAM;IAMzB,kEAAkE;IAC5D,OAAO,CAAC,MAAM,EAAE,iBAAiB;IAKvC;;;OAGG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAyE5B;;OAEG;IACH,IAAI,UAAU,WAEb;IAED;;;OAGG;IACH,IAAI,YAAY,WAEf;IAED;;;OAGG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,SAAK,GAAG,IAAI;IAQrD,QAAQ,IAAI,eAAe;IA8B3B,mBAAmB,CAAC,MAAM,EAAE,SAAS,GAAG,WAAW;IAOnD;;;;;;;;;;OAUG;IACH,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,aAAa;IAa/D,eAAe,CAAC,GAAG,EAAE,SAAS;IAS9B;;;;;;;;;;;OAWG;IACH,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO;IAInB,KAAK,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,MAAM;CA4G1C"}

package/out/zero-cache/src/services/change-streamer/subscriber.js

@@ -18,6 +18,7 @@ var DEFAULT_BACKLOG_LOW_WATER_RATIO = .8;
 var Subscriber = class {
 	#protocolVersion;
 	id;
+	mode;
 	#downstream;
 	#latestStatus;
 	#watermark;
@@ -29,9 +30,10 @@ var Subscriber = class {
 	#backlogBackpressure;
 	#backlogFullWaiters = /* @__PURE__ */ new Set();
 	#onAck;
-	constructor(protocolVersion, id, watermark, downstream, latestStatus, options = {}) {
+	constructor(protocolVersion, id, mode, watermark, downstream, latestStatus, options = {}) {
 		this.#protocolVersion = protocolVersion;
 		this.id = id;
+		this.mode = mode;
 		this.#downstream = downstream;
 		this.#latestStatus = latestStatus;
 		this.#watermark = watermark;
@@ -56,6 +58,13 @@ var Subscriber = class {
 		return this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes;
 	}
 	/**
+	* @returns whether the subscriber is currently sending backlogged messages,
+	*          vs caught up and sending the "head" of the replication stream.
+	*/
+	isBacklogged() {
+		return this.#backlog !== null || this.#bufferedBacklogBytes > 0;
+	}
+	/**
 	* Resolves the first time {@link backlogFull} becomes true, so that a caller
 	* holding the subscriber in catchup can give up at the moment the subscriber
 	* starts costing replication rather than on a timer. Call cancel() when the

package/out/zero-cache/src/services/change-streamer/subscriber.js.map

@@ -1 +1 @@
-{"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#latestStatus","#backlogBackpressure","#backlogFullWaiters","#onAck","#watermark","#acked","#backlog","#bufferedBacklogBytes","#pushBacklog","#maybeWaitForBacklogSpace","#sendChange","#initialized","#sendDownstream","#initialize","#backlogDrain","#drainBacklog","#sendStringifiedDownstream","#pending","#pendingBytes","#processed","#backlogCount","#samples","#totalBufferedBytes","#missedLastTimeout","#laggingSinceMs","#backlogBytes","#resolveBacklogFullWaiters","#backlogInFlightBytes","#lowWaterBytes","#waiters"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n};\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly #downstream: Subscription<string>;\n  readonly #latestStatus: () => Status;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<WatermarkedChange> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    watermark: string,\n    downstream: Subscription<string>,\n    latestStatus: () => Status,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.#downstream = downstream;\n    this.#latestStatus = latestStatus;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      this.sendStatus(this.#latestStatus());\n    }\n  }\n\n  sendStatus(status: Status) {\n    if (this.#initialized) {\n      void this.#sendDownstream(['status', status]);\n    }\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      // Sends can complete out of order (e.g. the bounded window in\n      // #drainBacklog), so the ack only advances monotonically, and listeners\n      // are only notified when it does.\n      const acked = max(this.#acked, watermark);\n      if (acked !== this.#acked) {\n        this.#acked = acked;\n        this.#onAck?.(acked);\n      }\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(error?: ErrorType, message?: string) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream(['error', {type: error, message}]).finally(() =>\n        this.#downstream.cancel(),\n      );\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(change: WatermarkedChange) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(change);\n    this.#backlogBytes += change[2].length;\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const change = this.#backlog?.shift();\n        if (!change) {\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          break;\n        }\n\n        const bytes = change[2].length;\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const promise = this.#sendChange(change).finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAmBA,IAAM,mCAAmC,KAAK,OAAO;AACrD,IAAM,kCAAkC;;;;;;;;AAmCxC,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CAEA,YACE,iBACA,IACA,WACA,YACA,cACA,UAA6B,CAAC,GAC9B;EACA,KAAKA,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAKC,cAAc;EACnB,KAAKC,gBAAgB;EACrB,KAAKI,aAAa;EAClB,KAAKC,SAAS;EACd,KAAKC,WAAW,IAAI,WAAW;EAC/B,KAAKL,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAKE,SAAS,QAAQ;CACxB;CAEA,IAAI,YAAY;EACd,OAAO,KAAKC;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAKC;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAKE,yBAAyB,KAAKN,qBAAqB;CAE5D;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAKC,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAKA,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAKE,YAAY;GAC/B,IAAI,KAAKE,UAAU;IAIjB,KAAKE,aAAa,MAAM;IACxB,OAAO,KAAKC,0BAA0B;GACxC;GACA,OAAO,KAAKC,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAKC,cAAc;GACtB,KAAKA,eAAe;GACpB,KAAK,WAAW,KAAKX,cAAc,CAAC;EACtC;CACF;CAEA,WAAW,QAAgB;EACzB,IAAI,KAAKW,cACP,KAAUC,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAEhD;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAKC,YAAY;EACjB,MAAM,KAAKH,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAKG,YAAY;EACjB,IAAI,CAAC,KAAKP,UACR,OAAO,KAAKQ,iBAAiB;EAE/B,IAAI,CAAC,KAAKA,eAAe;GAKvB,KAAKA,gBAAgB,KAAKC,cAAc;GACxC,KAAUD,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAKA;CACd;CAEA,MAAMJ,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAKN,aAAa;EAEpB,MAAM,SAAS,MAAM,KAAKY,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YAAY;GAI7C,MAAM,QAAQ,IAAI,KAAKX,QAAQ,SAAS;GACxC,IAAI,UAAU,KAAKA,QAAQ;IACzB,KAAKA,SAAS;IACd,KAAKF,SAAS,KAAK;GACrB;EACF;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAKa,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAMA,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAKC;EACL,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAKnB,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAKkB;GACL,KAAKC,iBAAiB;GACtB,KAAKC;EACP;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAKF,WAAW,KAAKG;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAKD;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAKE,SAAS,UAAU,YAC7B,KAAKA,SAAS,MAAM;EAEtB,KAAKA,SAAS,KAAK;GAAC,WAAW,KAAKF;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAKE,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAKD;GACd,cAAc,KAAKb;GACnB,oBAAoB,KAAKe;GACzB,mBAAmB,KAAKC;EAC1B;EAEF,MAAM,OAAO,KAAKF,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAKA,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAKD;GACd,cAAc,KAAKb;GACnB,oBAAoB,KAAKe;GACzB,mBAAmB,KAAKC;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAKA,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAKC,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAKD,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAKC,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAKA,oBAAoB;EACzB,OAAO,MAAM,KAAKA;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAK1B,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MAAM,OAAmB,SAAkB;EAGzC,KAAKQ,WAAW;EAChB,KAAKmB,gBAAgB;EACrB,KAAKxB,qBAAqB,WAAW;EAGrC,KAAKyB,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAUd,gBAAgB,CAAC,SAAS;GAAC,MAAM;GAAO;EAAO,CAAC,CAAC,EAAE,cAC3D,KAAKb,YAAY,OAAO,CAC1B;OAEA,KAAKA,YAAY,OAAO;CAE5B;CAEA,IAAIqB,gBAAgB;EAClB,OAAO,KAAKd,UAAU,QAAQ;CAChC;CAEA,IAAIC,wBAAwB;EAI1B,OAAO,KAAKkB,gBAAgB,KAAKE;CACnC;CAEA,IAAIL,sBAAsB;EACxB,OAAO,KAAKf,wBAAwB,KAAKW;CAC3C;CAEA,aAAa,QAA2B;EACtC,OAAO,KAAKZ,UAAU,gDAAgD;EACtE,KAAKA,SAAS,KAAK,MAAM;EACzB,KAAKmB,iBAAiB,OAAO,GAAG;EAChC,IAAI,KAAK,aACP,KAAKC,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAKxB,mBAAmB;EAC5C,KAAKA,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAKD,qBAAqB,aAAa,KAAKM,qBAAqB;CAC1E;CAEA,MAAMQ,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,SAAS,KAAKT,UAAU,MAAM;IACpC,IAAI,CAAC,QAAQ;KACX,KAAKA,WAAW;KAChB,KAAKmB,gBAAgB;KACrB,KAAKxB,qBAAqB,uBACxB,KAAKM,qBACP;KACA;IACF;IAEA,MAAM,QAAQ,OAAO,GAAG;IACxB,KAAKkB,iBAAiB;IACtB,KAAKE,yBAAyB;IAC9B,KAAK1B,qBAAqB,uBACxB,KAAKM,qBACP;IAKA,MAAM,UAAU,KAAKG,YAAY,MAAM,EAAE,cAAc;KACrD,KAAKiB,yBAAyB;KAC9B,KAAK1B,qBAAqB,uBACxB,KAAKM,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAKN,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAKa,gBAAgB;GACrB,KAAKb,qBAAqB,uBACxB,KAAKM,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAKqB,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAKC,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAKA,SAAS,WAAW,KAAK,gBAAgB,KAAKD,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAKC,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF"}
+{"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#latestStatus","#backlogBackpressure","#backlogFullWaiters","#onAck","#watermark","#acked","#backlog","#bufferedBacklogBytes","#pushBacklog","#maybeWaitForBacklogSpace","#sendChange","#initialized","#sendDownstream","#initialize","#backlogDrain","#drainBacklog","#sendStringifiedDownstream","#pending","#pendingBytes","#processed","#backlogCount","#samples","#totalBufferedBytes","#missedLastTimeout","#laggingSinceMs","#backlogBytes","#resolveBacklogFullWaiters","#backlogInFlightBytes","#lowWaterBytes","#waiters"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n};\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly mode: ReplicatorMode;\n  readonly #downstream: Subscription<string>;\n  readonly #latestStatus: () => Status;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<WatermarkedChange> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    mode: ReplicatorMode,\n    watermark: string,\n    downstream: Subscription<string>,\n    latestStatus: () => Status,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.mode = mode;\n    this.#downstream = downstream;\n    this.#latestStatus = latestStatus;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * @returns whether the subscriber is currently sending backlogged messages,\n   *          vs caught up and sending the \"head\" of the replication stream.\n   */\n  isBacklogged() {\n    return this.#backlog !== null || this.#bufferedBacklogBytes > 0;\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      this.sendStatus(this.#latestStatus());\n    }\n  }\n\n  sendStatus(status: Status) {\n    if (this.#initialized) {\n      void this.#sendDownstream(['status', status]);\n    }\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      // Sends can complete out of order (e.g. the bounded window in\n      // #drainBacklog), so the ack only advances monotonically, and listeners\n      // are only notified when it does.\n      const acked = max(this.#acked, watermark);\n      if (acked !== this.#acked) {\n        this.#acked = acked;\n        this.#onAck?.(acked);\n      }\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(error?: ErrorType, message?: string) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream(['error', {type: error, message}]).finally(() =>\n        this.#downstream.cancel(),\n      );\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(change: WatermarkedChange) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(change);\n    this.#backlogBytes += change[2].length;\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const change = this.#backlog?.shift();\n        if (!change) {\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          break;\n        }\n\n        const bytes = change[2].length;\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const promise = this.#sendChange(change).finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAoBA,IAAM,mCAAmC,KAAK,OAAO;AACrD,IAAM,kCAAkC;;;;;;;;AAmCxC,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CAEA,YACE,iBACA,IACA,MACA,WACA,YACA,cACA,UAA6B,CAAC,GAC9B;EACA,KAAKA,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAKC,cAAc;EACnB,KAAKC,gBAAgB;EACrB,KAAKI,aAAa;EAClB,KAAKC,SAAS;EACd,KAAKC,WAAW,IAAI,WAAW;EAC/B,KAAKL,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAKE,SAAS,QAAQ;CACxB;CAEA,IAAI,YAAY;EACd,OAAO,KAAKC;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAKC;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAKE,yBAAyB,KAAKN,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAKK,aAAa,QAAQ,KAAKC,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAKL,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAKA,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAKE,YAAY;GAC/B,IAAI,KAAKE,UAAU;IAIjB,KAAKE,aAAa,MAAM;IACxB,OAAO,KAAKC,0BAA0B;GACxC;GACA,OAAO,KAAKC,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAKC,cAAc;GACtB,KAAKA,eAAe;GACpB,KAAK,WAAW,KAAKX,cAAc,CAAC;EACtC;CACF;CAEA,WAAW,QAAgB;EACzB,IAAI,KAAKW,cACP,KAAUC,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAEhD;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAKC,YAAY;EACjB,MAAM,KAAKH,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAKG,YAAY;EACjB,IAAI,CAAC,KAAKP,UACR,OAAO,KAAKQ,iBAAiB;EAE/B,IAAI,CAAC,KAAKA,eAAe;GAKvB,KAAKA,gBAAgB,KAAKC,cAAc;GACxC,KAAUD,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAKA;CACd;CAEA,MAAMJ,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAKN,aAAa;EAEpB,MAAM,SAAS,MAAM,KAAKY,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YAAY;GAI7C,MAAM,QAAQ,IAAI,KAAKX,QAAQ,SAAS;GACxC,IAAI,UAAU,KAAKA,QAAQ;IACzB,KAAKA,SAAS;IACd,KAAKF,SAAS,KAAK;GACrB;EACF;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAKa,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAMA,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAKC;EACL,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAKnB,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAKkB;GACL,KAAKC,iBAAiB;GACtB,KAAKC;EACP;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAKF,WAAW,KAAKG;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAKD;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAKE,SAAS,UAAU,YAC7B,KAAKA,SAAS,MAAM;EAEtB,KAAKA,SAAS,KAAK;GAAC,WAAW,KAAKF;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAKE,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAKD;GACd,cAAc,KAAKb;GACnB,oBAAoB,KAAKe;GACzB,mBAAmB,KAAKC;EAC1B;EAEF,MAAM,OAAO,KAAKF,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAKA,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAKD;GACd,cAAc,KAAKb;GACnB,oBAAoB,KAAKe;GACzB,mBAAmB,KAAKC;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAKA,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAKC,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAKD,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAKC,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAKA,oBAAoB;EACzB,OAAO,MAAM,KAAKA;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAK1B,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MAAM,OAAmB,SAAkB;EAGzC,KAAKQ,WAAW;EAChB,KAAKmB,gBAAgB;EACrB,KAAKxB,qBAAqB,WAAW;EAGrC,KAAKyB,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAUd,gBAAgB,CAAC,SAAS;GAAC,MAAM;GAAO;EAAO,CAAC,CAAC,EAAE,cAC3D,KAAKb,YAAY,OAAO,CAC1B;OAEA,KAAKA,YAAY,OAAO;CAE5B;CAEA,IAAIqB,gBAAgB;EAClB,OAAO,KAAKd,UAAU,QAAQ;CAChC;CAEA,IAAIC,wBAAwB;EAI1B,OAAO,KAAKkB,gBAAgB,KAAKE;CACnC;CAEA,IAAIL,sBAAsB;EACxB,OAAO,KAAKf,wBAAwB,KAAKW;CAC3C;CAEA,aAAa,QAA2B;EACtC,OAAO,KAAKZ,UAAU,gDAAgD;EACtE,KAAKA,SAAS,KAAK,MAAM;EACzB,KAAKmB,iBAAiB,OAAO,GAAG;EAChC,IAAI,KAAK,aACP,KAAKC,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAKxB,mBAAmB;EAC5C,KAAKA,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAKD,qBAAqB,aAAa,KAAKM,qBAAqB;CAC1E;CAEA,MAAMQ,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,SAAS,KAAKT,UAAU,MAAM;IACpC,IAAI,CAAC,QAAQ;KACX,KAAKA,WAAW;KAChB,KAAKmB,gBAAgB;KACrB,KAAKxB,qBAAqB,uBACxB,KAAKM,qBACP;KACA;IACF;IAEA,MAAM,QAAQ,OAAO,GAAG;IACxB,KAAKkB,iBAAiB;IACtB,KAAKE,yBAAyB;IAC9B,KAAK1B,qBAAqB,uBACxB,KAAKM,qBACP;IAKA,MAAM,UAAU,KAAKG,YAAY,MAAM,EAAE,cAAc;KACrD,KAAKiB,yBAAyB;KAC9B,KAAK1B,qBAAqB,uBACxB,KAAKM,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAKN,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAKa,gBAAgB;GACrB,KAAKb,qBAAqB,uBACxB,KAAKM,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAKqB,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAKC,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAKA,SAAS,WAAW,KAAK,gBAAgB,KAAKD,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAKC,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF"}

package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts

@@ -0,0 +1,29 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Source } from '../../types/streams.ts';
+import type { BackedUpWatermark } from './backup-monitor.ts';
+export type VfsPollerConfig = {
+    executable: string;
+    remotePollIntervalMs: number;
+    remoteQueryTimeoutMs?: number;
+    backupURL: string;
+    region?: string | undefined;
+    endpoint?: string | undefined;
+    logLevel?: string;
+    logFormat?: string;
+};
+export declare class VfsWatermarkPoller {
+    #private;
+    constructor(lc: LogContext, replicaFile: string, pollerConfig: VfsPollerConfig);
+    /**
+     * Starts polling for watermarks and pushes new ones to returned stream.
+     * Must only be called once.
+     */
+    start(): Source<BackedUpWatermark>;
+    readonly checkLocalWatermark: () => void;
+}
+export declare function buildLitestreamVfsReplicaURL({ backupURL, endpoint, region, }: {
+    readonly backupURL: string;
+    readonly endpoint?: string | undefined;
+    readonly region?: string | undefined;
+}): string;
+//# sourceMappingURL=vfs-watermark-poller.d.ts.map

package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-poller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/vfs-watermark-poller.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,qBAAqB,CAAC;AAe3D,MAAM,MAAM,eAAe,GAAG;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAyBF,qBAAa,kBAAkB;;gBAmB3B,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,eAAe;IAyB/B;;;OAGG;IACH,KAAK,IAAI,MAAM,CAAC,iBAAiB,CAAC;IAYlC,QAAQ,CAAC,mBAAmB,aAiB1B;CA+GH;AAED,wBAAgB,4BAA4B,CAAC,EAC3C,SAAS,EACT,QAAQ,EACR,MAAM,GACP,EAAE;IACD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC,GAAG,MAAM,CAiBT"}