@rocicorp/zero 0.26.1-canary.9 → 0.26.2-canary.1

package/out/zero-cache/src/services/change-streamer/storer.js.map

@@ -1 +1 @@
-{"version":3,"file":"storer.js","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {getHeapStatistics} from 'node:v8';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {\n  backfillRequestSchema,\n  isDataChange,\n  isSchemaChange,\n  type BackfillID,\n  type BackfillRequest,\n  type Change,\n  type DataChange,\n  type Identifier,\n  type SchemaChange,\n  type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {type Commit} from '../change-source/protocol/current/downstream.ts';\nimport type {\n  DownstreamStatusMessage,\n  UpstreamStatusMessage,\n} from '../change-source/protocol/current/status.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type ChangeEntry} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n  AutoResetSignal,\n  markResetRequired,\n  type BackfillingColumn,\n  type ReplicationState,\n  type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n  subscriber: Subscriber;\n  mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n  | [\n      'change',\n      watermark: string,\n      json: string,\n      orig: Exclude<Change, DataChange> | null, // null for DataChanges\n    ]\n  | ['ready', callback: () => void]\n  | ['subscriber', SubscriberAndMode]\n  | DownstreamStatusMessage\n  | ['abort']\n  | 'stop';\n\ntype PendingTransaction = {\n  pool: TransactionPool;\n  preCommitWatermark: string;\n  pos: number;\n  startingReplicationState: Promise<ReplicationState>;\n  ack: boolean;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n *   `replicaVersion`, which is the version at which initial sync\n *   was performed, and any attempts to catchup from a different\n *   point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n *   a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n *   subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n  readonly id = 'storer';\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #taskID: string;\n  readonly #discoveryAddress: string;\n  readonly #discoveryProtocol: string;\n  readonly #db: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #onConsumed: (c: Commit | UpstreamStatusMessage) => void;\n  readonly #onFatal: (err: Error) => void;\n  readonly #queue = new Queue<QueueEntry>();\n  readonly #backPressureThresholdBytes: number;\n\n  #approximateQueuedBytes = 0;\n  #running = false;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    db: PostgresDB,\n    replicaVersion: string,\n    onConsumed: (c: Commit | UpstreamStatusMessage) => void,\n    onFatal: (err: Error) => void,\n    backPressureLimitHeapProportion: number,\n  ) {\n    this.#lc = lc.withContext('component', 'change-log');\n    this.#shard = shard;\n    this.#taskID = taskID;\n    this.#discoveryAddress = discoveryAddress;\n    this.#discoveryProtocol = discoveryProtocol;\n    this.#db = db;\n    this.#replicaVersion = replicaVersion;\n    this.#onConsumed = onConsumed;\n    this.#onFatal = onFatal;\n\n    const heapStats = getHeapStatistics();\n    this.#backPressureThresholdBytes =\n      (heapStats.heap_size_limit - heapStats.used_heap_size) *\n      backPressureLimitHeapProportion;\n\n    this.#lc.info?.(\n      `Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of ` +\n        `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) ` +\n        `to absorb upstream spikes`,\n      {heapStats},\n    );\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async assumeOwnership() {\n    const db = this.#db;\n    const owner = this.#taskID;\n    const ownerAddress = this.#discoveryAddress;\n    const ownerProtocol = this.#discoveryProtocol;\n    // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n    const addressWithProtocol =\n      ownerProtocol === 'ws'\n        ? ownerAddress\n        : `${ownerProtocol}://${ownerAddress}`;\n    this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n    const start = performance.now();\n    await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n    const elapsed = (performance.now() - start).toFixed(2);\n    this.#lc.info?.(\n      `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n    );\n  }\n\n  async getStartStreamInitializationParameters(): Promise<{\n    lastWatermark: string;\n    backfillRequests: BackfillRequest[];\n  }> {\n    const [[{lastWatermark}], result] = await runTx(\n      this.#db,\n      sql => [\n        sql<{lastWatermark: string}[]>`\n        SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n        // Formats a BackfillRequest using json_object_agg() to construct the\n        // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n        // to make it optional and possibly `null`.\n        sql`\n        SELECT \n            json_build_object(\n              'schema', b.\"schema\",\n              'name', b.\"table\",\n              'metadata', t.\"metadata\"\n            ) as \"table\",\n            json_object_agg(b.\"column\", b.\"backfill\") \n              as \"columns\"\n          FROM ${this.#cdc('backfilling')} as b\n          LEFT JOIN ${this.#cdc('tableMetadata')} as t\n          ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n          GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n        `,\n      ],\n      {mode: Mode.READONLY},\n    );\n\n    return {\n      lastWatermark,\n      backfillRequests: v.parse(result, backfillRequestsSchema),\n    };\n  }\n\n  async getMinWatermarkForCatchup(): Promise<string | null> {\n    const [{minWatermark}] = await this.#db<\n      {minWatermark: string | null}[]\n    > /*sql*/ `\n      SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n    return minWatermark;\n  }\n\n  purgeRecordsBefore(watermark: string): Promise<number> {\n    return runTx(this.#db, async sql => {\n      const [{deleted}] = await sql<{deleted: bigint}[]>`\n        WITH purged AS (\n          DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n            RETURNING watermark, pos\n        ) SELECT COUNT(*) as deleted FROM purged;`;\n\n      // Before committing the purge, check that this process is still the\n      // owner. This is done after the DELETE to minimize the amount of time\n      // that writes to the changeLog are delayed.\n      const [{owner}] = await sql<ReplicationState[]>`\n        SELECT * FROM ${this.#cdc('replicationState')} FOR SHARE`;\n      if (owner !== this.#taskID) {\n        throw new AbortError(\n          `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n        );\n      }\n      return Number(deleted);\n    });\n  }\n\n  /**\n   * @returns The size of the serialized entry, for memory / I/O estimations.\n   */\n  store(entry: WatermarkedChange) {\n    const [watermark, [_tag, change]] = entry;\n    // Eagerly stringify the JSON object so that the memory usage can be\n    // more accurately measured (i.e. without an extra object traversal and\n    // ad hoc memory counting heuristics).\n    //\n    // This essentially moves the stringify() computation out of the pg client,\n    // which is instead configured to pass `string` objects directly as JSON\n    // strings for JSON-valued columns (see TypeOptions.sendStringAsJson).\n    const json = BigIntJSON.stringify(change);\n    this.#approximateQueuedBytes += json.length;\n\n    this.#queue.enqueue([\n      'change',\n      watermark,\n      json,\n      isDataChange(change) ? null : change, // drop DataChanges to save memory\n    ]);\n\n    return json.length;\n  }\n\n  abort() {\n    this.#queue.enqueue(['abort']);\n  }\n\n  status(s: DownstreamStatusMessage) {\n    this.#queue.enqueue(s);\n  }\n\n  catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n    this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n  }\n\n  #readyForMore: Resolver<void> | null = null;\n\n  readyForMore(): Promise<void> | undefined {\n    if (!this.#running) {\n      return undefined;\n    }\n    if (\n      this.#readyForMore === null &&\n      this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n    ) {\n      this.#lc.warn?.(\n        `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\\n` +\n          `\\n` +\n          `To inspect changeLog backlog in your change DB:\\n` +\n          `  SELECT\\n` +\n          `    (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\\n` +\n          `    change->>'tag' AS operation,\\n` +\n          `    COUNT(*) AS count\\n` +\n          `  FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n          `  GROUP BY 1, 2\\n` +\n          `  ORDER BY 3 DESC\\n` +\n          `  LIMIT 20;`,\n      );\n      this.#readyForMore = resolver();\n    }\n    return this.#readyForMore?.promise;\n  }\n\n  #maybeReleaseBackPressure() {\n    if (\n      this.#readyForMore !== null &&\n      // Wait for at least 20% of the threshold to free up.\n      this.#approximateQueuedBytes < this.#backPressureThresholdBytes * 0.8\n    ) {\n      this.#lc.info?.(\n        `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`,\n      );\n      this.#readyForMore.resolve();\n      this.#readyForMore = null;\n    }\n  }\n\n  #stopped = promiseVoid;\n\n  /**\n   * Runs the storer loop until {@link stop()} is called, or an error is thrown.\n   * Once {@link run()} completes, it can be called again.\n   */\n  async run() {\n    assert(!this.#running, `storer is already running`);\n\n    const {promise: stopped, resolve: signalStopped} = resolver();\n    this.#running = true;\n    this.#stopped = stopped;\n\n    this.#lc.info?.('starting storer');\n    try {\n      await this.#processQueue();\n    } finally {\n      // Release any pending backpressure so the upstream can proceed\n      if (this.#readyForMore !== null) {\n        this.#readyForMore.resolve();\n        this.#readyForMore = null;\n      }\n      const unprocessed = this.#queue.drain();\n      if (unprocessed.length) {\n        this.#lc.warn?.(\n          `dropped ${unprocessed.length} entries from the changeLog queue`,\n        );\n      }\n      this.#running = false;\n      signalStopped();\n      this.#lc.info?.('storer stopped');\n    }\n  }\n\n  async #processQueue() {\n    let tx: PendingTransaction | null = null;\n    let msg: QueueEntry | false;\n\n    const catchupQueue: SubscriberAndMode[] = [];\n    while ((msg = await this.#queue.dequeue()) !== 'stop') {\n      const [msgType] = msg;\n      switch (msgType) {\n        case 'ready': {\n          const signalReady = msg[1];\n          signalReady();\n          continue;\n        }\n        case 'subscriber': {\n          const subscriber = msg[1];\n          if (tx) {\n            catchupQueue.push(subscriber); // Wait for the current tx to complete.\n          } else {\n            await this.#startCatchup([subscriber]); // Catch up immediately.\n          }\n          continue;\n        }\n        case 'status':\n          this.#onConsumed(msg);\n          continue;\n        case 'abort': {\n          if (tx) {\n            tx.pool.abort();\n            await tx.pool.done();\n            tx = null;\n          }\n          continue;\n        }\n      }\n      // msgType === 'change'\n      const [_, watermark, json, change] = msg;\n      const tag = change?.tag;\n      this.#approximateQueuedBytes -= json.length;\n\n      if (tag === 'begin') {\n        assert(!tx, 'received BEGIN in the middle of a transaction');\n        const {promise, resolve, reject} = resolver<ReplicationState>();\n        tx = {\n          pool: new TransactionPool(\n            this.#lc.withContext('watermark', watermark),\n            Mode.READ_COMMITTED,\n          ),\n          preCommitWatermark: watermark,\n          pos: 0,\n          startingReplicationState: promise,\n          ack: !change.skipAck,\n        };\n        tx.pool.run(this.#db);\n        // Acquire a lock on the replicationState row to detect and/or prevent\n        // a concurrent ownership change.\n        void tx.pool.process(tx => {\n          tx<ReplicationState[]> /*sql*/ `\n          SELECT * FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n            ([result]) => resolve(result),\n            reject,\n          );\n          return [];\n        });\n      } else {\n        assert(tx, () => `received change outside of transaction: ${json}`);\n        tx.pos++;\n      }\n\n      const entry = {\n        watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n        precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n        pos: tx.pos,\n        change: json,\n      };\n\n      const processed = tx.pool.process(sql => [\n        sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n        ...(change !== null && isSchemaChange(change)\n          ? this.#trackBackfillMetadata(sql, change)\n          : []),\n      ]);\n\n      if (tx.pos % 100 === 0) {\n        // Backpressure is exerted on commit when awaiting tx.pool.done().\n        // However, backpressure checks need to be regularly done for\n        // very large transactions in order to avoid memory blowup.\n        await processed;\n      }\n      this.#maybeReleaseBackPressure();\n\n      if (tag === 'commit') {\n        const {owner} = await tx.startingReplicationState;\n        if (owner !== this.#taskID) {\n          // Ownership change reflected in the replicationState read in 'begin'.\n          tx.pool.fail(\n            new AbortError(`changeLog ownership has been assumed by ${owner}`),\n          );\n        } else {\n          // Update the replication state.\n          const lastWatermark = watermark;\n          void tx.pool.process(tx => [\n            tx`\n            UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n          ]);\n          tx.pool.setDone();\n        }\n\n        await tx.pool.done();\n\n        // ACK the LSN to the upstream Postgres.\n        if (tx.ack) {\n          this.#onConsumed(['commit', change, {watermark}]);\n        }\n        tx = null;\n\n        // Before beginning the next transaction, open a READONLY snapshot to\n        // concurrently catchup any queued subscribers.\n        await this.#startCatchup(catchupQueue.splice(0));\n      } else if (tag === 'rollback') {\n        // Aborted transactions are not stored in the changeLog. Abort the current tx\n        // and process catchup of subscribers that were waiting for it to end.\n        tx.pool.abort();\n        await tx.pool.done();\n        tx = null;\n\n        await this.#startCatchup(catchupQueue.splice(0));\n      }\n    }\n  }\n\n  async #startCatchup(subs: SubscriberAndMode[]) {\n    if (subs.length === 0) {\n      return;\n    }\n\n    const reader = new TransactionPool(\n      this.#lc.withContext('pool', 'catchup'),\n      Mode.READONLY,\n    );\n    reader.run(this.#db);\n\n    // Ensure that the transaction has started (and is thus holding a snapshot\n    // of the database) before continuing on to commit more changes. This is\n    // done by performing a single read on the db, which determines the\n    // snapshot for the REPEATABLE_READ transaction.\n    const [{lastWatermark}] = await reader.processReadTask(\n      sql => sql<ReplicationState[]>`\n        SELECT * FROM ${this.#cdc('replicationState')}\n      `,\n    );\n\n    // Run the actual catchup queries in the background. Errors are handled in\n    // #catchup() by disconnecting the associated subscriber.\n    void Promise.all(\n      subs.map(sub => this.#catchup(sub, lastWatermark, reader)),\n    ).finally(() => reader.setDone());\n  }\n\n  async #catchup(\n    {subscriber: sub, mode}: SubscriberAndMode,\n    lastWatermark: string,\n    reader: TransactionPool,\n  ) {\n    try {\n      await reader.processReadTask(async tx => {\n        const start = Date.now();\n\n        // When starting from initial-sync, there won't be a change with a watermark\n        // equal to the replica version. This is the empty changeLog scenario.\n        let watermarkFound = sub.watermark === this.#replicaVersion;\n        let count = 0;\n        let lastBatchConsumed: Promise<unknown> | undefined;\n\n        for await (const entries of tx<ChangeEntry[]> /*sql*/ `\n          SELECT watermark, change FROM ${this.#cdc('changeLog')}\n           WHERE watermark >= ${sub.watermark}\n             AND watermark <= ${lastWatermark}\n           ORDER BY watermark, pos`.cursor(2000)) {\n          // Wait for the last batch of entries to be consumed by the\n          // subscriber before sending down the current batch. This pipelining\n          // allows one batch of changes to be received from the change-db\n          // while the previous batch of changes are sent to the subscriber,\n          // resulting in flow control that caps the number of changes\n          // referenced in memory to 2 * batch-size.\n          const start = performance.now();\n          await lastBatchConsumed;\n          const elapsed = performance.now() - start;\n          if (lastBatchConsumed) {\n            (elapsed > 100 ? this.#lc.info : this.#lc.debug)?.(\n              `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n            );\n          }\n\n          for (const entry of entries) {\n            if (entry.watermark === sub.watermark) {\n              // This should be the first entry.\n              // Catchup starts from *after* the watermark.\n              watermarkFound = true;\n            } else if (watermarkFound) {\n              lastBatchConsumed = sub.catchup(toDownstream(entry));\n              count++;\n            } else if (mode === 'backup') {\n              throw new AutoResetSignal(\n                `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n              );\n            } else {\n              this.#lc.warn?.(\n                `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n              );\n              sub.close(\n                ErrorType.WatermarkTooOld,\n                `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n              );\n              return;\n            }\n          }\n        }\n        if (watermarkFound) {\n          await lastBatchConsumed;\n          this.#lc.info?.(\n            `caught up ${sub.id} with ${count} changes (${\n              Date.now() - start\n            } ms)`,\n          );\n        } else {\n          this.#lc.warn?.(\n            `subscriber at watermark ${sub.watermark} is ahead of latest watermark`,\n          );\n        }\n        // Flushes the backlog of messages buffered during catchup and\n        // allows the subscription to forward subsequent messages immediately.\n        sub.setCaughtUp();\n      });\n    } catch (err) {\n      this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n      if (err instanceof AutoResetSignal) {\n        await markResetRequired(this.#db, this.#shard);\n        this.#onFatal(err);\n      }\n      sub.fail(err);\n    }\n  }\n\n  /**\n   * Returns the db statements necessary to track backfill and table metadata\n   * presented in the `change`, if any.\n   */\n  #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n    const stmts: PendingQuery<Row[]>[] = [];\n\n    switch (change.tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        stmts.push(this.#upsertTableMetadataStmt(sql, table, metadata));\n        break;\n      }\n\n      case 'create-table': {\n        const {spec, metadata, backfill} = change;\n        if (metadata) {\n          stmts.push(this.#upsertTableMetadataStmt(sql, spec, metadata));\n        }\n        if (backfill) {\n          Object.entries(backfill).forEach(([col, backfill]) => {\n            stmts.push(\n              this.#upsertColumnBackfillStmt(sql, spec, col, backfill),\n            );\n          });\n        }\n        break;\n      }\n\n      case 'rename-table': {\n        const {old} = change;\n        const row = {schema: change.new.schema, table: change.new.name};\n        stmts.push(\n          sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n          sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n        );\n        break;\n      }\n\n      case 'drop-table': {\n        const {\n          id: {schema, name},\n        } = change;\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('tableMetadata')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n        );\n        break;\n      }\n\n      case 'add-column': {\n        const {table, tableMetadata, column, backfill} = change;\n        if (tableMetadata) {\n          stmts.push(this.#upsertTableMetadataStmt(sql, table, tableMetadata));\n        }\n        if (backfill) {\n          stmts.push(\n            this.#upsertColumnBackfillStmt(sql, table, column.name, backfill),\n          );\n        }\n        break;\n      }\n\n      case 'update-column': {\n        const {\n          table: {schema, name: table},\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          stmts.push(\n            sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${newName}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${oldName}`,\n          );\n        }\n        break;\n      }\n\n      case 'drop-column': {\n        const {\n          table: {schema, name},\n          column,\n        } = change;\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${column}`,\n        );\n        break;\n      }\n\n      case 'backfill-completed': {\n        const {\n          relation: {schema, name: table, rowKey},\n          columns,\n        } = change;\n        const cols = [...rowKey.columns, ...columns];\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql(cols)}`,\n        );\n      }\n    }\n    return stmts;\n  }\n\n  #upsertTableMetadataStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    metadata: TableMetadata,\n  ) {\n    const row: TableMetadataRow = {schema, table, metadata};\n    return sql`\n        INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\") \n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  #upsertColumnBackfillStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    column: string,\n    backfill: BackfillID,\n  ) {\n    const row: BackfillingColumn = {schema, table, column, backfill};\n    return sql`\n        INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\", \"column\") \n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  /**\n   * Waits until all currently queued entries have been processed.\n   * This is only used in tests.\n   */\n  async allProcessed() {\n    if (this.#running) {\n      const {promise, resolve} = resolver();\n      this.#queue.enqueue(['ready', resolve]);\n      await promise;\n    }\n  }\n\n  stop() {\n    if (this.#running) {\n      this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n      this.#queue.enqueue('stop');\n    }\n    return this.#stopped;\n  }\n}\n\nfunction toDownstream(entry: ChangeEntry): WatermarkedChange {\n  const {watermark, change} = entry;\n  switch (change.tag) {\n    case 'begin':\n      return [watermark, ['begin', change, {commitWatermark: watermark}]];\n    case 'commit':\n      return [watermark, ['commit', change, {watermark}]];\n    case 'rollback':\n      return [watermark, ['rollback', change]];\n    default:\n      return [watermark, ['data', change]];\n  }\n}\n"],"names":["v.array","Mode.READONLY","v.parse","Mode.READ_COMMITTED","tx","start","ErrorType.WatermarkTooOld","backfill"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAwEA,MAAM,yBAAyBA,MAAQ,qBAAqB;AAgCrD,MAAM,OAA0B;AAAA,EAC5B,KAAK;AAAA,EACL;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS,IAAI,MAAA;AAAA,EACb;AAAA,EAET,0BAA0B;AAAA,EAC1B,WAAW;AAAA,EAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,YACA,SACA,iCACA;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,YAAY;AACnD,SAAK,SAAS;AACd,SAAK,UAAU;AACf,SAAK,oBAAoB;AACzB,SAAK,qBAAqB;AAC1B,SAAK,MAAM;AACX,SAAK,kBAAkB;AACvB,SAAK,cAAc;AACnB,SAAK,WAAW;AAEhB,UAAM,YAAY,kBAAA;AAClB,SAAK,+BACF,UAAU,kBAAkB,UAAU,kBACvC;AAEF,SAAK,IAAI;AAAA,MACP,gBAAgB,KAAK,8BAA8B,QAAQ,GAAG,QAAQ,CAAC,CAAC,kCAC3C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,CAAC,CAAC;AAAA,MAE9E,EAAC,UAAA;AAAA,IAAS;AAAA,EAEd;AAAA;AAAA,EAGA,KAAK,OAAe;AAClB,WAAO,KAAK,IAAI,GAAG,UAAU,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE;AAAA,EACtD;AAAA,EAEA,MAAM,kBAAkB;AACtB,UAAM,KAAK,KAAK;AAChB,UAAM,QAAQ,KAAK;AACnB,UAAM,eAAe,KAAK;AAC1B,UAAM,gBAAgB,KAAK;AAE3B,UAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,aAAa,MAAM,YAAY;AACxC,SAAK,IAAI,OAAO,yBAAyB,mBAAmB,EAAE;AAC9D,UAAM,QAAQ,YAAY,IAAA;AAC1B,UAAM,YAAY,KAAK,KAAK,kBAAkB,CAAC,QAAQ,GAAG,EAAC,OAAO,cAAc,oBAAA,CAAoB,CAAC;AACrG,UAAM,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACrD,SAAK,IAAI;AAAA,MACP,wBAAwB,mBAAmB,KAAK,OAAO;AAAA,IAAA;AAAA,EAE3D;AAAA,EAEA,MAAM,yCAGH;AACD,UAAM,CAAC,CAAC,EAAC,cAAA,CAAc,GAAG,MAAM,IAAI,MAAM;AAAA,MACxC,KAAK;AAAA,MACL,CAAA,QAAO;AAAA,QACL;AAAA,sCAC8B,KAAK,KAAK,kBAAkB,CAAC;AAAA;AAAA;AAAA;AAAA,QAK3D;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBASS,KAAK,KAAK,aAAa,CAAC;AAAA,sBACnB,KAAK,KAAK,eAAe,CAAC;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,MAK1C,EAAC,MAAMC,SAAK;AAAA,IAAQ;AAGtB,WAAO;AAAA,MACL;AAAA,MACA,kBAAkBC,MAAQ,QAAQ,sBAAsB;AAAA,IAAA;AAAA,EAE5D;AAAA,EAEA,MAAM,4BAAoD;AACxD,UAAM,CAAC,EAAC,aAAA,CAAa,IAAI,MAAM,KAAK;AAAA,qDAGa,KAAK,KAAK,WAAW,CAAC;AACvE,WAAO;AAAA,EACT;AAAA,EAEA,mBAAmB,WAAoC;AACrD,WAAO,MAAM,KAAK,KAAK,OAAM,QAAO;AAClC,YAAM,CAAC,EAAC,SAAQ,IAAI,MAAM;AAAA;AAAA,wBAER,KAAK,KAAK,WAAW,CAAC,sBAAsB,SAAS;AAAA;AAAA;AAOvE,YAAM,CAAC,EAAC,OAAM,IAAI,MAAM;AAAA,wBACN,KAAK,KAAK,kBAAkB,CAAC;AAC/C,UAAI,UAAU,KAAK,SAAS;AAC1B,cAAM,IAAI;AAAA,UACR,+BAA+B,SAAS,wCAAwC,KAAK;AAAA,QAAA;AAAA,MAEzF;AACA,aAAO,OAAO,OAAO;AAAA,IACvB,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,OAA0B;AAC9B,UAAM,CAAC,WAAW,CAAC,MAAM,MAAM,CAAC,IAAI;AAQpC,UAAM,OAAO,WAAW,UAAU,MAAM;AACxC,SAAK,2BAA2B,KAAK;AAErC,SAAK,OAAO,QAAQ;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,MAAM,IAAI,OAAO;AAAA;AAAA,IAAA,CAC/B;AAED,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,QAAQ;AACN,SAAK,OAAO,QAAQ,CAAC,OAAO,CAAC;AAAA,EAC/B;AAAA,EAEA,OAAO,GAA4B;AACjC,SAAK,OAAO,QAAQ,CAAC;AAAA,EACvB;AAAA,EAEA,QAAQ,YAAwB,MAAsB;AACpD,SAAK,OAAO,QAAQ,CAAC,cAAc,EAAC,YAAY,KAAA,CAAK,CAAC;AAAA,EACxD;AAAA,EAEA,gBAAuC;AAAA,EAEvC,eAA0C;AACxC,QAAI,CAAC,KAAK,UAAU;AAClB,aAAO;AAAA,IACT;AACA,QACE,KAAK,kBAAkB,QACvB,KAAK,0BAA0B,KAAK,6BACpC;AACA,WAAK,IAAI;AAAA,QACP,+BAA+B,KAAK,OAAO,KAAA,CAAM,sBAAsB,KAAK,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA;AAY7H,WAAK,gBAAgB,SAAA;AAAA,IACvB;AACA,WAAO,KAAK,eAAe;AAAA,EAC7B;AAAA,EAEA,4BAA4B;AAC1B,QACE,KAAK,kBAAkB;AAAA,IAEvB,KAAK,0BAA0B,KAAK,8BAA8B,KAClE;AACA,WAAK,IAAI;AAAA,QACP,gCAAgC,KAAK,OAAO,KAAA,CAAM,sBAAsB,KAAK,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,CAAC;AAAA,MAAA;AAE9H,WAAK,cAAc,QAAA;AACnB,WAAK,gBAAgB;AAAA,IACvB;AAAA,EACF;AAAA,EAEA,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA,EAMX,MAAM,MAAM;AACV,WAAO,CAAC,KAAK,UAAU,2BAA2B;AAElD,UAAM,EAAC,SAAS,SAAS,SAAS,cAAA,IAAiB,SAAA;AACnD,SAAK,WAAW;AAChB,SAAK,WAAW;AAEhB,SAAK,IAAI,OAAO,iBAAiB;AACjC,QAAI;AACF,YAAM,KAAK,cAAA;AAAA,IACb,UAAA;AAEE,UAAI,KAAK,kBAAkB,MAAM;AAC/B,aAAK,cAAc,QAAA;AACnB,aAAK,gBAAgB;AAAA,MACvB;AACA,YAAM,cAAc,KAAK,OAAO,MAAA;AAChC,UAAI,YAAY,QAAQ;AACtB,aAAK,IAAI;AAAA,UACP,WAAW,YAAY,MAAM;AAAA,QAAA;AAAA,MAEjC;AACA,WAAK,WAAW;AAChB,oBAAA;AACA,WAAK,IAAI,OAAO,gBAAgB;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,MAAM,gBAAgB;AACpB,QAAI,KAAgC;AACpC,QAAI;AAEJ,UAAM,eAAoC,CAAA;AAC1C,YAAQ,MAAM,MAAM,KAAK,OAAO,QAAA,OAAe,QAAQ;AACrD,YAAM,CAAC,OAAO,IAAI;AAClB,cAAQ,SAAA;AAAA,QACN,KAAK,SAAS;AACZ,gBAAM,cAAc,IAAI,CAAC;AACzB,sBAAA;AACA;AAAA,QACF;AAAA,QACA,KAAK,cAAc;AACjB,gBAAM,aAAa,IAAI,CAAC;AACxB,cAAI,IAAI;AACN,yBAAa,KAAK,UAAU;AAAA,UAC9B,OAAO;AACL,kBAAM,KAAK,cAAc,CAAC,UAAU,CAAC;AAAA,UACvC;AACA;AAAA,QACF;AAAA,QACA,KAAK;AACH,eAAK,YAAY,GAAG;AACpB;AAAA,QACF,KAAK,SAAS;AACZ,cAAI,IAAI;AACN,eAAG,KAAK,MAAA;AACR,kBAAM,GAAG,KAAK,KAAA;AACd,iBAAK;AAAA,UACP;AACA;AAAA,QACF;AAAA,MAAA;AAGF,YAAM,CAAC,GAAG,WAAW,MAAM,MAAM,IAAI;AACrC,YAAM,MAAM,QAAQ;AACpB,WAAK,2BAA2B,KAAK;AAErC,UAAI,QAAQ,SAAS;AACnB,eAAO,CAAC,IAAI,+CAA+C;AAC3D,cAAM,EAAC,SAAS,SAAS,OAAA,IAAU,SAAA;AACnC,aAAK;AAAA,UACH,MAAM,IAAI;AAAA,YACR,KAAK,IAAI,YAAY,aAAa,SAAS;AAAA,YAC3CC;AAAAA,UAAK;AAAA,UAEP,oBAAoB;AAAA,UACpB,KAAK;AAAA,UACL,0BAA0B;AAAA,UAC1B,KAAK,CAAC,OAAO;AAAA,QAAA;AAEf,WAAG,KAAK,IAAI,KAAK,GAAG;AAGpB,aAAK,GAAG,KAAK,QAAQ,CAAAC,QAAM;AACzBA;AAAAA,0BACgB,KAAK,KAAK,kBAAkB,CAAC,cAAc;AAAA,YACzD,CAAC,CAAC,MAAM,MAAM,QAAQ,MAAM;AAAA,YAC5B;AAAA,UAAA;AAEF,iBAAO,CAAA;AAAA,QACT,CAAC;AAAA,MACH,OAAO;AACL,eAAO,IAAI,MAAM,2CAA2C,IAAI,EAAE;AAClE,WAAG;AAAA,MACL;AAEA,YAAM,QAAQ;AAAA,QACZ,WAAW,QAAQ,WAAW,YAAY,GAAG;AAAA,QAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;AAAA,QACtD,KAAK,GAAG;AAAA,QACR,QAAQ;AAAA,MAAA;AAGV,YAAM,YAAY,GAAG,KAAK,QAAQ,CAAA,QAAO;AAAA,QACvC,kBAAkB,KAAK,KAAK,WAAW,CAAC,IAAI,IAAI,KAAK,CAAC;AAAA,QACtD,GAAI,WAAW,QAAQ,eAAe,MAAM,IACxC,KAAK,uBAAuB,KAAK,MAAM,IACvC,CAAA;AAAA,MAAC,CACN;AAED,UAAI,GAAG,MAAM,QAAQ,GAAG;AAItB,cAAM;AAAA,MACR;AACA,WAAK,0BAAA;AAEL,UAAI,QAAQ,UAAU;AACpB,cAAM,EAAC,MAAA,IAAS,MAAM,GAAG;AACzB,YAAI,UAAU,KAAK,SAAS;AAE1B,aAAG,KAAK;AAAA,YACN,IAAI,WAAW,2CAA2C,KAAK,EAAE;AAAA,UAAA;AAAA,QAErE,OAAO;AAEL,gBAAM,gBAAgB;AACtB,eAAK,GAAG,KAAK,QAAQ,CAAAA,QAAM;AAAA,YACzBA;AAAAA,qBACS,KAAK,KAAK,kBAAkB,CAAC,QAAQA,IAAG,EAAC,cAAA,CAAc,CAAC;AAAA,UAAA,CAClE;AACD,aAAG,KAAK,QAAA;AAAA,QACV;AAEA,cAAM,GAAG,KAAK,KAAA;AAGd,YAAI,GAAG,KAAK;AACV,eAAK,YAAY,CAAC,UAAU,QAAQ,EAAC,UAAA,CAAU,CAAC;AAAA,QAClD;AACA,aAAK;AAIL,cAAM,KAAK,cAAc,aAAa,OAAO,CAAC,CAAC;AAAA,MACjD,WAAW,QAAQ,YAAY;AAG7B,WAAG,KAAK,MAAA;AACR,cAAM,GAAG,KAAK,KAAA;AACd,aAAK;AAEL,cAAM,KAAK,cAAc,aAAa,OAAO,CAAC,CAAC;AAAA,MACjD;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,MAA2B;AAC7C,QAAI,KAAK,WAAW,GAAG;AACrB;AAAA,IACF;AAEA,UAAM,SAAS,IAAI;AAAA,MACjB,KAAK,IAAI,YAAY,QAAQ,SAAS;AAAA,MACtCH;AAAAA,IAAK;AAEP,WAAO,IAAI,KAAK,GAAG;AAMnB,UAAM,CAAC,EAAC,cAAA,CAAc,IAAI,MAAM,OAAO;AAAA,MACrC,CAAA,QAAO;AAAA,wBACW,KAAK,KAAK,kBAAkB,CAAC;AAAA;AAAA,IAAA;AAMjD,SAAK,QAAQ;AAAA,MACX,KAAK,IAAI,CAAA,QAAO,KAAK,SAAS,KAAK,eAAe,MAAM,CAAC;AAAA,IAAA,EACzD,QAAQ,MAAM,OAAO,SAAS;AAAA,EAClC;AAAA,EAEA,MAAM,SACJ,EAAC,YAAY,KAAK,KAAA,GAClB,eACA,QACA;AACA,QAAI;AACF,YAAM,OAAO,gBAAgB,OAAM,OAAM;AACvC,cAAM,QAAQ,KAAK,IAAA;AAInB,YAAI,iBAAiB,IAAI,cAAc,KAAK;AAC5C,YAAI,QAAQ;AACZ,YAAI;AAEJ,yBAAiB,WAAW;AAAA,0CACM,KAAK,KAAK,WAAW,CAAC;AAAA,gCAChC,IAAI,SAAS;AAAA,gCACb,aAAa;AAAA,oCACT,OAAO,GAAI,GAAG;AAOxC,gBAAMI,SAAQ,YAAY,IAAA;AAC1B,gBAAM;AACN,gBAAM,UAAU,YAAY,IAAA,IAAQA;AACpC,cAAI,mBAAmB;AACrB,aAAC,UAAU,MAAM,KAAK,IAAI,OAAO,KAAK,IAAI;AAAA,cACxC,UAAU,QAAQ,QAAQ,CAAC,CAAC,WAAW,IAAI,EAAE;AAAA,YAAA;AAAA,UAEjD;AAEA,qBAAW,SAAS,SAAS;AAC3B,gBAAI,MAAM,cAAc,IAAI,WAAW;AAGrC,+BAAiB;AAAA,YACnB,WAAW,gBAAgB;AACzB,kCAAoB,IAAI,QAAQ,aAAa,KAAK,CAAC;AACnD;AAAA,YACF,WAAW,SAAS,UAAU;AAC5B,oBAAM,IAAI;AAAA,gBACR,+BAA+B,IAAI,SAAS,yBAAyB,MAAM,SAAS;AAAA,cAAA;AAAA,YAExF,OAAO;AACL,mBAAK,IAAI;AAAA,gBACP,qCAAqC,IAAI,SAAS,yBAAyB,MAAM,SAAS;AAAA,cAAA;AAE5F,kBAAI;AAAA,gBACFC;AAAAA,gBACA,mCAAmC,MAAM,SAAS,eAAe,IAAI,SAAS;AAAA,cAAA;AAEhF;AAAA,YACF;AAAA,UACF;AAAA,QACF;AACA,YAAI,gBAAgB;AAClB,gBAAM;AACN,eAAK,IAAI;AAAA,YACP,aAAa,IAAI,EAAE,SAAS,KAAK,aAC/B,KAAK,QAAQ,KACf;AAAA,UAAA;AAAA,QAEJ,OAAO;AACL,eAAK,IAAI;AAAA,YACP,2BAA2B,IAAI,SAAS;AAAA,UAAA;AAAA,QAE5C;AAGA,YAAI,YAAA;AAAA,MACN,CAAC;AAAA,IACH,SAAS,KAAK;AACZ,WAAK,IAAI,QAAQ,sCAAsC,IAAI,EAAE,IAAI,GAAG;AACpE,UAAI,eAAe,iBAAiB;AAClC,cAAM,kBAAkB,KAAK,KAAK,KAAK,MAAM;AAC7C,aAAK,SAAS,GAAG;AAAA,MACnB;AACA,UAAI,KAAK,GAAG;AAAA,IACd;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,uBAAuB,KAA0B,QAAsB;AACrE,UAAM,QAA+B,CAAA;AAErC,YAAQ,OAAO,KAAA;AAAA,MACb,KAAK,yBAAyB;AAC5B,cAAM,EAAC,OAAO,KAAK,SAAA,IAAY;AAC/B,cAAM,KAAK,KAAK,yBAAyB,KAAK,OAAO,QAAQ,CAAC;AAC9D;AAAA,MACF;AAAA,MAEA,KAAK,gBAAgB;AACnB,cAAM,EAAC,MAAM,UAAU,SAAA,IAAY;AACnC,YAAI,UAAU;AACZ,gBAAM,KAAK,KAAK,yBAAyB,KAAK,MAAM,QAAQ,CAAC;AAAA,QAC/D;AACA,YAAI,UAAU;AACZ,iBAAO,QAAQ,QAAQ,EAAE,QAAQ,CAAC,CAAC,KAAKC,SAAQ,MAAM;AACpD,kBAAM;AAAA,cACJ,KAAK,0BAA0B,KAAK,MAAM,KAAKA,SAAQ;AAAA,YAAA;AAAA,UAE3D,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAAA,MAEA,KAAK,gBAAgB;AACnB,cAAM,EAAC,QAAO;AACd,cAAM,MAAM,EAAC,QAAQ,OAAO,IAAI,QAAQ,OAAO,OAAO,IAAI,KAAA;AAC1D,cAAM;AAAA,UACJ,aAAa,KAAK,KAAK,eAAe,CAAC,QAAQ,IAAI,GAAG,CAAC;AAAA,mCAC9B,IAAI,MAAM,kBAAkB,IAAI,IAAI;AAAA,UAC7D,aAAa,KAAK,KAAK,aAAa,CAAC,QAAQ,IAAI,GAAG,CAAC;AAAA,mCAC5B,IAAI,MAAM,kBAAkB,IAAI,IAAI;AAAA,QAAA;AAE/D;AAAA,MACF;AAAA,MAEA,KAAK,cAAc;AACjB,cAAM;AAAA,UACJ,IAAI,EAAC,QAAQ,KAAA;AAAA,QAAI,IACf;AACJ,cAAM;AAAA,UACJ,kBAAkB,KAAK,KAAK,eAAe,CAAC;AAAA,mCACnB,MAAM,kBAAkB,IAAI;AAAA,UACrD,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAAA,mCACjB,MAAM,kBAAkB,IAAI;AAAA,QAAA;AAEvD;AAAA,MACF;AAAA,MAEA,KAAK,cAAc;AACjB,cAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;AACjD,YAAI,eAAe;AACjB,gBAAM,KAAK,KAAK,yBAAyB,KAAK,OAAO,aAAa,CAAC;AAAA,QACrE;AACA,YAAI,UAAU;AACZ,gBAAM;AAAA,YACJ,KAAK,0BAA0B,KAAK,OAAO,OAAO,MAAM,QAAQ;AAAA,UAAA;AAAA,QAEpE;AACA;AAAA,MACF;AAAA,MAEA,KAAK,iBAAiB;AACpB,cAAM;AAAA,UACJ,OAAO,EAAC,QAAQ,MAAM,MAAA;AAAA,UACtB,KAAK,EAAC,MAAM,QAAA;AAAA,UACZ,KAAK,EAAC,MAAM,QAAA;AAAA,QAAO,IACjB;AACJ,YAAI,YAAY,SAAS;AACvB,gBAAM;AAAA,YACJ,aAAa,KAAK,KAAK,aAAa,CAAC,mBAAmB,OAAO;AAAA,mCACxC,MAAM,kBAAkB,KAAK,mBAAmB,OAAO;AAAA,UAAA;AAAA,QAElF;AACA;AAAA,MACF;AAAA,MAEA,KAAK,eAAe;AAClB,cAAM;AAAA,UACJ,OAAO,EAAC,QAAQ,KAAA;AAAA,UAChB;AAAA,QAAA,IACE;AACJ,cAAM;AAAA,UACJ,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAAA,mCACjB,MAAM,kBAAkB,IAAI,mBAAmB,MAAM;AAAA,QAAA;AAEhF;AAAA,MACF;AAAA,MAEA,KAAK,sBAAsB;AACzB,cAAM;AAAA,UACJ,UAAU,EAAC,QAAQ,MAAM,OAAO,OAAA;AAAA,UAChC;AAAA,QAAA,IACE;AACJ,cAAM,OAAO,CAAC,GAAG,OAAO,SAAS,GAAG,OAAO;AAC3C,cAAM;AAAA,UACJ,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAAA,mCACjB,MAAM,kBAAkB,KAAK,oBAAoB,IAAI,IAAI,CAAC;AAAA,QAAA;AAAA,MAEvF;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEA,yBACE,KACA,EAAC,QAAQ,MAAM,MAAA,GACf,UACA;AACA,UAAM,MAAwB,EAAC,QAAQ,OAAO,SAAA;AAC9C,WAAO;AAAA,sBACW,KAAK,KAAK,eAAe,CAAC,IAAI,IAAI,GAAG,CAAC;AAAA;AAAA,0BAElC,IAAI,GAAG,CAAC;AAAA;AAAA,EAEhC;AAAA,EAEA,0BACE,KACA,EAAC,QAAQ,MAAM,MAAA,GACf,QACA,UACA;AACA,UAAM,MAAyB,EAAC,QAAQ,OAAO,QAAQ,SAAA;AACvD,WAAO;AAAA,sBACW,KAAK,KAAK,aAAa,CAAC,IAAI,IAAI,GAAG,CAAC;AAAA;AAAA,0BAEhC,IAAI,GAAG,CAAC;AAAA;AAAA,EAEhC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,eAAe;AACnB,QAAI,KAAK,UAAU;AACjB,YAAM,EAAC,SAAS,QAAA,IAAW,SAAA;AAC3B,WAAK,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;AACtC,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,OAAO;AACL,QAAI,KAAK,UAAU;AACjB,WAAK,IAAI,OAAO,YAAY,KAAK,OAAO,MAAM,oBAAoB;AAClE,WAAK,OAAO,QAAQ,MAAM;AAAA,IAC5B;AACA,WAAO,KAAK;AAAA,EACd;AACF;AAEA,SAAS,aAAa,OAAuC;AAC3D,QAAM,EAAC,WAAW,OAAA,IAAU;AAC5B,UAAQ,OAAO,KAAA;AAAA,IACb,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,SAAS,QAAQ,EAAC,iBAAiB,UAAA,CAAU,CAAC;AAAA,IACpE,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,UAAU,QAAQ,EAAC,UAAA,CAAU,CAAC;AAAA,IACpD,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,YAAY,MAAM,CAAC;AAAA,IACzC;AACE,aAAO,CAAC,WAAW,CAAC,QAAQ,MAAM,CAAC;AAAA,EAAA;AAEzC;"}
+{"version":3,"file":"storer.js","names":["#lc","#shard","#taskID","#discoveryAddress","#discoveryProtocol","#db","#replicaVersion","#onConsumed","#onFatal","#queue","#backPressureThresholdBytes","#cdc","#approximateQueuedBytes","#running","#readyForMore","#stopped","#processQueue","#startCatchup","#trackBackfillMetadata","#maybeReleaseBackPressure","#catchup","#upsertTableMetadataStmt","#upsertColumnBackfillStmt"],"sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {getHeapStatistics} from 'node:v8';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {\n  backfillRequestSchema,\n  isDataChange,\n  isSchemaChange,\n  type BackfillID,\n  type BackfillRequest,\n  type Change,\n  type DataChange,\n  type Identifier,\n  type SchemaChange,\n  type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {type Commit} from '../change-source/protocol/current/downstream.ts';\nimport type {\n  DownstreamStatusMessage,\n  UpstreamStatusMessage,\n} from '../change-source/protocol/current/status.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type ChangeEntry} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n  AutoResetSignal,\n  markResetRequired,\n  type BackfillingColumn,\n  type ReplicationState,\n  type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n  subscriber: Subscriber;\n  mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n  | [\n      'change',\n      watermark: string,\n      json: string,\n      orig: Exclude<Change, DataChange> | null, // null for DataChanges\n    ]\n  | ['ready', callback: () => void]\n  | ['subscriber', SubscriberAndMode]\n  | DownstreamStatusMessage\n  | ['abort']\n  | 'stop';\n\ntype PendingTransaction = {\n  pool: TransactionPool;\n  preCommitWatermark: string;\n  pos: number;\n  startingReplicationState: Promise<ReplicationState>;\n  ack: boolean;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n *   `replicaVersion`, which is the version at which initial sync\n *   was performed, and any attempts to catchup from a different\n *   point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n *   a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n *   subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n  readonly id = 'storer';\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #taskID: string;\n  readonly #discoveryAddress: string;\n  readonly #discoveryProtocol: string;\n  readonly #db: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #onConsumed: (c: Commit | UpstreamStatusMessage) => void;\n  readonly #onFatal: (err: Error) => void;\n  readonly #queue = new Queue<QueueEntry>();\n  readonly #backPressureThresholdBytes: number;\n\n  #approximateQueuedBytes = 0;\n  #running = false;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    db: PostgresDB,\n    replicaVersion: string,\n    onConsumed: (c: Commit | UpstreamStatusMessage) => void,\n    onFatal: (err: Error) => void,\n    backPressureLimitHeapProportion: number,\n  ) {\n    this.#lc = lc.withContext('component', 'change-log');\n    this.#shard = shard;\n    this.#taskID = taskID;\n    this.#discoveryAddress = discoveryAddress;\n    this.#discoveryProtocol = discoveryProtocol;\n    this.#db = db;\n    this.#replicaVersion = replicaVersion;\n    this.#onConsumed = onConsumed;\n    this.#onFatal = onFatal;\n\n    const heapStats = getHeapStatistics();\n    this.#backPressureThresholdBytes =\n      (heapStats.heap_size_limit - heapStats.used_heap_size) *\n      backPressureLimitHeapProportion;\n\n    this.#lc.info?.(\n      `Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of ` +\n        `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) ` +\n        `to absorb upstream spikes`,\n      {heapStats},\n    );\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async assumeOwnership() {\n    const db = this.#db;\n    const owner = this.#taskID;\n    const ownerAddress = this.#discoveryAddress;\n    const ownerProtocol = this.#discoveryProtocol;\n    // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n    const addressWithProtocol =\n      ownerProtocol === 'ws'\n        ? ownerAddress\n        : `${ownerProtocol}://${ownerAddress}`;\n    this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n    const start = performance.now();\n    await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n    const elapsed = (performance.now() - start).toFixed(2);\n    this.#lc.info?.(\n      `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n    );\n  }\n\n  async getStartStreamInitializationParameters(): Promise<{\n    lastWatermark: string;\n    backfillRequests: BackfillRequest[];\n  }> {\n    const [[{lastWatermark}], result] = await runTx(\n      this.#db,\n      sql => [\n        sql<{lastWatermark: string}[]>`\n        SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n        // Formats a BackfillRequest using json_object_agg() to construct the\n        // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n        // to make it optional and possibly `null`.\n        sql`\n        SELECT \n            json_build_object(\n              'schema', b.\"schema\",\n              'name', b.\"table\",\n              'metadata', t.\"metadata\"\n            ) as \"table\",\n            json_object_agg(b.\"column\", b.\"backfill\") \n              as \"columns\"\n          FROM ${this.#cdc('backfilling')} as b\n          LEFT JOIN ${this.#cdc('tableMetadata')} as t\n          ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n          GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n        `,\n      ],\n      {mode: Mode.READONLY},\n    );\n\n    return {\n      lastWatermark,\n      backfillRequests: v.parse(result, backfillRequestsSchema),\n    };\n  }\n\n  async getMinWatermarkForCatchup(): Promise<string | null> {\n    const [{minWatermark}] = await this.#db<\n      {minWatermark: string | null}[]\n    > /*sql*/ `\n      SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n    return minWatermark;\n  }\n\n  purgeRecordsBefore(watermark: string): Promise<number> {\n    return runTx(this.#db, async sql => {\n      const [{deleted}] = await sql<{deleted: bigint}[]>`\n        WITH purged AS (\n          DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n            RETURNING watermark, pos\n        ) SELECT COUNT(*) as deleted FROM purged;`;\n\n      // Before committing the purge, check that this process is still the\n      // owner. This is done after the DELETE to minimize the amount of time\n      // that writes to the changeLog are delayed.\n      const [{owner}] = await sql<ReplicationState[]>`\n        SELECT * FROM ${this.#cdc('replicationState')} FOR SHARE`;\n      if (owner !== this.#taskID) {\n        throw new AbortError(\n          `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n        );\n      }\n      return Number(deleted);\n    });\n  }\n\n  /**\n   * @returns The size of the serialized entry, for memory / I/O estimations.\n   */\n  store(entry: WatermarkedChange) {\n    const [watermark, [_tag, change]] = entry;\n    // Eagerly stringify the JSON object so that the memory usage can be\n    // more accurately measured (i.e. without an extra object traversal and\n    // ad hoc memory counting heuristics).\n    //\n    // This essentially moves the stringify() computation out of the pg client,\n    // which is instead configured to pass `string` objects directly as JSON\n    // strings for JSON-valued columns (see TypeOptions.sendStringAsJson).\n    const json = BigIntJSON.stringify(change);\n    this.#approximateQueuedBytes += json.length;\n\n    this.#queue.enqueue([\n      'change',\n      watermark,\n      json,\n      isDataChange(change) ? null : change, // drop DataChanges to save memory\n    ]);\n\n    return json.length;\n  }\n\n  abort() {\n    this.#queue.enqueue(['abort']);\n  }\n\n  status(s: DownstreamStatusMessage) {\n    this.#queue.enqueue(s);\n  }\n\n  catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n    this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n  }\n\n  #readyForMore: Resolver<void> | null = null;\n\n  readyForMore(): Promise<void> | undefined {\n    if (!this.#running) {\n      return undefined;\n    }\n    if (\n      this.#readyForMore === null &&\n      this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n    ) {\n      this.#lc.warn?.(\n        `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\\n` +\n          `\\n` +\n          `To inspect changeLog backlog in your change DB:\\n` +\n          `  SELECT\\n` +\n          `    (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\\n` +\n          `    change->>'tag' AS operation,\\n` +\n          `    COUNT(*) AS count\\n` +\n          `  FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n          `  GROUP BY 1, 2\\n` +\n          `  ORDER BY 3 DESC\\n` +\n          `  LIMIT 20;`,\n      );\n      this.#readyForMore = resolver();\n    }\n    return this.#readyForMore?.promise;\n  }\n\n  #maybeReleaseBackPressure() {\n    if (\n      this.#readyForMore !== null &&\n      // Wait for at least 20% of the threshold to free up.\n      this.#approximateQueuedBytes < this.#backPressureThresholdBytes * 0.8\n    ) {\n      this.#lc.info?.(\n        `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`,\n      );\n      this.#readyForMore.resolve();\n      this.#readyForMore = null;\n    }\n  }\n\n  #stopped = promiseVoid;\n\n  /**\n   * Runs the storer loop until {@link stop()} is called, or an error is thrown.\n   * Once {@link run()} completes, it can be called again.\n   */\n  async run() {\n    assert(!this.#running, `storer is already running`);\n\n    const {promise: stopped, resolve: signalStopped} = resolver();\n    this.#running = true;\n    this.#stopped = stopped;\n\n    this.#lc.info?.('starting storer');\n    try {\n      await this.#processQueue();\n    } finally {\n      // Release any pending backpressure so the upstream can proceed\n      if (this.#readyForMore !== null) {\n        this.#readyForMore.resolve();\n        this.#readyForMore = null;\n      }\n      const unprocessed = this.#queue.drain();\n      if (unprocessed.length) {\n        this.#lc.warn?.(\n          `dropped ${unprocessed.length} entries from the changeLog queue`,\n        );\n      }\n      this.#running = false;\n      signalStopped();\n      this.#lc.info?.('storer stopped');\n    }\n  }\n\n  async #processQueue() {\n    let tx: PendingTransaction | null = null;\n    let msg: QueueEntry | false;\n\n    const catchupQueue: SubscriberAndMode[] = [];\n    while ((msg = await this.#queue.dequeue()) !== 'stop') {\n      const [msgType] = msg;\n      switch (msgType) {\n        case 'ready': {\n          const signalReady = msg[1];\n          signalReady();\n          continue;\n        }\n        case 'subscriber': {\n          const subscriber = msg[1];\n          if (tx) {\n            catchupQueue.push(subscriber); // Wait for the current tx to complete.\n          } else {\n            await this.#startCatchup([subscriber]); // Catch up immediately.\n          }\n          continue;\n        }\n        case 'status':\n          this.#onConsumed(msg);\n          continue;\n        case 'abort': {\n          if (tx) {\n            tx.pool.abort();\n            await tx.pool.done();\n            tx = null;\n          }\n          continue;\n        }\n      }\n      // msgType === 'change'\n      const [_, watermark, json, change] = msg;\n      const tag = change?.tag;\n      this.#approximateQueuedBytes -= json.length;\n\n      if (tag === 'begin') {\n        assert(!tx, 'received BEGIN in the middle of a transaction');\n        const {promise, resolve, reject} = resolver<ReplicationState>();\n        tx = {\n          pool: new TransactionPool(\n            this.#lc.withContext('watermark', watermark),\n            Mode.READ_COMMITTED,\n          ),\n          preCommitWatermark: watermark,\n          pos: 0,\n          startingReplicationState: promise,\n          ack: !change.skipAck,\n        };\n        tx.pool.run(this.#db);\n        // Acquire a lock on the replicationState row to detect and/or prevent\n        // a concurrent ownership change.\n        void tx.pool.process(tx => {\n          tx<ReplicationState[]> /*sql*/ `\n          SELECT * FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n            ([result]) => resolve(result),\n            reject,\n          );\n          return [];\n        });\n      } else {\n        assert(tx, () => `received change outside of transaction: ${json}`);\n        tx.pos++;\n      }\n\n      const entry = {\n        watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n        precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n        pos: tx.pos,\n        change: json,\n      };\n\n      const processed = tx.pool.process(sql => [\n        sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n        ...(change !== null && isSchemaChange(change)\n          ? this.#trackBackfillMetadata(sql, change)\n          : []),\n      ]);\n\n      if (tx.pos % 100 === 0) {\n        // Backpressure is exerted on commit when awaiting tx.pool.done().\n        // However, backpressure checks need to be regularly done for\n        // very large transactions in order to avoid memory blowup.\n        await processed;\n      }\n      this.#maybeReleaseBackPressure();\n\n      if (tag === 'commit') {\n        const {owner} = await tx.startingReplicationState;\n        if (owner !== this.#taskID) {\n          // Ownership change reflected in the replicationState read in 'begin'.\n          tx.pool.fail(\n            new AbortError(`changeLog ownership has been assumed by ${owner}`),\n          );\n        } else {\n          // Update the replication state.\n          const lastWatermark = watermark;\n          void tx.pool.process(tx => [\n            tx`\n            UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n          ]);\n          tx.pool.setDone();\n        }\n\n        await tx.pool.done();\n\n        // ACK the LSN to the upstream Postgres.\n        if (tx.ack) {\n          this.#onConsumed(['commit', change, {watermark}]);\n        }\n        tx = null;\n\n        // Before beginning the next transaction, open a READONLY snapshot to\n        // concurrently catchup any queued subscribers.\n        await this.#startCatchup(catchupQueue.splice(0));\n      } else if (tag === 'rollback') {\n        // Aborted transactions are not stored in the changeLog. Abort the current tx\n        // and process catchup of subscribers that were waiting for it to end.\n        tx.pool.abort();\n        await tx.pool.done();\n        tx = null;\n\n        await this.#startCatchup(catchupQueue.splice(0));\n      }\n    }\n  }\n\n  async #startCatchup(subs: SubscriberAndMode[]) {\n    if (subs.length === 0) {\n      return;\n    }\n\n    const reader = new TransactionPool(\n      this.#lc.withContext('pool', 'catchup'),\n      Mode.READONLY,\n    );\n    reader.run(this.#db);\n\n    // Ensure that the transaction has started (and is thus holding a snapshot\n    // of the database) before continuing on to commit more changes. This is\n    // done by performing a single read on the db, which determines the\n    // snapshot for the REPEATABLE_READ transaction.\n    const [{lastWatermark}] = await reader.processReadTask(\n      sql => sql<ReplicationState[]>`\n        SELECT * FROM ${this.#cdc('replicationState')}\n      `,\n    );\n\n    // Run the actual catchup queries in the background. Errors are handled in\n    // #catchup() by disconnecting the associated subscriber.\n    void Promise.all(\n      subs.map(sub => this.#catchup(sub, lastWatermark, reader)),\n    ).finally(() => reader.setDone());\n  }\n\n  async #catchup(\n    {subscriber: sub, mode}: SubscriberAndMode,\n    lastWatermark: string,\n    reader: TransactionPool,\n  ) {\n    try {\n      await reader.processReadTask(async tx => {\n        const start = Date.now();\n\n        // When starting from initial-sync, there won't be a change with a watermark\n        // equal to the replica version. This is the empty changeLog scenario.\n        let watermarkFound = sub.watermark === this.#replicaVersion;\n        let count = 0;\n        let lastBatchConsumed: Promise<unknown> | undefined;\n\n        for await (const entries of tx<ChangeEntry[]> /*sql*/ `\n          SELECT watermark, change FROM ${this.#cdc('changeLog')}\n           WHERE watermark >= ${sub.watermark}\n             AND watermark <= ${lastWatermark}\n           ORDER BY watermark, pos`.cursor(2000)) {\n          // Wait for the last batch of entries to be consumed by the\n          // subscriber before sending down the current batch. This pipelining\n          // allows one batch of changes to be received from the change-db\n          // while the previous batch of changes are sent to the subscriber,\n          // resulting in flow control that caps the number of changes\n          // referenced in memory to 2 * batch-size.\n          const start = performance.now();\n          await lastBatchConsumed;\n          const elapsed = performance.now() - start;\n          if (lastBatchConsumed) {\n            (elapsed > 100 ? this.#lc.info : this.#lc.debug)?.(\n              `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n            );\n          }\n\n          for (const entry of entries) {\n            if (entry.watermark === sub.watermark) {\n              // This should be the first entry.\n              // Catchup starts from *after* the watermark.\n              watermarkFound = true;\n            } else if (watermarkFound) {\n              lastBatchConsumed = sub.catchup(toDownstream(entry));\n              count++;\n            } else if (mode === 'backup') {\n              throw new AutoResetSignal(\n                `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n              );\n            } else {\n              this.#lc.warn?.(\n                `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n              );\n              sub.close(\n                ErrorType.WatermarkTooOld,\n                `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n              );\n              return;\n            }\n          }\n        }\n        if (watermarkFound) {\n          await lastBatchConsumed;\n          this.#lc.info?.(\n            `caught up ${sub.id} with ${count} changes (${\n              Date.now() - start\n            } ms)`,\n          );\n        } else {\n          this.#lc.warn?.(\n            `subscriber at watermark ${sub.watermark} is ahead of latest watermark`,\n          );\n        }\n        // Flushes the backlog of messages buffered during catchup and\n        // allows the subscription to forward subsequent messages immediately.\n        sub.setCaughtUp();\n      });\n    } catch (err) {\n      this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n      if (err instanceof AutoResetSignal) {\n        await markResetRequired(this.#db, this.#shard);\n        this.#onFatal(err);\n      }\n      sub.fail(err);\n    }\n  }\n\n  /**\n   * Returns the db statements necessary to track backfill and table metadata\n   * presented in the `change`, if any.\n   */\n  #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n    const stmts: PendingQuery<Row[]>[] = [];\n\n    switch (change.tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        stmts.push(this.#upsertTableMetadataStmt(sql, table, metadata));\n        break;\n      }\n\n      case 'create-table': {\n        const {spec, metadata, backfill} = change;\n        if (metadata) {\n          stmts.push(this.#upsertTableMetadataStmt(sql, spec, metadata));\n        }\n        if (backfill) {\n          Object.entries(backfill).forEach(([col, backfill]) => {\n            stmts.push(\n              this.#upsertColumnBackfillStmt(sql, spec, col, backfill),\n            );\n          });\n        }\n        break;\n      }\n\n      case 'rename-table': {\n        const {old} = change;\n        const row = {schema: change.new.schema, table: change.new.name};\n        stmts.push(\n          sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n          sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n        );\n        break;\n      }\n\n      case 'drop-table': {\n        const {\n          id: {schema, name},\n        } = change;\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('tableMetadata')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n        );\n        break;\n      }\n\n      case 'add-column': {\n        const {table, tableMetadata, column, backfill} = change;\n        if (tableMetadata) {\n          stmts.push(this.#upsertTableMetadataStmt(sql, table, tableMetadata));\n        }\n        if (backfill) {\n          stmts.push(\n            this.#upsertColumnBackfillStmt(sql, table, column.name, backfill),\n          );\n        }\n        break;\n      }\n\n      case 'update-column': {\n        const {\n          table: {schema, name: table},\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          stmts.push(\n            sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${newName}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${oldName}`,\n          );\n        }\n        break;\n      }\n\n      case 'drop-column': {\n        const {\n          table: {schema, name},\n          column,\n        } = change;\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${column}`,\n        );\n        break;\n      }\n\n      case 'backfill-completed': {\n        const {\n          relation: {schema, name: table, rowKey},\n          columns,\n        } = change;\n        const cols = [...rowKey.columns, ...columns];\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql(cols)}`,\n        );\n      }\n    }\n    return stmts;\n  }\n\n  #upsertTableMetadataStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    metadata: TableMetadata,\n  ) {\n    const row: TableMetadataRow = {schema, table, metadata};\n    return sql`\n        INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\") \n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  #upsertColumnBackfillStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    column: string,\n    backfill: BackfillID,\n  ) {\n    const row: BackfillingColumn = {schema, table, column, backfill};\n    return sql`\n        INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\", \"column\") \n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  /**\n   * Waits until all currently queued entries have been processed.\n   * This is only used in tests.\n   */\n  async allProcessed() {\n    if (this.#running) {\n      const {promise, resolve} = resolver();\n      this.#queue.enqueue(['ready', resolve]);\n      await promise;\n    }\n  }\n\n  stop() {\n    if (this.#running) {\n      this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n      this.#queue.enqueue('stop');\n    }\n    return this.#stopped;\n  }\n}\n\nfunction toDownstream(entry: ChangeEntry): WatermarkedChange {\n  const {watermark, change} = entry;\n  switch (change.tag) {\n    case 'begin':\n      return [watermark, ['begin', change, {commitWatermark: watermark}]];\n    case 'commit':\n      return [watermark, ['commit', change, {watermark}]];\n    case 'rollback':\n      return [watermark, ['rollback', change]];\n    default:\n      return [watermark, ['data', change]];\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAwEA,IAAM,yBAAyB,eAAE,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgC7D,IAAa,SAAb,MAAuC;CACrC,KAAc;CACd;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,OAAmB;CACzC;CAEA,0BAA0B;CAC1B,WAAW;CAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,YACA,SACA,iCACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,aAAa;AACpD,QAAA,QAAc;AACd,QAAA,SAAe;AACf,QAAA,mBAAyB;AACzB,QAAA,oBAA0B;AAC1B,QAAA,KAAW;AACX,QAAA,iBAAuB;AACvB,QAAA,aAAmB;AACnB,QAAA,UAAgB;EAEhB,MAAM,YAAY,mBAAmB;AACrC,QAAA,8BACG,UAAU,kBAAkB,UAAU,kBACvC;AAEF,QAAA,GAAS,OACP,gBAAgB,MAAA,6BAAmC,QAAQ,GAAG,QAAQ,EAAE,CAAC,iCAC5C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,EAAE,CAAC,iCAE/E,EAAC,WAAU,CACZ;;CAIH,KAAK,OAAe;AAClB,SAAO,MAAA,GAAS,GAAG,UAAU,MAAA,MAAY,CAAC,GAAG,QAAQ;;CAGvD,MAAM,kBAAkB;EACtB,MAAM,KAAK,MAAA;EACX,MAAM,QAAQ,MAAA;EACd,MAAM,eAAe,MAAA;EACrB,MAAM,gBAAgB,MAAA;EAEtB,MAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,cAAc,KAAK;AAC5B,QAAA,GAAS,OAAO,yBAAyB,sBAAsB;EAC/D,MAAM,QAAQ,YAAY,KAAK;AAC/B,QAAM,EAAE,UAAU,MAAA,IAAU,mBAAmB,CAAC,OAAO,GAAG;GAAC;GAAO,cAAc;GAAoB,CAAC;EACrG,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,QAAA,GAAS,OACP,wBAAwB,oBAAoB,IAAI,QAAQ,MACzD;;CAGH,MAAM,yCAGH;EACD,MAAM,CAAC,CAAC,EAAC,kBAAiB,UAAU,MAAM,MACxC,MAAA,KACA,QAAO,CACL,GAA8B;sCACA,MAAA,IAAU,mBAAmB,IAK3D,GAAG;;;;;;;;;iBASM,MAAA,IAAU,cAAc,CAAC;sBACpB,MAAA,IAAU,gBAAgB,CAAC;;;UAI1C,EACD,EAAC,MAAM,UAAc,CACtB;AAED,SAAO;GACL;GACA,kBAAkB,MAAQ,QAAQ,uBAAuB;GAC1D;;CAGH,MAAM,4BAAoD;EACxD,MAAM,CAAC,EAAC,kBAAiB,MAAM,MAAA,EAErB;qDACuC,MAAA,IAAU,YAAY;AACvE,SAAO;;CAGT,mBAAmB,WAAoC;AACrD,SAAO,MAAM,MAAA,IAAU,OAAM,QAAO;GAClC,MAAM,CAAC,EAAC,aAAY,MAAM,GAAwB;;wBAEhC,MAAA,IAAU,YAAY,CAAC,qBAAqB,UAAU;;;GAOxE,MAAM,CAAC,EAAC,WAAU,MAAM,GAAuB;wBAC7B,MAAA,IAAU,mBAAmB,CAAC;AAChD,OAAI,UAAU,MAAA,OACZ,OAAM,IAAI,WACR,+BAA+B,UAAU,uCAAuC,QACjF;AAEH,UAAO,OAAO,QAAQ;IACtB;;;;;CAMJ,MAAM,OAA0B;EAC9B,MAAM,CAAC,WAAW,CAAC,MAAM,WAAW;EAQpC,MAAM,OAAO,WAAW,UAAU,OAAO;AACzC,QAAA,0BAAgC,KAAK;AAErC,QAAA,MAAY,QAAQ;GAClB;GACA;GACA;GACA,aAAa,OAAO,GAAG,OAAO;GAC/B,CAAC;AAEF,SAAO,KAAK;;CAGd,QAAQ;AACN,QAAA,MAAY,QAAQ,CAAC,QAAQ,CAAC;;CAGhC,OAAO,GAA4B;AACjC,QAAA,MAAY,QAAQ,EAAE;;CAGxB,QAAQ,YAAwB,MAAsB;AACpD,QAAA,MAAY,QAAQ,CAAC,cAAc;GAAC;GAAY;GAAK,CAAC,CAAC;;CAGzD,gBAAuC;CAEvC,eAA0C;AACxC,MAAI,CAAC,MAAA,QACH;AAEF,MACE,MAAA,iBAAuB,QACvB,MAAA,yBAA+B,MAAA,4BAC/B;AACA,SAAA,GAAS,OACP,+BAA+B,MAAA,MAAY,MAAM,CAAC,qBAAqB,MAAA,yBAA+B,QAAQ,GAAG,QAAQ,EAAE,CAAC,4SAW7H;AACD,SAAA,eAAqB,UAAU;;AAEjC,SAAO,MAAA,cAAoB;;CAG7B,4BAA4B;AAC1B,MACE,MAAA,iBAAuB,QAEvB,MAAA,yBAA+B,MAAA,6BAAmC,IAClE;AACA,SAAA,GAAS,OACP,gCAAgC,MAAA,MAAY,MAAM,CAAC,qBAAqB,MAAA,yBAA+B,QAAQ,GAAG,QAAQ,EAAE,CAAC,MAC9H;AACD,SAAA,aAAmB,SAAS;AAC5B,SAAA,eAAqB;;;CAIzB,WAAW;;;;;CAMX,MAAM,MAAM;AACV,SAAO,CAAC,MAAA,SAAe,4BAA4B;EAEnD,MAAM,EAAC,SAAS,SAAS,SAAS,kBAAiB,UAAU;AAC7D,QAAA,UAAgB;AAChB,QAAA,UAAgB;AAEhB,QAAA,GAAS,OAAO,kBAAkB;AAClC,MAAI;AACF,SAAM,MAAA,cAAoB;YAClB;AAER,OAAI,MAAA,iBAAuB,MAAM;AAC/B,UAAA,aAAmB,SAAS;AAC5B,UAAA,eAAqB;;GAEvB,MAAM,cAAc,MAAA,MAAY,OAAO;AACvC,OAAI,YAAY,OACd,OAAA,GAAS,OACP,WAAW,YAAY,OAAO,mCAC/B;AAEH,SAAA,UAAgB;AAChB,kBAAe;AACf,SAAA,GAAS,OAAO,iBAAiB;;;CAIrC,OAAA,eAAsB;EACpB,IAAI,KAAgC;EACpC,IAAI;EAEJ,MAAM,eAAoC,EAAE;AAC5C,UAAQ,MAAM,MAAM,MAAA,MAAY,SAAS,MAAM,QAAQ;GACrD,MAAM,CAAC,WAAW;AAClB,WAAQ,SAAR;IACE,KAAK,SAAS;KACZ,MAAM,cAAc,IAAI;AACxB,kBAAa;AACb;;IAEF,KAAK,cAAc;KACjB,MAAM,aAAa,IAAI;AACvB,SAAI,GACF,cAAa,KAAK,WAAW;SAE7B,OAAM,MAAA,aAAmB,CAAC,WAAW,CAAC;AAExC;;IAEF,KAAK;AACH,WAAA,WAAiB,IAAI;AACrB;IACF,KAAK;AACH,SAAI,IAAI;AACN,SAAG,KAAK,OAAO;AACf,YAAM,GAAG,KAAK,MAAM;AACpB,WAAK;;AAEP;;GAIJ,MAAM,CAAC,GAAG,WAAW,MAAM,UAAU;GACrC,MAAM,MAAM,QAAQ;AACpB,SAAA,0BAAgC,KAAK;AAErC,OAAI,QAAQ,SAAS;AACnB,WAAO,CAAC,IAAI,gDAAgD;IAC5D,MAAM,EAAC,SAAS,SAAS,WAAU,UAA4B;AAC/D,SAAK;KACH,MAAM,IAAI,gBACR,MAAA,GAAS,YAAY,aAAa,UAAU,EAC5C,eACD;KACD,oBAAoB;KACpB,KAAK;KACL,0BAA0B;KAC1B,KAAK,CAAC,OAAO;KACd;AACD,OAAG,KAAK,IAAI,MAAA,GAAS;AAGhB,OAAG,KAAK,SAAQ,OAAM;AACzB,OAA+B;0BACf,MAAA,IAAU,mBAAmB,CAAC,aAAa,MACxD,CAAC,YAAY,QAAQ,OAAO,EAC7B,OACD;AACD,YAAO,EAAE;MACT;UACG;AACL,WAAO,UAAU,2CAA2C,OAAO;AACnE,OAAG;;GAGL,MAAM,QAAQ;IACZ,WAAW,QAAQ,WAAW,YAAY,GAAG;IAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;IACtD,KAAK,GAAG;IACR,QAAQ;IACT;GAED,MAAM,YAAY,GAAG,KAAK,SAAQ,QAAO,CACvC,GAAG,eAAe,MAAA,IAAU,YAAY,CAAC,GAAG,IAAI,MAAM,IACtD,GAAI,WAAW,QAAQ,eAAe,OAAO,GACzC,MAAA,sBAA4B,KAAK,OAAO,GACxC,EAAE,CACP,CAAC;AAEF,OAAI,GAAG,MAAM,QAAQ,EAInB,OAAM;AAER,SAAA,0BAAgC;AAEhC,OAAI,QAAQ,UAAU;IACpB,MAAM,EAAC,UAAS,MAAM,GAAG;AACzB,QAAI,UAAU,MAAA,OAEZ,IAAG,KAAK,KACN,IAAI,WAAW,2CAA2C,QAAQ,CACnE;SACI;KAEL,MAAM,gBAAgB;AACjB,QAAG,KAAK,SAAQ,OAAM,CACzB,EAAE;qBACO,MAAA,IAAU,mBAAmB,CAAC,OAAO,GAAG,EAAC,eAAc,CAAC,GAClE,CAAC;AACF,QAAG,KAAK,SAAS;;AAGnB,UAAM,GAAG,KAAK,MAAM;AAGpB,QAAI,GAAG,IACL,OAAA,WAAiB;KAAC;KAAU;KAAQ,EAAC,WAAU;KAAC,CAAC;AAEnD,SAAK;AAIL,UAAM,MAAA,aAAmB,aAAa,OAAO,EAAE,CAAC;cACvC,QAAQ,YAAY;AAG7B,OAAG,KAAK,OAAO;AACf,UAAM,GAAG,KAAK,MAAM;AACpB,SAAK;AAEL,UAAM,MAAA,aAAmB,aAAa,OAAO,EAAE,CAAC;;;;CAKtD,OAAA,aAAoB,MAA2B;AAC7C,MAAI,KAAK,WAAW,EAClB;EAGF,MAAM,SAAS,IAAI,gBACjB,MAAA,GAAS,YAAY,QAAQ,UAAU,EACvC,SACD;AACD,SAAO,IAAI,MAAA,GAAS;EAMpB,MAAM,CAAC,EAAC,mBAAkB,MAAM,OAAO,iBACrC,QAAO,GAAuB;wBACZ,MAAA,IAAU,mBAAmB,CAAC;QAEjD;AAII,UAAQ,IACX,KAAK,KAAI,QAAO,MAAA,QAAc,KAAK,eAAe,OAAO,CAAC,CAC3D,CAAC,cAAc,OAAO,SAAS,CAAC;;CAGnC,OAAA,QACE,EAAC,YAAY,KAAK,QAClB,eACA,QACA;AACA,MAAI;AACF,SAAM,OAAO,gBAAgB,OAAM,OAAM;IACvC,MAAM,QAAQ,KAAK,KAAK;IAIxB,IAAI,iBAAiB,IAAI,cAAc,MAAA;IACvC,IAAI,QAAQ;IACZ,IAAI;AAEJ,eAAW,MAAM,WAAW,EAA0B;0CACpB,MAAA,IAAU,YAAY,CAAC;gCACjC,IAAI,UAAU;gCACd,cAAc;oCACV,OAAO,IAAK,EAAE;KAOxC,MAAM,QAAQ,YAAY,KAAK;AAC/B,WAAM;KACN,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,SAAI,kBACF,EAAC,UAAU,MAAM,MAAA,GAAS,OAAO,MAAA,GAAS,SACxC,UAAU,QAAQ,QAAQ,EAAE,CAAC,UAAU,IAAI,GAAG,2CAC/C;AAGH,UAAK,MAAM,SAAS,QAClB,KAAI,MAAM,cAAc,IAAI,UAG1B,kBAAiB;cACR,gBAAgB;AACzB,0BAAoB,IAAI,QAAQ,aAAa,MAAM,CAAC;AACpD;gBACS,SAAS,SAClB,OAAM,IAAI,gBACR,+BAA+B,IAAI,UAAU,wBAAwB,MAAM,UAAU,GACtF;UACI;AACL,YAAA,GAAS,OACP,qCAAqC,IAAI,UAAU,wBAAwB,MAAM,UAAU,GAC5F;AACD,UAAI,MACF,GACA,mCAAmC,MAAM,UAAU,cAAc,IAAI,UAAU,GAChF;AACD;;;AAIN,QAAI,gBAAgB;AAClB,WAAM;AACN,WAAA,GAAS,OACP,aAAa,IAAI,GAAG,QAAQ,MAAM,YAChC,KAAK,KAAK,GAAG,MACd,MACF;UAED,OAAA,GAAS,OACP,2BAA2B,IAAI,UAAU,+BAC1C;AAIH,QAAI,aAAa;KACjB;WACK,KAAK;AACZ,SAAA,GAAS,QAAQ,sCAAsC,IAAI,MAAM,IAAI;AACrE,OAAI,eAAe,iBAAiB;AAClC,UAAM,kBAAkB,MAAA,IAAU,MAAA,MAAY;AAC9C,UAAA,QAAc,IAAI;;AAEpB,OAAI,KAAK,IAAI;;;;;;;CAQjB,uBAAuB,KAA0B,QAAsB;EACrE,MAAM,QAA+B,EAAE;AAEvC,UAAQ,OAAO,KAAf;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;AAC/B,UAAM,KAAK,MAAA,wBAA8B,KAAK,OAAO,SAAS,CAAC;AAC/D;;GAGF,KAAK,gBAAgB;IACnB,MAAM,EAAC,MAAM,UAAU,aAAY;AACnC,QAAI,SACF,OAAM,KAAK,MAAA,wBAA8B,KAAK,MAAM,SAAS,CAAC;AAEhE,QAAI,SACF,QAAO,QAAQ,SAAS,CAAC,SAAS,CAAC,KAAK,cAAc;AACpD,WAAM,KACJ,MAAA,yBAA+B,KAAK,MAAM,KAAK,SAAS,CACzD;MACD;AAEJ;;GAGF,KAAK,gBAAgB;IACnB,MAAM,EAAC,QAAO;IACd,MAAM,MAAM;KAAC,QAAQ,OAAO,IAAI;KAAQ,OAAO,OAAO,IAAI;KAAK;AAC/D,UAAM,KACJ,GAAG,UAAU,MAAA,IAAU,gBAAgB,CAAC,OAAO,IAAI,IAAI,CAAC;mCAC/B,IAAI,OAAO,iBAAiB,IAAI,QACzD,GAAG,UAAU,MAAA,IAAU,cAAc,CAAC,OAAO,IAAI,IAAI,CAAC;mCAC7B,IAAI,OAAO,iBAAiB,IAAI,OAC1D;AACD;;GAGF,KAAK,cAAc;IACjB,MAAM,EACJ,IAAI,EAAC,QAAQ,WACX;AACJ,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,gBAAgB,CAAC;mCACpB,OAAO,iBAAiB,QACjD,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,OAClD;AACD;;GAGF,KAAK,cAAc;IACjB,MAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;AACjD,QAAI,cACF,OAAM,KAAK,MAAA,wBAA8B,KAAK,OAAO,cAAc,CAAC;AAEtE,QAAI,SACF,OAAM,KACJ,MAAA,yBAA+B,KAAK,OAAO,OAAO,MAAM,SAAS,CAClE;AAEH;;GAGF,KAAK,iBAAiB;IACpB,MAAM,EACJ,OAAO,EAAC,QAAQ,MAAM,SACtB,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;AACJ,QAAI,YAAY,QACd,OAAM,KACJ,GAAG,UAAU,MAAA,IAAU,cAAc,CAAC,kBAAkB,QAAQ;mCACzC,OAAO,iBAAiB,MAAM,kBAAkB,UACxE;AAEH;;GAGF,KAAK,eAAe;IAClB,MAAM,EACJ,OAAO,EAAC,QAAQ,QAChB,WACE;AACJ,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,KAAK,kBAAkB,SACzE;AACD;;GAGF,KAAK,sBAAsB;IACzB,MAAM,EACJ,UAAU,EAAC,QAAQ,MAAM,OAAO,UAChC,YACE;IACJ,MAAM,OAAO,CAAC,GAAG,OAAO,SAAS,GAAG,QAAQ;AAC5C,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,MAAM,mBAAmB,IAAI,KAAK,GACpF;;;AAGL,SAAO;;CAGT,yBACE,KACA,EAAC,QAAQ,MAAM,SACf,UACA;EACA,MAAM,MAAwB;GAAC;GAAQ;GAAO;GAAS;AACvD,SAAO,GAAG;sBACQ,MAAA,IAAU,gBAAgB,CAAC,GAAG,IAAI,IAAI,CAAC;;0BAEnC,IAAI,IAAI,CAAC;;;CAIjC,0BACE,KACA,EAAC,QAAQ,MAAM,SACf,QACA,UACA;EACA,MAAM,MAAyB;GAAC;GAAQ;GAAO;GAAQ;GAAS;AAChE,SAAO,GAAG;sBACQ,MAAA,IAAU,cAAc,CAAC,GAAG,IAAI,IAAI,CAAC;;0BAEjC,IAAI,IAAI,CAAC;;;;;;;CAQjC,MAAM,eAAe;AACnB,MAAI,MAAA,SAAe;GACjB,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,SAAA,MAAY,QAAQ,CAAC,SAAS,QAAQ,CAAC;AACvC,SAAM;;;CAIV,OAAO;AACL,MAAI,MAAA,SAAe;AACjB,SAAA,GAAS,OAAO,YAAY,MAAA,MAAY,MAAM,CAAC,oBAAoB;AACnE,SAAA,MAAY,QAAQ,OAAO;;AAE7B,SAAO,MAAA;;;AAIX,SAAS,aAAa,OAAuC;CAC3D,MAAM,EAAC,WAAW,WAAU;AAC5B,SAAQ,OAAO,KAAf;EACE,KAAK,QACH,QAAO,CAAC,WAAW;GAAC;GAAS;GAAQ,EAAC,iBAAiB,WAAU;GAAC,CAAC;EACrE,KAAK,SACH,QAAO,CAAC,WAAW;GAAC;GAAU;GAAQ,EAAC,WAAU;GAAC,CAAC;EACrD,KAAK,WACH,QAAO,CAAC,WAAW,CAAC,YAAY,OAAO,CAAC;EAC1C,QACE,QAAO,CAAC,WAAW,CAAC,QAAQ,OAAO,CAAC"}

package/out/zero-cache/src/services/change-streamer/subscriber.d.ts

@@ -26,6 +26,24 @@ export declare class Subscriber {
      * entries that were received during the catchup.
      */
     setCaughtUp(): void;
+    /**
+     * The number of downstream messages that have yet to be acked.
+     */
+    get numPending(): number;
+    /**
+     * The total number of downstream messages that the subscriber has
+     * processed (i.e. acked).
+     */
+    get numProcessed(): number;
+    /**
+     * Records a new history entry for the number of messages processed,
+     * keeping the number of samples bounded to `maxSamples`.
+     */
+    sampleProcessRate(now: number, maxSamples?: number): this;
+    getStats(): {
+        processRate: number;
+        pending: number;
+    };
     supportsMessage(change: ChangeStreamData[1]): boolean;
     fail(err?: unknown): void;
     close(error?: ErrorType, message?: string): void;

package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"subscriber.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AAEzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,sCAAsC,CAAC;AAC3E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAElD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC;;;;;;GAMG;AACH,qBAAa,UAAU;;IAErB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAOlB,eAAe,EAAE,MAAM,EACvB,EAAE,EAAE,MAAM,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,YAAY,CAAC,UAAU,CAAC;IAUtC,IAAI,SAAS,WAEZ;IAED,IAAI,KAAK,WAER;IAEK,IAAI,CAAC,MAAM,EAAE,iBAAiB;IAoBpC,kEAAkE;IAC5D,OAAO,CAAC,MAAM,EAAE,iBAAiB;IAKvC;;;OAGG;IACH,WAAW;IAoCX,eAAe,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAS3C,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO;IAIlB,KAAK,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,MAAM;CAS1C"}
+{"version":3,"file":"subscriber.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AAGzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,sCAAsC,CAAC;AAC3E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAElD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC;;;;;;GAMG;AACH,qBAAa,UAAU;;IAErB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAOlB,eAAe,EAAE,MAAM,EACvB,EAAE,EAAE,MAAM,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,YAAY,CAAC,UAAU,CAAC;IAUtC,IAAI,SAAS,WAEZ;IAED,IAAI,KAAK,WAER;IAEK,IAAI,CAAC,MAAM,EAAE,iBAAiB;IAoBpC,kEAAkE;IAC5D,OAAO,CAAC,MAAM,EAAE,iBAAiB;IAKvC;;;OAGG;IACH,WAAW;IA0DX;;OAEG;IACH,IAAI,UAAU,WAEb;IAED;;;OAGG;IACH,IAAI,YAAY,WAEf;IAED;;;OAGG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,SAAK,GAAG,IAAI;IAQrD,QAAQ,IAAI;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAC;IAalD,eAAe,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAS3C,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO;IAIlB,KAAK,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,MAAM;CAS1C"}

package/out/zero-cache/src/services/change-streamer/subscriber.js

@@ -1,104 +1,147 @@
 import { assert } from "../../../../shared/src/asserts.js";
+import { must } from "../../../../shared/src/must.js";
 import { max } from "../../types/lexi-version.js";
 import "./change-streamer.js";
-import { Unknown } from "./error-type-enum.js";
-class Subscriber {
-  #protocolVersion;
-  id;
-  #downstream;
-  #watermark;
-  #acked;
-  #backlog;
-  constructor(protocolVersion, id, watermark, downstream) {
-    this.#protocolVersion = protocolVersion;
-    this.id = id;
-    this.#downstream = downstream;
-    this.#watermark = watermark;
-    this.#acked = watermark;
-    this.#backlog = [];
-  }
-  get watermark() {
-    return this.#watermark;
-  }
-  get acked() {
-    return this.#acked;
-  }
-  async send(change) {
-    const [watermark] = change;
-    if (watermark > this.#watermark) {
-      if (this.#backlog) {
-        this.#backlog.push(change);
-      } else {
-        await this.#send(change);
-      }
-    }
-  }
-  #initialStatusSent = false;
-  #ensureInitialStatusSent() {
-    if (this.#protocolVersion >= 2 && !this.#initialStatusSent) {
-      this.#downstream.push(["status", { tag: "status" }]);
-      this.#initialStatusSent = true;
-    }
-  }
-  /** catchup() is called on ChangeEntries loaded from the store. */
-  async catchup(change) {
-    this.#ensureInitialStatusSent();
-    await this.#send(change);
-  }
-  /**
-   * Marks the Subscribe as "caught up" and flushes any backlog of
-   * entries that were received during the catchup.
-   */
-  setCaughtUp() {
-    this.#ensureInitialStatusSent();
-    assert(
-      this.#backlog,
-      "setCaughtUp() called but subscriber is not in catchup mode"
-    );
-    for (const change of this.#backlog) {
-      void this.#send(change);
-    }
-    this.#backlog = null;
-  }
-  async #send(change) {
-    const [watermark, downstream] = change;
-    if (watermark <= this.watermark) {
-      return;
-    }
-    if (!this.supportsMessage(downstream[1])) {
-      return;
-    }
-    const pending = this.#downstream.push(downstream);
-    if (downstream[0] === "commit") {
-      this.#watermark = watermark;
-      void pending.result.then((val) => {
-        if (val === "consumed") {
-          this.#acked = max(this.#acked, watermark);
-        }
-      });
-    }
-    await pending.result;
-  }
-  supportsMessage(change) {
-    switch (change.tag) {
-      case "update-table-metadata":
-        return this.#protocolVersion >= 5;
-    }
-    return true;
-  }
-  fail(err) {
-    this.close(Unknown, String(err));
-  }
-  close(error, message) {
-    if (error) {
-      const { result } = this.#downstream.push(["error", { type: error, message }]);
-      void result.then(() => this.#downstream.cancel());
-    } else {
-      this.#downstream.cancel();
-    }
-  }
-}
-export {
-  Subscriber
+//#region ../zero-cache/src/services/change-streamer/subscriber.ts
+/**
+* Encapsulates a subscriber to changes. All subscribers start in a
+* "catchup" phase in which changes are buffered in a backlog while the
+* storer is queried to send any changes that were committed since the
+* subscriber's watermark. Once the catchup is complete, calls to
+* {@link send()} result in immediately sending the change.
+*/
+var Subscriber = class {
+	#protocolVersion;
+	id;
+	#downstream;
+	#watermark;
+	#acked;
+	#backlog;
+	constructor(protocolVersion, id, watermark, downstream) {
+		this.#protocolVersion = protocolVersion;
+		this.id = id;
+		this.#downstream = downstream;
+		this.#watermark = watermark;
+		this.#acked = watermark;
+		this.#backlog = [];
+	}
+	get watermark() {
+		return this.#watermark;
+	}
+	get acked() {
+		return this.#acked;
+	}
+	async send(change) {
+		const [watermark] = change;
+		if (watermark > this.#watermark) if (this.#backlog) this.#backlog.push(change);
+		else await this.#sendChange(change);
+	}
+	#initialStatusSent = false;
+	#ensureInitialStatusSent() {
+		if (this.#protocolVersion >= 2 && !this.#initialStatusSent) {
+			this.#sendDownstream(["status", { tag: "status" }]);
+			this.#initialStatusSent = true;
+		}
+	}
+	/** catchup() is called on ChangeEntries loaded from the store. */
+	async catchup(change) {
+		this.#ensureInitialStatusSent();
+		await this.#sendChange(change);
+	}
+	/**
+	* Marks the Subscribe as "caught up" and flushes any backlog of
+	* entries that were received during the catchup.
+	*/
+	setCaughtUp() {
+		this.#ensureInitialStatusSent();
+		assert(this.#backlog, "setCaughtUp() called but subscriber is not in catchup mode");
+		for (const change of this.#backlog) this.#sendChange(change);
+		this.#backlog = null;
+	}
+	async #sendChange(change) {
+		const [watermark, downstream] = change;
+		if (watermark <= this.watermark) return;
+		if (!this.supportsMessage(downstream[1])) return;
+		if (downstream[0] === "commit") this.#watermark = watermark;
+		const result = await this.#sendDownstream(downstream);
+		if (downstream[0] === "commit" && result === "consumed") this.#acked = max(this.#acked, watermark);
+	}
+	async #sendDownstream(downstream) {
+		this.#pending++;
+		const { result } = this.#downstream.push(downstream);
+		try {
+			return await result;
+		} finally {
+			this.#pending--;
+			this.#processed++;
+		}
+	}
+	#pending = 0;
+	#processed = 0;
+	#samples = [{
+		processed: 0,
+		timestamp: performance.now()
+	}];
+	/**
+	* The number of downstream messages that have yet to be acked.
+	*/
+	get numPending() {
+		return this.#pending;
+	}
+	/**
+	* The total number of downstream messages that the subscriber has
+	* processed (i.e. acked).
+	*/
+	get numProcessed() {
+		return this.#processed;
+	}
+	/**
+	* Records a new history entry for the number of messages processed,
+	* keeping the number of samples bounded to `maxSamples`.
+	*/
+	sampleProcessRate(now, maxSamples = 10) {
+		while (this.#samples.length >= maxSamples) this.#samples.shift();
+		this.#samples.push({
+			processed: this.#processed,
+			timestamp: now
+		});
+		return this;
+	}
+	getStats() {
+		const pending = this.#pending;
+		if (this.#samples.length < 2) return {
+			processRate: 0,
+			pending
+		};
+		const from = this.#samples[0];
+		const to = must(this.#samples.at(-1));
+		const processed = to.processed - from.processed;
+		const seconds = (to.timestamp - from.timestamp) / 1e3;
+		return {
+			processRate: seconds === 0 ? 0 : processed / seconds,
+			pending
+		};
+	}
+	supportsMessage(change) {
+		switch (change.tag) {
+			case "update-table-metadata": return this.#protocolVersion >= 5;
+		}
+		return true;
+	}
+	fail(err) {
+		this.close(0, String(err));
+	}
+	close(error, message) {
+		if (error) {
+			const { result } = this.#downstream.push(["error", {
+				type: error,
+				message
+			}]);
+			result.then(() => this.#downstream.cancel());
+		} else this.#downstream.cancel();
+	}
 };
-//# sourceMappingURL=subscriber.js.map
+//#endregion
+export { Subscriber };
+
+//# sourceMappingURL=subscriber.js.map

package/out/zero-cache/src/services/change-streamer/subscriber.js.map

@@ -1 +1 @@
-{"version":3,"file":"subscriber.js","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type Downstream} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly #downstream: Subscription<Downstream>;\n  #watermark: string;\n  #acked: string;\n  #backlog: WatermarkedChange[] | null;\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    watermark: string,\n    downstream: Subscription<Downstream>,\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.#downstream = downstream;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = [];\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  async send(change: WatermarkedChange) {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        this.#backlog.push(change);\n      } else {\n        await this.#send(change);\n      }\n    }\n  }\n\n  #initialStatusSent = false;\n\n  #ensureInitialStatusSent() {\n    if (this.#protocolVersion >= 2 && !this.#initialStatusSent) {\n      this.#downstream.push(['status', {tag: 'status'}]);\n      this.#initialStatusSent = true;\n    }\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#ensureInitialStatusSent();\n    await this.#send(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp() {\n    this.#ensureInitialStatusSent();\n    assert(\n      this.#backlog,\n      'setCaughtUp() called but subscriber is not in catchup mode',\n    );\n    // Note that this method must be asynchronous in order for send() to\n    // interpret the #backlog variable correctly. This is the only place\n    // where I/O flow control is not heeded. However, it will be awaited\n    // by the next caller to send().\n    for (const change of this.#backlog) {\n      void this.#send(change);\n    }\n    this.#backlog = null;\n  }\n\n  async #send(change: WatermarkedChange) {\n    const [watermark, downstream] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(downstream[1])) {\n      return;\n    }\n    const pending = this.#downstream.push(downstream);\n    if (downstream[0] === 'commit') {\n      this.#watermark = watermark;\n      void pending.result.then(val => {\n        if (val === 'consumed') {\n          this.#acked = max(this.#acked, watermark);\n        }\n      });\n    }\n    await pending.result;\n  }\n\n  supportsMessage(change: ChangeStreamData[1]) {\n    switch (change.tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  fail(err?: unknown) {\n    this.close(ErrorType.Unknown, String(err));\n  }\n\n  close(error?: ErrorType, message?: string) {\n    if (error) {\n      const {result} = this.#downstream.push(['error', {type: error, message}]);\n      // Wait for the ACK of the error message before closing the connection.\n      void result.then(() => this.#downstream.cancel());\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n}\n"],"names":["ErrorType.Unknown"],"mappings":";;;;AAkBO,MAAM,WAAW;AAAA,EACb;AAAA,EACA;AAAA,EACA;AAAA,EACT;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,iBACA,IACA,WACA,YACA;AACA,SAAK,mBAAmB;AACxB,SAAK,KAAK;AACV,SAAK,cAAc;AACnB,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,WAAW,CAAA;AAAA,EAClB;AAAA,EAEA,IAAI,YAAY;AACd,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,QAAQ;AACV,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,KAAK,QAA2B;AACpC,UAAM,CAAC,SAAS,IAAI;AACpB,QAAI,YAAY,KAAK,YAAY;AAC/B,UAAI,KAAK,UAAU;AACjB,aAAK,SAAS,KAAK,MAAM;AAAA,MAC3B,OAAO;AACL,cAAM,KAAK,MAAM,MAAM;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA,EAEA,qBAAqB;AAAA,EAErB,2BAA2B;AACzB,QAAI,KAAK,oBAAoB,KAAK,CAAC,KAAK,oBAAoB;AAC1D,WAAK,YAAY,KAAK,CAAC,UAAU,EAAC,KAAK,SAAA,CAAS,CAAC;AACjD,WAAK,qBAAqB;AAAA,IAC5B;AAAA,EACF;AAAA;AAAA,EAGA,MAAM,QAAQ,QAA2B;AACvC,SAAK,yBAAA;AACL,UAAM,KAAK,MAAM,MAAM;AAAA,EACzB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,cAAc;AACZ,SAAK,yBAAA;AACL;AAAA,MACE,KAAK;AAAA,MACL;AAAA,IAAA;AAMF,eAAW,UAAU,KAAK,UAAU;AAClC,WAAK,KAAK,MAAM,MAAM;AAAA,IACxB;AACA,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAM,MAAM,QAA2B;AACrC,UAAM,CAAC,WAAW,UAAU,IAAI;AAChC,QAAI,aAAa,KAAK,WAAW;AAC/B;AAAA,IACF;AACA,QAAI,CAAC,KAAK,gBAAgB,WAAW,CAAC,CAAC,GAAG;AACxC;AAAA,IACF;AACA,UAAM,UAAU,KAAK,YAAY,KAAK,UAAU;AAChD,QAAI,WAAW,CAAC,MAAM,UAAU;AAC9B,WAAK,aAAa;AAClB,WAAK,QAAQ,OAAO,KAAK,CAAA,QAAO;AAC9B,YAAI,QAAQ,YAAY;AACtB,eAAK,SAAS,IAAI,KAAK,QAAQ,SAAS;AAAA,QAC1C;AAAA,MACF,CAAC;AAAA,IACH;AACA,UAAM,QAAQ;AAAA,EAChB;AAAA,EAEA,gBAAgB,QAA6B;AAC3C,YAAQ,OAAO,KAAA;AAAA,MACb,KAAK;AAEH,eAAO,KAAK,oBAAoB;AAAA,IAAA;AAEpC,WAAO;AAAA,EACT;AAAA,EAEA,KAAK,KAAe;AAClB,SAAK,MAAMA,SAAmB,OAAO,GAAG,CAAC;AAAA,EAC3C;AAAA,EAEA,MAAM,OAAmB,SAAkB;AACzC,QAAI,OAAO;AACT,YAAM,EAAC,OAAA,IAAU,KAAK,YAAY,KAAK,CAAC,SAAS,EAAC,MAAM,OAAO,QAAA,CAAQ,CAAC;AAExE,WAAK,OAAO,KAAK,MAAM,KAAK,YAAY,QAAQ;AAAA,IAClD,OAAO;AACL,WAAK,YAAY,OAAA;AAAA,IACnB;AAAA,EACF;AACF;"}
+{"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#watermark","#acked","#backlog","#sendChange","#initialStatusSent","#sendDownstream","#ensureInitialStatusSent","#pending","#processed","#samples"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type Downstream} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly #downstream: Subscription<Downstream>;\n  #watermark: string;\n  #acked: string;\n  #backlog: WatermarkedChange[] | null;\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    watermark: string,\n    downstream: Subscription<Downstream>,\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.#downstream = downstream;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = [];\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  async send(change: WatermarkedChange) {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        this.#backlog.push(change);\n      } else {\n        await this.#sendChange(change);\n      }\n    }\n  }\n\n  #initialStatusSent = false;\n\n  #ensureInitialStatusSent() {\n    if (this.#protocolVersion >= 2 && !this.#initialStatusSent) {\n      void this.#sendDownstream(['status', {tag: 'status'}]);\n      this.#initialStatusSent = true;\n    }\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#ensureInitialStatusSent();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp() {\n    this.#ensureInitialStatusSent();\n    assert(\n      this.#backlog,\n      'setCaughtUp() called but subscriber is not in catchup mode',\n    );\n    // Note that this method must be asynchronous in order for send() to\n    // interpret the #backlog variable correctly. This is the only place\n    // where I/O flow control is not heeded. However, it will be awaited\n    // by the next caller to send().\n    for (const change of this.#backlog) {\n      void this.#sendChange(change);\n    }\n    this.#backlog = null;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, downstream] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(downstream[1])) {\n      return;\n    }\n    if (downstream[0] === 'commit') {\n      this.#watermark = watermark;\n    }\n    const result = await this.#sendDownstream(downstream);\n    if (downstream[0] === 'commit' && result === 'consumed') {\n      this.#acked = max(this.#acked, watermark);\n    }\n  }\n\n  async #sendDownstream(downstream: Downstream) {\n    this.#pending++;\n    const {result} = this.#downstream.push(downstream);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#processed++;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): {processRate: number; pending: number} {\n    const pending = this.#pending;\n    if (this.#samples.length < 2) {\n      return {processRate: 0, pending};\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {processRate, pending};\n  }\n\n  supportsMessage(change: ChangeStreamData[1]) {\n    switch (change.tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  fail(err?: unknown) {\n    this.close(ErrorType.Unknown, String(err));\n  }\n\n  close(error?: ErrorType, message?: string) {\n    if (error) {\n      const {result} = this.#downstream.push(['error', {type: error, message}]);\n      // Wait for the ACK of the error message before closing the connection.\n      void result.then(() => this.#downstream.cancel());\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAmBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,iBACA,IACA,WACA,YACA;AACA,QAAA,kBAAwB;AACxB,OAAK,KAAK;AACV,QAAA,aAAmB;AACnB,QAAA,YAAkB;AAClB,QAAA,QAAc;AACd,QAAA,UAAgB,EAAE;;CAGpB,IAAI,YAAY;AACd,SAAO,MAAA;;CAGT,IAAI,QAAQ;AACV,SAAO,MAAA;;CAGT,MAAM,KAAK,QAA2B;EACpC,MAAM,CAAC,aAAa;AACpB,MAAI,YAAY,MAAA,UACd,KAAI,MAAA,QACF,OAAA,QAAc,KAAK,OAAO;MAE1B,OAAM,MAAA,WAAiB,OAAO;;CAKpC,qBAAqB;CAErB,2BAA2B;AACzB,MAAI,MAAA,mBAAyB,KAAK,CAAC,MAAA,mBAAyB;AACrD,SAAA,eAAqB,CAAC,UAAU,EAAC,KAAK,UAAS,CAAC,CAAC;AACtD,SAAA,oBAA0B;;;;CAK9B,MAAM,QAAQ,QAA2B;AACvC,QAAA,yBAA+B;AAC/B,QAAM,MAAA,WAAiB,OAAO;;;;;;CAOhC,cAAc;AACZ,QAAA,yBAA+B;AAC/B,SACE,MAAA,SACA,6DACD;AAKD,OAAK,MAAM,UAAU,MAAA,QACd,OAAA,WAAiB,OAAO;AAE/B,QAAA,UAAgB;;CAGlB,OAAA,WAAkB,QAA2B;EAC3C,MAAM,CAAC,WAAW,cAAc;AAChC,MAAI,aAAa,KAAK,UACpB;AAEF,MAAI,CAAC,KAAK,gBAAgB,WAAW,GAAG,CACtC;AAEF,MAAI,WAAW,OAAO,SACpB,OAAA,YAAkB;EAEpB,MAAM,SAAS,MAAM,MAAA,eAAqB,WAAW;AACrD,MAAI,WAAW,OAAO,YAAY,WAAW,WAC3C,OAAA,QAAc,IAAI,MAAA,OAAa,UAAU;;CAI7C,OAAA,eAAsB,YAAwB;AAC5C,QAAA;EACA,MAAM,EAAC,WAAU,MAAA,WAAiB,KAAK,WAAW;AAClD,MAAI;AACF,UAAO,MAAM;YACL;AACR,SAAA;AACA,SAAA;;;CAYJ,WAAW;CACX,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,KAAK;EAAC,CAC7C;;;;CAKD,IAAI,aAAa;AACf,SAAO,MAAA;;;;;;CAOT,IAAI,eAAe;AACjB,SAAO,MAAA;;;;;;CAOT,kBAAkB,KAAa,aAAa,IAAU;AACpD,SAAO,MAAA,QAAc,UAAU,WAC7B,OAAA,QAAc,OAAO;AAEvB,QAAA,QAAc,KAAK;GAAC,WAAW,MAAA;GAAiB,WAAW;GAAI,CAAC;AAChE,SAAO;;CAGT,WAAmD;EACjD,MAAM,UAAU,MAAA;AAChB,MAAI,MAAA,QAAc,SAAS,EACzB,QAAO;GAAC,aAAa;GAAG;GAAQ;EAElC,MAAM,OAAO,MAAA,QAAc;EAC3B,MAAM,KAAK,KAAK,MAAA,QAAc,GAAG,GAAG,CAAC;EACrC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;AAElD,SAAO;GAAC,aADY,YAAY,IAAI,IAAI,YAAY;GAC/B;GAAQ;;CAG/B,gBAAgB,QAA6B;AAC3C,UAAQ,OAAO,KAAf;GACE,KAAK,wBAEH,QAAO,MAAA,mBAAyB;;AAEpC,SAAO;;CAGT,KAAK,KAAe;AAClB,OAAK,MAAM,GAAmB,OAAO,IAAI,CAAC;;CAG5C,MAAM,OAAmB,SAAkB;AACzC,MAAI,OAAO;GACT,MAAM,EAAC,WAAU,MAAA,WAAiB,KAAK,CAAC,SAAS;IAAC,MAAM;IAAO;IAAQ,CAAC,CAAC;AAEpE,UAAO,WAAW,MAAA,WAAiB,QAAQ,CAAC;QAEjD,OAAA,WAAiB,QAAQ"}

package/out/zero-cache/src/services/heapz.js

@@ -1,25 +1,23 @@
+import { isAdminPasswordValid } from "../config/zero-config.js";
 import auth from "basic-auth";
 import fs from "fs";
 import v8 from "v8";
-import { isAdminPasswordValid } from "../config/zero-config.js";
+//#region ../zero-cache/src/services/heapz.ts
 function handleHeapzRequest(lc, config, req, res) {
-  const credentials = auth(req);
-  if (!isAdminPasswordValid(lc, config, credentials?.pass)) {
-    void res.code(401).header("WWW-Authenticate", 'Basic realm="Heapz Protected Area"').send("Unauthorized");
-    return;
-  }
-  const filename = v8.writeHeapSnapshot();
-  const stream = fs.createReadStream(filename);
-  void res.header("Content-Type", "application/octet-stream").header("Content-Disposition", `attachment; filename=${filename}`).send(stream);
-  stream.on("end", () => {
-    fs.unlink(filename, (err) => {
-      if (err) {
-        lc.error?.("Error deleting heap snapshot:", err);
-      }
-    });
-  });
+	if (!isAdminPasswordValid(lc, config, auth(req)?.pass)) {
+		res.code(401).header("WWW-Authenticate", "Basic realm=\"Heapz Protected Area\"").send("Unauthorized");
+		return;
+	}
+	const filename = v8.writeHeapSnapshot();
+	const stream = fs.createReadStream(filename);
+	res.header("Content-Type", "application/octet-stream").header("Content-Disposition", `attachment; filename=${filename}`).send(stream);
+	stream.on("end", () => {
+		fs.unlink(filename, (err) => {
+			if (err) lc.error?.("Error deleting heap snapshot:", err);
+		});
+	});
 }
-export {
-  handleHeapzRequest
-};
-//# sourceMappingURL=heapz.js.map
+//#endregion
+export { handleHeapzRequest };
+
+//# sourceMappingURL=heapz.js.map

package/out/zero-cache/src/services/heapz.js.map

@@ -1 +1 @@
-{"version":3,"file":"heapz.js","sources":["../../../../../zero-cache/src/services/heapz.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport fs from 'fs';\nimport v8 from 'v8';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {isAdminPasswordValid} from '../config/zero-config.ts';\n\nexport function handleHeapzRequest(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  const credentials = auth(req);\n  if (!isAdminPasswordValid(lc, config, credentials?.pass)) {\n    void res\n      .code(401)\n      .header('WWW-Authenticate', 'Basic realm=\"Heapz Protected Area\"')\n      .send('Unauthorized');\n    return;\n  }\n\n  const filename = v8.writeHeapSnapshot();\n  const stream = fs.createReadStream(filename);\n  void res\n    .header('Content-Type', 'application/octet-stream')\n    .header('Content-Disposition', `attachment; filename=${filename}`)\n    .send(stream);\n\n  // Clean up temp file after streaming\n  stream.on('end', () => {\n    fs.unlink(filename, err => {\n      if (err) {\n        lc.error?.('Error deleting heap snapshot:', err);\n      }\n    });\n  });\n}\n"],"names":[],"mappings":";;;;AAQO,SAAS,mBACd,IACA,QACA,KACA,KACA;AACA,QAAM,cAAc,KAAK,GAAG;AAC5B,MAAI,CAAC,qBAAqB,IAAI,QAAQ,aAAa,IAAI,GAAG;AACxD,SAAK,IACF,KAAK,GAAG,EACR,OAAO,oBAAoB,oCAAoC,EAC/D,KAAK,cAAc;AACtB;AAAA,EACF;AAEA,QAAM,WAAW,GAAG,kBAAA;AACpB,QAAM,SAAS,GAAG,iBAAiB,QAAQ;AAC3C,OAAK,IACF,OAAO,gBAAgB,0BAA0B,EACjD,OAAO,uBAAuB,wBAAwB,QAAQ,EAAE,EAChE,KAAK,MAAM;AAGd,SAAO,GAAG,OAAO,MAAM;AACrB,OAAG,OAAO,UAAU,CAAA,QAAO;AACzB,UAAI,KAAK;AACP,WAAG,QAAQ,iCAAiC,GAAG;AAAA,MACjD;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AACH;"}
+{"version":3,"file":"heapz.js","names":[],"sources":["../../../../../zero-cache/src/services/heapz.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport fs from 'fs';\nimport v8 from 'v8';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {isAdminPasswordValid} from '../config/zero-config.ts';\n\nexport function handleHeapzRequest(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  const credentials = auth(req);\n  if (!isAdminPasswordValid(lc, config, credentials?.pass)) {\n    void res\n      .code(401)\n      .header('WWW-Authenticate', 'Basic realm=\"Heapz Protected Area\"')\n      .send('Unauthorized');\n    return;\n  }\n\n  const filename = v8.writeHeapSnapshot();\n  const stream = fs.createReadStream(filename);\n  void res\n    .header('Content-Type', 'application/octet-stream')\n    .header('Content-Disposition', `attachment; filename=${filename}`)\n    .send(stream);\n\n  // Clean up temp file after streaming\n  stream.on('end', () => {\n    fs.unlink(filename, err => {\n      if (err) {\n        lc.error?.('Error deleting heap snapshot:', err);\n      }\n    });\n  });\n}\n"],"mappings":";;;;;AAQA,SAAgB,mBACd,IACA,QACA,KACA,KACA;AAEA,KAAI,CAAC,qBAAqB,IAAI,QADV,KAAK,IAAI,EACsB,KAAK,EAAE;AACnD,MACF,KAAK,IAAI,CACT,OAAO,oBAAoB,uCAAqC,CAChE,KAAK,eAAe;AACvB;;CAGF,MAAM,WAAW,GAAG,mBAAmB;CACvC,MAAM,SAAS,GAAG,iBAAiB,SAAS;AACvC,KACF,OAAO,gBAAgB,2BAA2B,CAClD,OAAO,uBAAuB,wBAAwB,WAAW,CACjE,KAAK,OAAO;AAGf,QAAO,GAAG,aAAa;AACrB,KAAG,OAAO,WAAU,QAAO;AACzB,OAAI,IACF,IAAG,QAAQ,iCAAiC,IAAI;IAElD;GACF"}